whitening,gaussians, clt

notes/06_fastica/2_pcaica.tex

@@ -73,6 +73,8 @@ \subsection{Whitening in the context of ICA}
 \begin{center}
 	\includegraphics[width=0.4\textwidth]{img/meme_newalwaysbetter}%
 \end{center}
+
+- $\widetilde{\vec A}$ is orthogonal. Not necessarily the case for the original mixing matrix $\vec A$.
 }
 
 \end{frame}
@@ -183,6 +185,7 @@ \subsection{Whitening in the context of ICA}
 }\slidesonly{:\\
 $$N^2 \rightarrow N(N-1)/2$$
 }
+
 \end{frame}
 
 \begin{frame}
@@ -217,7 +220,7 @@ \subsection{Whitening in the context of ICA}
 \begin{itemize}
 \item[-] new ICA problem: $\vec u := \widetilde{\vec A}\, \vec s$\\
 \item[-] $\widetilde{\vec A}$ is orthogonal\\
-\item[-] unmixing the new problem involves only ``half'' the number of weights.
+\item[-] unmixing the new problem involves only ``half'' the number of free parameters.
 \end{itemize}
 
 \question{What is the transformation $\vec D^{-\frac{1}{2}} \vec U^\top$ called?}
@@ -229,6 +232,91 @@ \subsection{Whitening in the context of ICA}
 
 \end{frame}
 
+\subsubsection{Whitening solves half of the ICA problem}
+
+\begin{frame}{\subsubsecname}
+
+\slidesonly{
+
+Unmixing the new problem involves only ``half'' the number of free parameters.
+$$N^2 \rightarrow N(N-1)/2$$
+}
+
+\pause
+
+\svspace{-5mm}
+
+\question{Does this mean that $\vec W$ is no longer $N \times N$?}
+
+-No $\vec W$ is still $N \times N$, but we no longer have to search for each component individually.
+
+\end{frame}
+
+\begin{frame}{\subsubsecname}
+
+Example with $N=2$ uniformly distributed sources $s_1, s_2$:
+
+ \begin{center}
+  \begin{minipage}{0.29\textwidth}
+\includegraphics[width=0.99\textwidth]{./img/uniform_original_sources} 
+  \end{minipage}
+  \hspace{5mm}
+  \begin{minipage}{0.29\textwidth}
+\includegraphics[width=0.99\textwidth]{./img/uniform_mixtures_centered} 
+  \end{minipage}\\
+
+  \begin{minipage}{0.29\textwidth}
+\includegraphics[width=0.99\textwidth]{./img/uniform_mixtures_decorrelated} 
+  \end{minipage}
+  \hspace{5mm}
+  \begin{minipage}{0.29\textwidth}
+\includegraphics[width=0.99\textwidth]{./img/uniform_mixtures_whitened} 
+  \end{minipage}
+  \notesonly{
+  \captionof{figure}{Example with $N=2$ uniformly distributed sources $s_1, s_2$}
+  }
+\end{center}
+
+\end{frame}
+
+\begin{frame}{\subsubsecname}
+
+\slidesonly{
+Example with $N=2$ uniformly distributed sources $s_1, s_2$:
+
+ \begin{center}
+  \begin{minipage}{0.25\textwidth}
+\includegraphics[width=0.99\textwidth]{./img/uniform_original_sources} 
+  \end{minipage}
+  \hspace{5mm}
+  \begin{minipage}{0.25\textwidth}
+\includegraphics[width=0.99\textwidth]{./img/uniform_mixtures_whitened} 
+  \end{minipage}
+\end{center}
+}
+
+ICA on whitened mixtures is about ``rotating it back''.
+
+2D data can be rotated by an angle $\theta$ using:
+
+\begin{align}
+  \vec{x}_{\theta} & = 
+  \begin{pmatrix}
+	\cos(\theta) & -\sin(\theta) \\
+	\sin(\theta) & \cos(\theta)
+  \end{pmatrix}
+  \vec{x} \qquad \text{with} \quad \theta = 0, \, \ldots\, , 2\pi
+\end{align}
+
+\pause
+
+Find the right rotation for \textcolor{blue}{2}D data involves a \textcolor{red}{single} free parameter.
+\begin{equation}
+\# \text{free parameters} = \textcolor{blue}{N} \cdot (\textcolor{blue}{N}-1)/2 = \textcolor{blue}{2} \cdot (\textcolor{blue}{2}-1)/2 = \textcolor{red}{1}
+\end{equation}
+
+\end{frame}
+
 \slidesonly{
 \begin{frame}{Summary so far}
 \begin{enumerate}
@@ -240,7 +328,7 @@ \subsection{Whitening in the context of ICA}
 \item $\vec D$ and $\vec U$ can be obtained via PCA on $\vec x$.
 \item Applying ICA on whitened data reduces the number of free parameters.
 \item PCA simplifies the ICA problem.
-\item ICA on whitened data is about ``rotating'' it back.
+\item ICA on whitened data is about ``rotating it back''.
 \end{enumerate}
 
 \end{frame}

notes/06_fastica/3_badgaussians.tex

@@ -1,8 +1,20 @@
 \subsection{Gaussians are bad for ICA}
 
-\subsubsection{A more formal argument for why Gaussians are bad for ICA}
+\mode<presentation>{
+\begin{frame} 
+    \begin{center} \huge
+        \subsecname
+    \end{center}
+    \begin{center}
+    \includegraphics[width=0.4\textwidth]{./img/meme_breakingicabadgaussian}\\
+    And \textbf{white}ning won't help either.
+    \end{center}
+\end{frame}
+}
+
+\subsubsection{A formal argument for why Gaussians are bad for ICA}
 
-\begin{frame}{\subsecname}
+\begin{frame}{\subsubsecname}
 
 Recall that the joint density of independent sources is a factorizing density:
 
@@ -33,7 +45,7 @@ \subsubsection{A more formal argument for why Gaussians are bad for ICA}
 \end{equation}
 \end{frame}
 
-\begin{frame}{\subsecname (cont'd)}
+\begin{frame}{\subsecname~(cont'd)}
 
 %\slidesonly{\textbf{A more formal argument (cont'd):}}
 
@@ -153,19 +165,27 @@ \subsubsection{A more formal argument for why Gaussians are bad for ICA}
 \svspace{5mm}
 
 \begin{itemize}
-\item Mixing two independent Gaussians leads to a joint mixed distribution ${P}_{\vec x}(\vec x)$ that is effectively equal to the joint distribution of the original sources ${P}_{\vec s}(\vec s)$.
+\item Mixing two independent Gaussians leads to a joint mixed distribution ${P}_{\vec x}(\vec x)$ that is effectively equal to the joint distribution of the original sources ${P}_{\vec s}(\vec s)$.\\
+
 \pause
+
 \item No surprise: \emph{uncorrelated jointly Gaussian variables are necessarily independent.}
 \pause
-\item One Gaussian + other distributions is fine.
+
+\pause 
+
+\item One Gaussian + other distribution(s) is fine.
 \item Two ore more Gaussians. No way.
 \end{itemize}
 \end{minipage}
 }
 
 \slidesonly{
-\only<4>{
+\only<2>{
 	\placeimage{10.5}{6}{img/meme_icagaussian}{width=4cm}%
 }
+\only<4>{
+	\placeimage{10.5}{6}{img/meme_nomoregaussians}{width=4cm}%
+}
 }
 \end{frame}

notes/06_fastica/4_kurt.tex

@@ -14,18 +14,30 @@ \section{ICA by maximizing nongaussianity}
 \begin{frame}{Maximizing nongaussianity}
 
 \begin{block}{Intuition from the Central Limit Theorem}
-\emph{The distribution of the sum of independent random variables is ''more Gaussian'' than the original distributions of the random variables.}\\\vspace{2mm}
+\emph{The distribution of the sum of independent random variables is ``more Gaussian'' than the original distributions of the random variables.}\\\vspace{2mm}
 Searching for the direction of maximum deviation from a Gaussian distribution may recover the original sources.
 \end{block}
-\end{frame}
-
 
+\begin{center}
+    \includegraphics<2>[width=0.8\textwidth]{./img/clt_uniform_0}
+    \includegraphics<3>[width=0.8\textwidth]{./img/clt_uniform_1}
+    \includegraphics<4>[width=0.8\textwidth]{./img/clt_uniform_2}
+    \includegraphics<5>[width=0.8\textwidth]{./img/clt_uniform_3}
+    \includegraphics<6>[width=0.8\textwidth]{./img/clt_uniform_4}
+    \notesonly{
+    \captionof{figure}{The sum of independent random variables is ``more Gaussian'' than the original distributions.}
+    }
+\end{center}
+
+\end{frame}
 
 \begin{frame}{The setting}
 
-\textbf{The setting:}\\
+\notesonly{\textbf{The setting:}\\}
 
-Two statistically independent sources with $\langle s_i s_j \rangle = \delta_{ij} \quad \Leftrightarrow \quad \langle \vec s \, \vec s^\top \rangle = \vec I_N$
+Two statistically independent sources with
+\begin{equation}\langle s_i s_j \rangle = \delta_{ij} \quad \forall i,j=1,\ldots,N \quad \Leftrightarrow \quad \langle \vec s \, \vec s^\top \rangle = \vec I_N
+\end{equation}
 \notesonly{
 The sources are mixed using a mixing matrix $\vec A$ resulting in observations $\vec x$:
 }
@@ -72,15 +84,15 @@ \section{ICA by maximizing nongaussianity}
 This can be accomplished with a vector containing a single non-zero element:
 }
 
-\begin{equation*}
+\begin{equation}
 	\vec{z}_{\text{opt.}} = \left( \begin{array}{c}
 			0 \\ \pm 1 
 		\end{array} \right)
 	\quad \text{ or }\quad 
 	\vec{z}_{\text{opt.}} = \left( \begin{array}{c}
 			\pm 1 \\ 0 
 		\end{array} \right)
-\end{equation*}
+\end{equation}
 
 \end{frame}
 
@@ -91,57 +103,72 @@ \section{ICA by maximizing nongaussianity}
 We cannot have both independent sources contribute to $\widehat{s}_i$, only one can. Therefore, we only need a single non-zero component for $\vec z_i$.
 Wether $s_1$ is scaled by any factor before reaching $\widehat{s}_i$ does not make it more or less independent of $s_2$. Choosing $1$ for the non-zero component is therefore sufficient.
 Finally, negating the source by multiplying it by $(-1)$ also has no consequences on the independence criterion.
-
-We won't actually try and find $\vec z_i$ because we don't have $\vec s$ to apply them to. We use the requirements for $\vec z_i$ by finding a $\vec w_i$ that satisfies these requirements through:
-\begin{equation}
-\label{eq:zfromw}
-\vec z_i = \left(\vec w_i^\top \vec A\right)^\top =  \vec A^\top \vec w_i
-\end{equation}
 }
 
-\begin{frame}
-\question{Does maximizing nongaussianity deliver independent components?}
-\slidesonly{
 
-\begin{equation}
-\widehat s_i = \vec z_i^{\top} \vec s = z_1 s_1 + z_2 s_2
-\end{equation}
+%\begin{frame}
+%\question{Does maximizing nongaussianity deliver independent components?}
+%\slidesonly{
 
-\begin{equation*}
-	\vec{z}_{\text{opt.}} = \left( \begin{array}{c}
-			0 \\ \pm 1 
-		\end{array} \right)
-	\quad \text{ or }\quad 
-	\vec{z}_{\text{opt.}} = \left( \begin{array}{c}
-			\pm 1 \\ 0 
-		\end{array} \right)
-\end{equation*}
+%\begin{equation}
+%\widehat s_i = \vec z_i^{\top} \vec s = z_1 s_1 + z_2 s_2
+%\end{equation}
 
-$\vec z_i$ ensures independent $\widehat s_i$
+%\begin{equation*}
+	%\vec{z}_{\text{opt.}} = \left( \begin{array}{c}
+			%0 \\ \pm 1 
+		%\end{array} \right)
+	%\quad \text{ or }\quad 
+	%\vec{z}_{\text{opt.}} = \left( \begin{array}{c}
+			%\pm 1 \\ 0 
+		%\end{array} \right)
+%\end{equation*}
+
+%$\vec z_i$ ensures independent $\widehat s_i$
 
-\begin{equation}
-\widehat s_i = \vec w_i^\top \vec x
-\end{equation}
 
-$\vec w_i$ finds less gaussian $\widehat s_i$
 
+%\begin{equation}
+%\widehat s_i = \vec w_i^\top \vec x
+%\end{equation}
+
+%$\vec w_i$ finds less gaussian $\widehat s_i$
+
+\notesonly{
+
+We won't actually try and find $\vec z_i$ because we don't have $\vec s$ to apply them to. We use the requirements for $\vec z_i$ by finding a $\vec w_i$ that satisfies these requirements through:
 \begin{equation}
-\vec z_i =  \vec A^\top \vec w_i
+\label{eq:zfromw}
+\vec z_i = \left(\vec w_i^\top \vec A\right)^\top =  \vec A^\top \vec w_i
 \end{equation}
+}
 
-Maximizing nongaussianity is ensured to keep $\widehat s_i$ independent.
+%Maximizing nongaussianity is ensured to keep $\widehat s_i$ independent.
 
-}
-\end{frame}
+%}
+%\end{frame}
 \notesonly{
 - By (1) maximizing the nongaussianity of $\vec w_i^\top \vec x$ and (2) having $\vec z_i = \vec A^\top \vec w_i$ yield independent components and (3) knowing that 
 $\widehat s_i = \vec w_i^\top \vec x = \vec z_i^{\top} \vec s$, we conclude that maximizing $\vec w_i^\top \vec x$ gives us one independent component.
 }
 \newpage
 
+\mode<presentation>{
+\begin{frame}
+
+    \begin{center}
+    Deciding on a measure for nongaussianity
+    \end{center}    
+    \begin{center}
+    \includegraphics[width=0.4\textwidth]{./img/meme_thismuchgaussian}
+    \end{center}
+
+\end{frame}
+}
+
 \section{Kurtosis as a measure for nongaussianity}
 
-\begin{frame}
+\begin{frame}{\secname}
 
 \notesonly{
 Kurtosis represents the fourth-order cumulant\footnote{
@@ -175,7 +202,7 @@ \section{Kurtosis as a measure for nongaussianity}
 
 \begin{frame}
 
-\question{What does kurtosis measure?}\\
+\question{How do we interpret the kurtosis measure?}\\
 
 \begin{tabular}[h]{c c c c}
 &