Parametric t-SNE (Keras).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Currently:\n",
    "* training the original network (500-500-2000-2) on mnist with tsne loss\n",
    "* training the original network on a pre-embedded set of 2d points with mse loss\n",
    "* training a convolution net on a pre-embedded set of 2d points with mse loss\n",
    "\n",
    "Next:\n",
    "* check tsne loss on test set for all three models\n",
    "* if we lower the learning rate in response to test set loss can we acheive lower final loss?\n",
    "* what architecture allows us to minimize the number of parameters while also minimizing test tsne loss?\n",
    "* does dropout have an effect on the generalization of any of the models?\n",
    "* can we \"deep dream\" a point in 2d space to recover an input that produces it?\n",
    "* what happens if we permute the input data but use the same joint probabilities?\n",
    "* what if we instead use P from an unsupervised mid-dimensional representation like an autoencoder or VAE? (e.g., 48D)\n",
    "* what if we use t-SNE loss as a contributing cost function to an autoencoder?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['THEANO_FLAGS'] = 'device=gpu1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using gpu device 1: GeForce GTX TITAN X (CNMeM is disabled)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers.convolutional import Convolution2D, MaxPooling2D\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils\n",
    "from keras.objectives import categorical_crossentropy\n",
    "from keras.datasets import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 976 ms, sys: 130 ms, total: 1.11 s\n",
      "Wall time: 1.1 s\n",
      "(60000, 'train samples')\n",
      "(10000, 'test samples')\n"
     ]
    }
   ],
   "source": [
    "batch_size = 5000\n",
    "\n",
    "%time (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "X_train = X_train.reshape(-1, 784)\n",
    "X_test = X_test.reshape(-1, 784)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Hbeta(D, beta):\n",
    "    P = np.exp(-D * beta)\n",
    "    sumP = np.sum(P)\n",
    "    H = np.log(sumP) + beta * np.sum(np.multiply(D, P)) / sumP\n",
    "    P = P / sumP\n",
    "    return H, P\n",
    "\n",
    "def x2p(X, u=15, tol=1e-4, print_iter=500, max_tries=50, verbose=0):\n",
    "    # Initialize some variables\n",
    "    n = X.shape[0]                     # number of instances\n",
    "    P = np.zeros((n, n))               # empty probability matrix\n",
    "    beta = np.ones(n)                  # empty precision vector\n",
    "    logU = np.log(u)                   # log of perplexity (= entropy)\n",
    "    \n",
    "    # Compute pairwise distances\n",
    "    if verbose > 0: print('Computing pairwise distances...')\n",
    "    sum_X = np.sum(np.square(X), axis=1)\n",
    "    # note: translating sum_X' from matlab to numpy means using reshape to add a dimension\n",
    "    D = sum_X + sum_X[:,None] + -2 * X.dot(X.T)\n",
    "\n",
    "    # Run over all datapoints\n",
    "    if verbose > 0: print('Computing P-values...')\n",
    "    for i in range(n):\n",
    "        \n",
    "        if verbose > 1 and print_iter and i % print_iter == 0:\n",
    "            print('Computed P-values {} of {} datapoints...'.format(i, n))\n",
    "        \n",
    "        # Set minimum and maximum values for precision\n",
    "        betamin = float('-inf')\n",
    "        betamax = float('+inf')\n",
    "        \n",
    "        # Compute the Gaussian kernel and entropy for the current precision\n",
    "        indices = np.concatenate((np.arange(0, i), np.arange(i + 1, n)))\n",
    "        Di = D[i, indices]\n",
    "        H, thisP = Hbeta(Di, beta[i])\n",
    "        \n",
    "        # Evaluate whether the perplexity is within tolerance\n",
    "        Hdiff = H - logU\n",
    "        tries = 0\n",
    "        while abs(Hdiff) > tol and tries < max_tries:\n",
    "            \n",
    "            # If not, increase or decrease precision\n",
    "            if Hdiff > 0:\n",
    "                betamin = beta[i]\n",
    "                if np.isinf(betamax):\n",
    "                    beta[i] *= 2\n",
    "                else:\n",
    "                    beta[i] = (beta[i] + betamax) / 2\n",
    "            else:\n",
    "                betamax = beta[i]\n",
    "                if np.isinf(betamin):\n",
    "                    beta[i] /= 2\n",
    "                else:\n",
    "                    beta[i] = (beta[i] + betamin) / 2\n",
    "            \n",
    "            # Recompute the values\n",
    "            H, thisP = Hbeta(Di, beta[i])\n",
    "            Hdiff = H - logU\n",
    "            tries += 1\n",
    "        \n",
    "        # Set the final row of P\n",
    "        P[i, indices] = thisP\n",
    "        \n",
    "    if verbose > 0: \n",
    "        print('Mean value of sigma: {}'.format(np.mean(np.sqrt(1 / beta))))\n",
    "        print('Minimum value of sigma: {}'.format(np.min(np.sqrt(1 / beta))))\n",
    "        print('Maximum value of sigma: {}'.format(np.max(np.sqrt(1 / beta))))\n",
    "    \n",
    "    return P, beta\n",
    "\n",
    "def compute_joint_probabilities(samples, batch_size=5000, d=2, perplexity=30, tol=1e-5, verbose=0):\n",
    "    v = d - 1\n",
    "    \n",
    "    # Initialize some variables\n",
    "    n = samples.shape[0]\n",
    "    batch_size = min(batch_size, n)\n",
    "    \n",
    "    # Precompute joint probabilities for all batches\n",
    "    if verbose > 0: print('Precomputing P-values...')\n",
    "    batch_count = int(n / batch_size)\n",
    "    P = np.zeros((batch_count, batch_size, batch_size))\n",
    "    for i, start in enumerate(range(0, n - batch_size + 1, batch_size)):   \n",
    "        curX = samples[start:start+batch_size]                   # select batch\n",
    "        P[i], beta = x2p(curX, perplexity, tol, verbose=verbose) # compute affinities using fixed perplexity\n",
    "        P[i][np.isnan(P[i])] = 0                                 # make sure we don't have NaN's\n",
    "        P[i] = (P[i] + P[i].T) # / 2                             # make symmetric\n",
    "        P[i] = P[i] / P[i].sum()                                 # obtain estimation of joint probabilities\n",
    "        P[i] = np.maximum(P[i], np.finfo(P[i].dtype).eps)\n",
    "\n",
    "    return P"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time P = compute_joint_probabilities(X_train, batch_size=batch_size, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time np.save('P.npy', P)\n",
    "# %time P = np.load('P.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# P is the joint probabilities for this batch (Keras loss functions call this y_true)\n",
    "# activations is the low-dimensional output (Keras loss functions call this y_pred)\n",
    "def tsne(P, activations):\n",
    "#     d = K.shape(activations)[1]\n",
    "    d = 2 # TODO: should set this automatically, but the above is very slow for some reason\n",
    "    n = batch_size # TODO: should set this automatically\n",
    "    v = d - 1.\n",
    "    eps = K.variable(10e-15) # needs to be at least 10e-8 to get anything after Q /= K.sum(Q)\n",
    "    sum_act = K.sum(K.square(activations), axis=1)\n",
    "    Q = K.reshape(sum_act, [-1, 1]) + -2 * K.dot(activations, K.transpose(activations))\n",
    "    Q = (sum_act + Q) / v\n",
    "    Q = K.pow(1 + Q, -(v + 1) / 2)\n",
    "    Q *= K.variable(1 - np.eye(n))\n",
    "    Q /= K.sum(Q)\n",
    "    Q = K.maximum(Q, eps)\n",
    "    C = K.log((P + eps) / (Q + eps))\n",
    "    C = K.sum(P * C)\n",
    "    return C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(500, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "model.add(Dense(500, activation='relu'))\n",
    "model.add(Dense(2000, activation='relu'))\n",
    "model.add(Dense(2))\n",
    "sgd = SGD(lr=0.1)\n",
    "%time model.compile(loss=tsne, optimizer=sgd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Y_train = P.reshape(X_train.shape[0], -1)\n",
    "print(X_train.shape)\n",
    "print(Y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%time model.fit(X_train, Y_train, batch_size=batch_size, shuffle=False, nb_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_model(embedding, labels):\n",
    "    fig = plt.figure(figsize=(8,8))\n",
    "    plt.scatter(embedding[:,0], embedding[:,1], marker='o', s=1, edgecolor='', c=labels)\n",
    "    fig.tight_layout()\n",
    "    \n",
    "from matplotlib.lines import Line2D\n",
    "def plot_differences(embedding, actual, lim=1000):\n",
    "    fig = plt.figure(figsize=(8,8))\n",
    "    ax = fig.gca()\n",
    "    for a, b in zip(embedding, actual)[:lim]:\n",
    "        ax.add_line(Line2D((a[0], b[0]), (a[1], b[1]), linewidth=1))\n",
    "    ax.autoscale_view()\n",
    "    plt.show()\n",
    "# plot_model(model.predict(data), y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tsne import bh_sne\n",
    "%time X_2d = bh_sne(X_train.astype(np.float64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 883 µs, sys: 757 µs, total: 1.64 ms\n",
      "Wall time: 1.11 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(60000, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# %time np.save('X_2d.npy', X_2d)\n",
    "# %time X_2d = np.load('X_2d.npy')\n",
    "X_2d.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.04 s, sys: 253 ms, total: 2.29 s\n",
      "Wall time: 5.03 s\n"
     ]
    }
   ],
   "source": [
    "# around 1.2M parameters\n",
    "encoder = Sequential()\n",
    "encoder.add(Dense(500, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "encoder.add(Dense(500, activation='relu'))\n",
    "encoder.add(Dense(2000, activation='relu'))\n",
    "encoder.add(Dense(2))\n",
    "%time encoder.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time encoder.fit(X_train, X_2d, nb_epoch=100, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_model(encoder.predict(X_train), y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_differences(encoder.predict(X_train), X_2d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# around 600K parameters\n",
    "nb_filters = 32\n",
    "nb_pool = 2\n",
    "nb_conv = 3\n",
    "conv = Sequential()\n",
    "conv.add(Convolution2D(nb_filters, nb_conv, nb_conv, input_shape=(1, 28, 28)))\n",
    "conv.add(Activation('relu'))\n",
    "conv.add(Convolution2D(nb_filters, nb_conv, nb_conv))\n",
    "conv.add(Activation('relu'))\n",
    "conv.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))\n",
    "conv.add(Dropout(0.25))\n",
    "conv.add(Flatten())\n",
    "conv.add(Dense(128))\n",
    "conv.add(Activation('relu'))\n",
    "conv.add(Dense(2))\n",
    "%time conv.compile(loss='mean_squared_error', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_conv = X_train.reshape(-1, 1, 28, 28)\n",
    "%time conv.fit(X_train_conv, X_2d, nb_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_model(conv, X_train_conv, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_differences(conv, X_train_conv, X_2d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.09 s, sys: 211 ms, total: 2.3 s\n",
      "Wall time: 2.3 s\n"
     ]
    }
   ],
   "source": [
    "# around 1.2M parameters\n",
    "decoder = Sequential()\n",
    "decoder.add(Dense(2000, activation='relu', input_shape=(2,)))\n",
    "decoder.add(Dense(500, activation='relu'))\n",
    "decoder.add(Dense(500, activation='relu'))\n",
    "decoder.add(Dense(X_train.shape[1]))\n",
    "%time decoder.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time decoder.fit(X_2d, X_train, nb_epoch=100, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def make_mosaic(images_flat, n):\n",
    "    nx = ny = n\n",
    "    w = h = int(np.sqrt(len(images_flat[0])))\n",
    "    images = images_flat.reshape(-1, h, w)\n",
    "    image_gen = iter(images)\n",
    "    mosaic = np.empty((h*ny, w*nx))\n",
    "    for i in range(ny):\n",
    "        for j in range(nx):\n",
    "            mosaic[(nx-i-1)*w:(nx-i)*w, j*h:(j+1)*h] = next(image_gen)\n",
    "    return mosaic\n",
    "\n",
    "from cStringIO import StringIO\n",
    "import PIL.Image\n",
    "import IPython.display\n",
    "import shutil\n",
    "def showarray(a, fmt='png', filename=None):\n",
    "    a = np.uint8(np.clip(a, 0, 255))\n",
    "    image_data = StringIO()\n",
    "    PIL.Image.fromarray(a).save(image_data, fmt)\n",
    "    IPython.display.display(IPython.display.Image(data=image_data.getvalue()))\n",
    "    if filename is not None:\n",
    "        with open(filename, 'w') as f:\n",
    "            image_data.seek(0)\n",
    "            shutil.copyfileobj(image_data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "showarray(255 * make_mosaic(X_train, 12))\n",
    "showarray(255 * make_mosaic(decoder.predict(X_2d), 12))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.65 s, sys: 362 ms, total: 4.02 s\n",
      "Wall time: 4.01 s\n"
     ]
    }
   ],
   "source": [
    "n = X_train.shape[1]\n",
    "ae = Sequential()\n",
    "ae.add(Dense(500, activation='relu', weights=encoder.layers[0].get_weights(), input_shape=(n,)))\n",
    "ae.add(Dense(500, activation='relu', weights=encoder.layers[1].get_weights()))\n",
    "ae.add(Dense(2000, activation='relu', weights=encoder.layers[2].get_weights()))\n",
    "ae.add(Dense(2, weights=encoder.layers[3].get_weights()))\n",
    "ae.add(Dense(2000, activation='relu', weights=decoder.layers[0].get_weights()))\n",
    "ae.add(Dense(500, activation='relu', weights=decoder.layers[1].get_weights()))\n",
    "ae.add(Dense(500, activation='relu', weights=decoder.layers[2].get_weights()))\n",
    "ae.add(Dense(n, weights=decoder.layers[3].get_weights()))\n",
    "%time ae.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%time ae.fit(X_train, X_train, nb_epoch=100, verbose=2, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "decoded = ae.predict(X_train)\n",
    "showarray(255 * make_mosaic(decoded, 12))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "def get_submodel(model, start, end):\n",
    "    return theano.function([model.layers[start].input],\n",
    "                          model.layers[end].get_output(train=False),\n",
    "                          allow_input_downcast=True)\n",
    "def get_encoder(ae):\n",
    "    return get_submodel(ae, 0, (len(ae.layers) / 2) - 1)\n",
    "def get_decoder(ae):\n",
    "    return get_submodel(ae, len(ae.layers) / 2, (len(ae.layers) / 2) - 1) # this doesn't work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ae_encoder = get_encoder(ae)\n",
    "# ae_decoder = get_decoder(ae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoded = encoder.predict(X_train)\n",
    "ae_encoded = ae_encoder(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_differences(encoded, ae_encoded, lim=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_model(encoded, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_model(ae_encoded, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ae_decoder = Sequential()\n",
    "ae_decoder.add(Dense(2000, activation='relu', weights=ae.layers[4].get_weights(), input_shape=(2,)))\n",
    "ae_decoder.add(Dense(500, activation='relu', weights=ae.layers[5].get_weights()))\n",
    "ae_decoder.add(Dense(500, activation='relu', weights=ae.layers[6].get_weights()))\n",
    "ae_decoder.add(Dense(X_train.shape[1], weights=ae.layers[7].get_weights()))\n",
    "%time ae_decoder.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample_grid(model, example_data, filename=None, n=128):\n",
    "    x = np.linspace(example_data[:,0].min(), example_data[:,0].max(), n)\n",
    "    y = np.linspace(example_data[:,1].min(), example_data[:,1].max(), n)\n",
    "    xv, yv = np.meshgrid(x, y)\n",
    "    samples_encoded = np.array(zip(xv.flatten(), yv.flatten()))\n",
    "    samples_decoded = model.predict(samples_encoded)\n",
    "    showarray(255 * make_mosaic(samples_decoded, n), filename=filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sample_grid(decoder, encoded, filename='output-orig.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sample_grid(ae_decoder, ae_encoded, filename='output-finer.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.78 s, sys: 362 ms, total: 4.14 s\n",
      "Wall time: 4.14 s\n"
     ]
    }
   ],
   "source": [
    "n = X_train.shape[1]\n",
    "ae_scratch = Sequential()\n",
    "ae_scratch.add(Dense(500, activation='relu', input_shape=(n,)))\n",
    "ae_scratch.add(Dense(500, activation='relu'))\n",
    "ae_scratch.add(Dense(2000, activation='relu'))\n",
    "ae_scratch.add(Dense(2))\n",
    "ae_scratch.add(Dense(2000, activation='relu'))\n",
    "ae_scratch.add(Dense(500, activation='relu'))\n",
    "ae_scratch.add(Dense(500, activation='relu'))\n",
    "ae_scratch.add(Dense(n))\n",
    "%time ae_scratch.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ae_scratch.fit(X_train, X_train, nb_epoch=300, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f203b993890>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnUAAAF/CAYAAAAvhD+vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl4VNX9x/H3N4RdCARZBAHZVBRBRVQQMUIRsSJUBEUR\n1LqDC/25oFbBpXUtKlKwrmWxgkoVBRVFFllUVBRRVqGsAlEoBmSH7++PO0mHkIQhmckkk8/reeZJ\n5t4z934npU8/Peeec8zdEREREZHiLSneBYiIiIhIwSnUiYiIiCQAhToRERGRBKBQJyIiIpIAFOpE\nREREEoBCnYiIiEgCiHmoM7PzzWyxmS01s7tzaTPUzJaZ2bdmdnLo2LFm9o2ZzQv9/NXMbg2dq2pm\nH5nZEjObbGYpsf4eIiIiIkVZTEOdmSUBw4BOwIlALzM7PlubzkAjd28C3AA8D+DuS939FHc/FWgJ\n/Ab8O/SxgcAUdz8OmArcE8vvISIiIlLUxbqn7nRgmbuvcvc9wFiga7Y2XYFRAO7+BZBiZjWztfkd\nsNzd14Z9ZmTo95FAt1gULyIiIlJcxDrU1QHWhL1fGzqWV5t1ObS5FHg97H0Nd98I4O4bgBpRqVZE\nRESkmCryEyXMrDRwEfBmHs2015mIiIiUaMkxvv46oF7Y+6NDx7K3qZtHm87A1+7+c9ixjWZW0903\nmlktID2nm5uZwp6IiIgUG+5u+f1srEPdl0BjM6sPrAcuA3pla/Mu0A8YZ2ZnAlsyh1ZDenHg0Gvm\nZ64CHgf6AhNyK8BduU4ObfDgwQwePDjeZUgxoX8vEin9W5HDYZbvPAfEONS5+z4z6w98RDDU+7K7\nLzKzG4LT/oK7v29mF5jZjwQzXK/O/LyZVSCYJHF9tks/DrxhZtcAq4CesfweIiIiIkVdrHvqcPcP\ngeOyHftHtvf9c/nsdqB6Dsc3E4Q9EREREaEYTJQQKQxpaWnxLkGKEf17kUjp34oUJkvkZ87MzBP5\n+4mIiEjiMLMiPVFCREQkoR1zzDGsWrUq3mVIMVK/fn1WrlwZ9euqp05ERKQAQr0r8S5DipHc/s0U\ntKdOz9SJiIiIJACFOhEREZEEoFAnIiIikgAU6kREROSwPfroo1x/fbA3wKpVq0hKSmL//v0AnHvu\nubzyyivxLK9EUqgTERFJQJ07d85xi7IJEyZw1FFHZQWw/Lrnnnt44YUXst4XdIur7Bo2bEizZs0O\nOp6Wlkb58uWpXLly1qtr165RvXdxpVAnIiKSgPr27cuYMWMOOj5mzBiuvPJKkpKKbgT49NNP2bVr\nF+np6Xz99dcHnDMzhg8fTkZGRtZrwoRct4AvUYruf6IiIiKSb926dWPTpk3MmjUr69iWLVuYOHEi\nffr0AeD999/n1FNPJSUlhfr16/Pggw9mtc0cUh01ahT169enRo0a/PWvf806/+CDD3LllVceso4V\nK1bQoUMHjjzySGrUqEHv3r3JyMjI8zMjR47kkksuoVu3bvzzn/886LyWkMmZQp2IiEgCKleuHD16\n9GDUqFFZx8aNG0fTpk2zhjWPOOIIRo8eza+//sqkSZN4/vnneffddw+4zuzZs1m2bBlTpkzhoYce\nYsmSJVnnIhlydXfuvfdeNmzYwKJFi1i7dm2Ow8KZduzYwVtvvUXPnj3p0aMHY8eOZe/evYf57Usm\nhToREZEE1bdvX9588012794NwOjRo+nbt2/W+Xbt2nHiiScC0KxZMy677DJmzJiRdd7MGDx4MGXK\nlKF58+a0aNGC+fPnH1YNjRo1okOHDiQnJ1OtWjUGDBhwwD2yGz9+PJUrV+ass86iffv2mBmTJk06\noM0tt9xCamoqVatWJTU1lUGDBh1WTYlK24SJiIjEULTmD+RnxPGss86ievXqvPPOO5x22ml8+eWX\nvP3221nn586dy8CBA/n+++/ZvXs3u3fvpkePHgdco2bNmlm/V6hQgW3bth1WDenp6dx2223MnDmT\nbdu2sW/fPlJTU3NtP2rUKLp37w5AqVKl6NatGyNHjjxgMsRzzz3HNddcc1h1lAQKdSIiIjEU78e/\nrrzySkaOHMnixYvp1KkT1atXzzp3+eWXc+uttzJ58mRKly7NgAED2LRpU1Tvf++995KUlMQPP/xA\nSkoKEyZM4JZbbsmx7bp165g6dSpffvkl48aNA4Lh2J07d7J58+Y8w6Bo+FVERCSh9enThylTpvDS\nSy8dMPQKsG3bNqpWrUrp0qWZO3cu//rXvw44fzgTEnJru3XrVo444ggqVarEunXrePLJJ3O9xqhR\nozjuuONYunQp8+fPZ/78+SxdupQ6derw+uuvR1xLSaVQJyIiksDq169PmzZt2L59OxdddNEB54YP\nH879999PSkoKjzzyCJdeeukB57NPhMhrYkT4ufDfBw0axNdff02VKlXo0qVL1tBqTkaPHk2/fv2o\nXr06NWrUyHrdeOONjBw5Mqtd//79s9aoq1SpEq1atcr7j1BCWCJPCzYzT+TvJyIi8WdmWmJDDktu\n/2ZCx/P9FKZ66kREREQSgEKdiIiISAJI+FBXwK3tRERERIqFhA91O3bEuwIRERGR2Ev4UPfbb/Gu\nQERERCT2FOpEREREEoBCnYiIiEgCUKgTERERSQAKdSIiIiIJQKFOREQkQTVo0ICpU6fm2WbhwoV0\n6tSJatWqkZqaSqtWrfjwww8BmDFjBklJSfTv3/+Az5x99tmMGjUKgJEjR5KcnHzAtl2VK1dmw4YN\nud5z2LBhtGjRgooVK1K7dm3at2/PuHHjCvhtYyOSv2FRoVAnIiJSgnXp0oVOnTqxceNG0tPTGTp0\nKJUrV846X7FiRUaPHs3q1atzvUabNm3IyMggIyODrVu3kpGRQa1atXJse8sttzB06FCefvppNm/e\nzLp163jkkUeYPHly1L/boezbt69YXz87hToREZEE1KdPH1avXk2XLl2oXLkyTz311EFtNm3axMqV\nK7n22mtJTk4mOTmZ1q1b06ZNm6w2VapU4aqrrmLw4MEFrmnZsmWMGDGCcePG0b59e8qWLYuZ0aZN\nG1555ZWsdhkZGVx77bXUrl2bunXrcv/992ftlTpy5EjOPvts7rzzTlJTU2nUqFFWz2Ikn23bti1/\n+tOfOPLII3nwwQdZsWIFHTp04Mgjj6RGjRr07t2bjIyMPP+G7777Ls2aNSM1NZX27duzePHirPs3\naNCAJ554ghYtWnDEEUewvxB3QVCoExERSUCjRo2iXr16TJw4kYyMDO64446D2lSrVo3GjRtzxRVX\nMGHCBNLT0w9qY2bcd999jB8/nmXLlhWopqlTp1KvXj1OOeWUPNv17duXMmXKsGLFCr755hs+/vhj\nXnrppazzc+fOpWnTpmzatIk777yTP/7xjxF/9osvvqBx48akp6dz33334e7ce++9bNiwgUWLFrF2\n7dqsAJvT33Dp0qVcfvnlDB06lJ9//pnOnTvTpUsX9u7dm3WPsWPH8sEHH7BlyxaSkgovainUiYiI\nJLDMXqrcTJs2jQYNGnDHHXdQu3Zt0tLSWL58+QFtatSowY033sgDDzyQ4zU+++wzUlNTSU1NpWrV\nqjRp0iTHdr/88stBw7J169alatWqlC9fnjVr1pCens4HH3zA008/Tbly5TjyyCO5/fbbef3117M+\nU79+fa655hrMjL59+7J+/XrS09Mj+mydOnW4+eabSUpKomzZsjRq1IgOHTqQnJxMtWrVGDBgADNm\nzMj1b/jGG29w4YUX0r59e0qVKsUdd9zBjh07mDNnTlab2267jdq1a1O2bNk8//bRllyod4sDhToR\nEYkne9Cich0flHc4i8RNN93EmDFjMDPuvfdeBg4cSO3atRk6dCgA69at47rrrqNPnz7Mnj37gM/e\nfffdNG7cmO++++6g67Zu3ZpPP/30kPevVq0a69evP+DYmjVr2LdvH2XKlMHdWbVqFXv27OGoo44C\ngkDl7tSrVy/rM+HBsHz58gBs27aNTZs2HfKzdevWPeD+6enp3HbbbcycOZNt27axb98+UlNTc/0O\nP/30E/Xr1896b2bUrVuXdevWZR07+uijD/m3iIWYhzozOx94hqBX8GV3fzyHNkOBzsBvwFXu/m3o\neArwEtAM2A9c4+5fmNkg4Dogs5/4Xnf/MPt1QaFORETiKxphLL/MDgyUI0aMYMSIEbm2r1OnDv36\n9ePyyy8/6Fxqaiq33347999//0HXjVT79u255ZZbmDdvHqeeeuoB5zJ7w+rWrUu5cuXYtGnTYd8n\nks9mP37vvfeSlJTEDz/8QEpKChMmTOCWW27JtX3t2rX5/vvvDzi2Zs2aA4Jcfv8+BRXT4VczSwKG\nAZ2AE4FeZnZ8tjadgUbu3gS4AXg+7PSzwPvu3hRoASwKOzfE3U8NvXIMdKBQJyIiJVetWrVYsWJF\nrue3bNnC4MGDWb58Oe7OL7/8wiuvvELr1q1zbD9gwADmzJnDokWLDjh+qCHeTMceeyw33HADl112\nGVOmTGHnzp3s37+f2bNnZwWhWrVqcd555zFgwAC2bt2Ku7NixYqIegLz89mtW7dyxBFHUKlSJdat\nW8eTTz550DXD/4Y9e/Zk0qRJTJs2jb179/LUU09Rrly5XP9mhSnWz9SdDixz91XuvgcYC3TN1qYr\nMArA3b8AUsyspplVBs5291dD5/a6e0bY5yKKwQp1IiJSUg0cOJCHH36Y1NRUhgwZctD5MmXKsHLl\nSjp27EhKSgrNmzenXLlyvPrqqzler1KlStx1111s3rz5gOOff/75QevUff311zleY9iwYdx66638\n6U9/olq1atStW5dBgwbxxhtvZA2Tjho1it27d3PCCSeQmppKjx498lz3Lrxn7HA/O2jQIL7++muq\nVKlCly5d6N69+wHns/8Njz32WMaMGUP//v2pXr06kyZN4r333iM5OfmgWgqbRZqu83Vxs+5AJ3e/\nPvS+N3C6u98a1uY94FF3nxN6PwW4C9gHvAAsJOil+wq4zd13hIZfrwJ+DR3/P3f/NYf7e9euzjvv\nxOwriohICWdmEfdUiUDu/2ZCx/OdCovy7Ndk4FTg7+5+KrAdGBg6Nxxo6O4nAxuAg//vR4h66kRE\nRKQkiPVEiXVAvbD3R4eOZW9TN5c2a9z9q9DvbwF3A7j7z2HtXwTey62AxYsHk7leYlpaGmlpaYdT\nv4iIiEhMTJ8+nenTp0fterEefi0FLAE6AOuBuUAvd18U1uYCoJ+7/97MzgSecfczQ+dmANe5+9LQ\nkGsFd7/bzGq5+4ZQmwFAK3c/aKqOmXnz5s78+TH7iiIiUsJp+FUOV6yGX2PaU+fu+8ysP/AR/1vS\nZJGZ3RCc9hfc/X0zu8DMfiRY0uTqsEvcCrxmZqWBFWHnnjCzkwmWOVlJMGs2R9u3R/1riYiIiBQ5\nMe2pizcz86OOcn76Kd6ViIhIolJPnRyukjhRIio0UUJERERKghIR6vR/oERERCTRJXyoS0qC3bvj\nXYWIiIhIbCV8qKtYUUOwIiIikvgU6kRERBJUgwYNmDp1ap5tFi5cSKdOnahWrRqpqam0atWKDz8M\ntlSfMWMGSUlJ9O/f/4DPnH322YwaNQqAkSNHkpycfNA2YXltzQXQsGFDmjVrdtDxtLQ0ypcvn3W9\nypUr07Vr9h1GJScKdSIiIiVYly5d6NSpExs3biQ9PZ2hQ4dSuXLlrPMVK1Zk9OjRrF69OtdrtGnT\nhoyMDDIyMti6dSsZGRnUqlUr1/affvopu3btIj09/aA9Ys2M4cOHZ10vIyODCRMmFPyLlgAKdSIi\nIgmoT58+rF69mi5dulC5cmWeeuqpg9ps2rSJlStXcu2115KcnExycjKtW7emTZs2WW2qVKnCVVdd\nxeDM7ZmiYOTIkVxyySV069aNf/7znwed1xIx+aNQJyIikoBGjRpFvXr1mDhxIhkZGdxxxx0HtalW\nrRqNGzfmiiuuYMKECaSnpx/Uxsy47777GD9+PMuWLStwXTt27OCtt96iZ8+e9OjRg7Fjx7J3794C\nX1cU6kRERBLaoXq9pk2bRoMGDbjjjjuoXbs2aWlpLF++/IA2NWrU4MYbb+SBBx7I8RqfffYZqamp\npKamUrVqVZo0aZLr/caPH0/lypU566yzaN++PWbGpEmTDmhzyy23ZF0rNTWVQYMGRfhtSzaFOhER\nkVgyi84rCm666aasiQyPPfYYALVr12bo0KEsW7aMVatWUaFCBfr06XPQZ++++24mT57Md999d9C5\n1q1bs3nzZjZv3sx///vfPHv0Ro0aRffu3QEoVaoU3bp1Y+TIkQe0ee6557KutXnzZh588MGCfO0S\nI6Z7vxYFCnUiIhJXcXw+zLKFwREjRjBixIhc29epU4d+/fpx+eWXH3QuNTWV22+/nfvvv/+g60Zq\n3bp1TJ06lS+//JJx48YBwXDszp072bx5M6mpqfm6rgTUUyciIpKgatWqxYoVK3I9v2XLFgYPHszy\n5ctxd3755RdeeeUVWrdunWP7AQMGMGfOHBYtWnTA8UgnNowaNYrjjjuOpUuXMn/+fObPn8/SpUup\nU6cOr7/+euRfTHKkUCciIpKgBg4cyMMPP0xqaipDhgw56HyZMmVYuXIlHTt2JCUlhebNm1OuXDle\nffXVHK9XqVIl7rrrLjZv3nzA8c8///ygdeqyL1UCMHr0aPr160f16tWpUaNG1uvGG288YAi2f//+\nB1yvVatWBfxLlAyWyNOGzcwfeMAxgyjOxBYREcliZlqCQw5Lbv9mQsfz/QCleupEREREEoBCnYiI\niEgCUKgTERERSQAlItRt3x7vKkRERERiq0SEOvXUiYiISKJL+FBXoYJCnYiIiCS+hA916qkTERGR\nkkDbhImIiBRA/fr1871tlpRM9evXj8l1FepEREQKYOXKlfEuQQTQ8KuIiIhIQkj4UFe+wn6FOhER\nEUl4CR/qSN7Jnj2wb1+8CxERERGJnYQPdTv37tCyJiIiIpLwEj7U7di7Q8/ViYiISMJL/FC3R6FO\nREREEl/Ch7rte7Yr1ImIiEjCS/hQp+FXERERKQkSP9Tt2cGRR8LPP8e7EhEREZHYSfxQt3cHRx0F\n69fHuxIRERGR2Il5qDOz881ssZktNbO7c2kz1MyWmdm3ZnZy2PEUM3vTzBaZ2Q9mdkboeFUz+8jM\nlpjZZDNLye3+O/Yo1ImIiEjii2moM7MkYBjQCTgR6GVmx2dr0xlo5O5NgBuA58NOPwu87+5NgRbA\notDxgcAUdz8OmArck1sN2/dsV6gTERGRhBfrnrrTgWXuvsrd9wBjga7Z2nQFRgG4+xdAipnVNLPK\nwNnu/mro3F53zwj7zMjQ7yOBbrkVkDn8umFD1L6TiIiISJET61BXB1gT9n5t6FhebdaFjjUAfjGz\nV81snpm9YGblQ21quPtGAHffANTIrQANv4qIiEhJUJQnSiQDpwJ/d/dTge0Ew64Alq2t53aRHXt3\nUKuWQp2IiIgktuQYX38dUC/s/dGhY9nb1M2lzRp3/yr0+1tA5kSLDWZW0903mlktID23Aj548QN+\nq7eT9eth6tQ02rdPy9cXEREREYmm6dOnM3369Khdz9xz7eQq+MXNSgFLgA7AemAu0MvdF4W1uQDo\n5+6/N7MzgWfc/czQuRnAde6+1MwGARXc/W4zexzY7O6Ph2bUVnX3gdluj5n5HZPv4MnznuTII2Hh\nQqiR60CtiIiISPyYGe6efTQyYjHtqXP3fWbWH/iIYKj3ZXdfZGY3BKf9BXd/38wuMLMfgd+Aq8Mu\ncSvwmpmVBlaEnXsceMPMrgFWAT1zq2HH3h0AWc/VKdSJiIhIIor18Cvu/iFwXLZj/8j2vn8un50P\ntMrh+Gbgd5Hcf8eeA0NdixaR1S0iIiJSnBTliRJRsX3vdgDNgBUREZGElvChLntPnYiIiEgiSvxQ\nt1ehTkRERBJf4oc69dSJiIhICZDwoW77Hj1TJyIiIokv4UOdhl9FRESkJEj8UJdt+DWGay2LiIiI\nxE3ih7pQT13FilC6NPz6a5wLEhEREYmBhA91mc/UgYZgRUREJHElfKjLHH4FhToRERFJXAkf6vb5\nPvbt3wdArVoKdSIiIpKYEj7UlU8urxmwIiIikvASP9SVLq+16kRERCThJX6oSy6vXSVEREQk4SV8\nqKtQuoKGX0VERCThJR+qgZmlAK2BYwAHVgGfuXuxWPGtfGn11ImIiEjiy7Wnzszamtm7wKdAL6A+\n0CD0+0wzm2BmbQunzPwrn6xn6kRERCTx5dVTdzHwf+6+LKeTZnYscCMwKxaFRUv50v+b/Vq1arBN\nWHo61KgR58JEREREoijXnjp3/1NugS50fqm7/yk2ZUVPhdIVsoZfzaBzZ5gwIc5FiYiIiERZXsOv\nz4T9flu2c/+MYU1RFb5OHUD37jB+fBwLEhEREYmBvGa/tgv7vW+2c81jUEtMhK9TB0FP3Zw5sGVL\nHIsSERERibK8Qp3l8nuxEr5OHUClSnDuufDee3EsSkRERCTK8gp1SWZW1cyqhf2eamapQKlCqq/A\nwtepy6QhWBEREUk0ec1+TQG+5n+9dPPCznnMKoqy7D11AF26QP/+sG0bHHFEnAoTERERiaJcQ527\nH1OIdcRM9mfqIFjapE0b+OAD6NEjToWJiIiIRFFes1/rh3aTyHx/rpk9a2YDzKxM4ZRXcNlnv2a6\n+GINwYqIiEjiyOuZujeAigBmdjLwJrAaOBkYHvvSoiN8nbpw3brBhx/Czp1xKEpEREQkyvIKdeXd\n/afQ772BV9z9b8DVwOkxryxKwneUCFejBpx8Mnz0URyKEhEREYmySJc0aQ98AuDu+2NaUZSF7/2a\nXffu8O9/F3JBIiIiIjGQV6ibamZvmNmzQFVgKoCZHQXsLozioiG3njqAP/whWK9uz55CLkpEREQk\nyvIKdbcD/wZWAm3dPTP61ALui3FdUZPbM3UARx8NTZrAtGmFXJSIiIhIlOW1pIkDY3M4/k1MK4qy\n3Ga/ZspciPi88wqxKBEREZEoy2tJk61mlhH22hr+szCLLIic1qkL1707vPMO7NtXiEWJiIiIRFle\nw6+fAAuBR4Bm7l7J3Stn/oz0BmZ2vpktNrOlZnZ3Lm2GmtkyM/vWzE4JO77SzOab2TdmNjfs+CAz\nW2tm80Kv83O7f047SoRr2BBq14bZsyP9RiIiIiJFT66hzt27AZ2An4EXzWyGmd0c2vs1ImaWBAwL\nXedEoJeZHZ+tTWegkbs3AW4ARoSd3g+kufsp7p59GZUh7n5q6PVhbjXktPdrdhdfDG++Gem3EhER\nESl68uqpw91/dfdXgc7AP4CHgKsO4/qnA8vcfVVoosVYoGu2Nl2BUaH7fQGkmFnN0DnLo0bL5fgB\nypfOu6cOoHdveP11LUQsIiIixVeeoc7M2pjZc8A8oA3wB3cfchjXrwOsCXu/NnQsrzbrwto48LGZ\nfWlm12X7XP/QcO1L4duZZXeoiRIADRpAy5baNkxERESKr7wmSqwk2A5sHXA98Arwm5mdamanFk55\nnOXupwIXAP3MrG3o+HCgobufDGwAcg2amRMlgsm8ubv+enjhhShVLSIiIlLIcl3ShGB9Oid4Hu48\nDhzudIJdJg5lHVAv7P3RoWPZ29TNqY27rw/9/NnM3iYYzp3l7j+HtX8ReC+3Ah556BGYAQ/sfYAO\n7TuQlpaWY7suXaBfP1iyBI47LoJvJiIiIlIA06dPZ/r06VG7nh2qB6tAFzcrBSwBOgDrgblAL3df\nFNbmAqCfu//ezM4EnnH3M82sApDk7tvMrCLwEfCgu39kZrXcfUPo8wOAVu5+eQ73d3cn5bEUVt++\nmpRyuY7SAjBwIOzdC089FZ3vLyIiIhIpM8PdI5ozkJO8hl/PieDmaXmdd/d9QH+CQPYDMNbdF5nZ\nDWZ2fajN+8B/zOxHgskYN4c+XhOYZWbfAJ8D77n7R6FzT5jZd2b2LXAOMCCvOiJ5rg7g2mth1CjY\nteuQTUVERESKlFx76szsSaAdwXp1XxH0tBnBNmGtgN8BU909x7XnioLMnroGzzbgkz6f0LBqw0N+\npkMHuPrqYEasiIiISGEpaE9dXtuE3WlmRxAsOdIRqB86tQqYBfzF3bfl98aF6VALEIe791644Qa4\n9FIoXTrGhYmIiIhESV4TJQiFttdCr2IrkgWIM3XoAI0aBTNh+/WLcWEiIiIiUZLnOnWJ4ogyR7B1\n19aI2z/+ODzyCGyN/CMiIiIicVUiQl3dlLqsyVhz6IYhJ58c9Nj97W8xLEpEREQkikpEqDsm5RhW\nbll5WJ955BF47jlYl31VPREREZEi6JChzsx6mFml0O9/NrN/F+KOElFRv0r9ww51xxwDt90GV10F\n+/fHoioRERGR6Imkp+5+d98a2qLrd8DLwIjYlhVdx1Q5/J46CGbC7tgBTz8d/ZpEREREoimSULcv\n9PP3wAvuPgkoE7uSoi+/oS45GcaMCSZOfPNN9OsSERERiZZIQt06M/sHcCnwvpmVjfBzRUbdynVZ\nt3Ud+/bvO3TjbI45Bp55Bnr1gm3FYlU+ERERKYkiCWc9gclAJ3ffAqQCd8a0qigrm1yW6hWq89PW\nn/L1+csvhzZt4OabIYZb5YqIiIjkWySh7ihgkrsvC+312gOYG9OqYiC/Q7CZhg0LhmBffjl6NYmI\niIhESyShbjywz8waAy8AdYF/xbSqGMjPDNhwFSrAm2/CPffA/PnRq0tEREQkGiIJdfvdfS9wMfCc\nu99J0HtXrORnrbrsjj8enn0WLrkEtmyJTl0iIiIi0RBJqNtjZr2APsDE0LFit9V9QYdfM11+OXTu\nDFdcofXrREREpOiIJNRdDbQG/uLu/zGzBsDo2JYVfcdUOYaVv66MyrX+9rdgX9gHH4zK5UREREQK\n7JChzt0XAncAC8ysGbDW3R+PeWVRFq2eOoDSpYPn6155Bd59NyqXFBERESmQSLYJSwOWAX8HhgNL\nzaxdjOuKunop9ViXkb+16nJSsya89RZcey0sWRKVS4qIiIjkWyTDr38DznP3c9y9HdAJKHYbZ5VN\nLktq+VS/arKiAAAgAElEQVTWb1sftWuecQb89a/QrRtkZETtsiIiIiKHLZJQV9rds/qi3H0pxXCi\nBER3CDbTtddCu3Zw1VWaOCEiIiLxE0mo+8rMXjKztNDrReCrWBcWC7EIdQBDh8L69fDYY1G/tIiI\niEhEkiNocxPQD7g19H4mwfN1xU6sQl3ZsjB+PLRqBaeeCuefH/VbiIiIiOQpktmvu9x9iLtfHHo9\nTTFc0gRiF+oAateGceOgb19YvjwmtxARERHJVSTDrzlpHdUqCkn9lPqs+nVVzK7fti088AD84Q/w\n228xu42IiIjIQfIb6oqlWPbUZbr5Zjj5ZOjXL6a3ERERETlArs/UmdmpuZ2imM5+rZdSjzW/rmHf\n/n2USioVk3uYwYgRcPrp8OqrcPXVMbmNiIiIyAHymijxtzzOLY52IYWhfOny1K9Snx9+/oHmNZvH\n7D4VKwY7TpxzDpx2Gpx0UsxuJSIiIgLkEerc/dzCLKSwnFX3LGatnhXTUAdwwgnBHrE9esC8eVCh\nQkxvJyIiIiVciXqmDqBtvbbMWj2rUO7Vp0/wfN0DDxTK7URERKQEU6iLseeeg9deg88+K7RbioiI\nSAlU4kJdk9Qm7Ny7k9W/ri6U+1WvHgS7q6+GHTsK5ZYiIiJSAkUU6sysjpm1MbN2ma9YFxYrZkbb\nem2ZvXp2od3zkkugeXMYPLjQbikiIiIljLl73g3MHgcuBRYC+0KH3d0vinFtBWZmntP3G/LZEJZv\nXs7ff194u52lpwfBbsIEOOOMQrutiIiIFBNmhrtbfj8fyd6v3YDj3H1Xfm9S1LSt15aR80cW6j1r\n1IBnnw2GYefNg3LlCvX2IiIikuAiGX5dQQEWGzaz881ssZktNbO7c2kz1MyWmdm3ZnZK2PGVZjbf\nzL4xs7lhx6ua2UdmtsTMJptZyuHUdEqtU1jx3xVs2bklv18rX3r2hKZN4aGHCvW2IiIiUgJEEuq2\nA9+a2T9C4WuomQ2N5OJmlgQMAzoBJwK9zOz4bG06A43cvQlwAzAi7PR+IM3dT3H308OODwSmuPtx\nwFTgnkjqyVS6VGla1W7FZ2sKd0qqGQwfDi+/DF9+Wai3FhERkQQXSah7F3gYmAN8HfaKxOnAMndf\n5e57gLFA12xtugKjANz9CyDFzGqGzlkuNXYFMsdPRxIMER+Wwl7aJFPNmvD008Ew7K6EGdAWERGR\neDtkqHP3kcDr/C/M/St0LBJ1gDVh79eGjuXVZl1YGwc+NrMvzey6sDY13H1jqL4NQI0I68nStl5b\nPl396eF+LCp69YLGjeHhh+NyexEREUlAh5woYWZpBL1hKwl6zuqaWV93L4xEdJa7rzez6gThbpG7\n59S9lusU3sFh64ikpaWRlpYGwNn1zmbBxgWk/5ZOjYqHnQkLxAxGjIAWLeAPf4CWLQv19iIiIlIE\nTJ8+nenTp0ftepEsafI1cLm7Lwm9PxZ43d0PGUXM7ExgsLufH3o/kGA5lMfD2jwPTHP3caH3i4Fz\nMnviwtoNAra6+xAzW0TwrN1GM6sV+nzTHO6f45Imma749xW0rduWm1rddKivEhNjxsATT8BXX0GZ\nMnEpQURERIqIgi5pEskzdaUzAx2Auy8l8tmwXwKNzay+mZUBLiN4Ri/cu0AfyAqBW0JhrYKZHRE6\nXhE4D/g+7DNXhX7vC0yIsJ4D9DyhJ28sfCM/H42KK66ABg3gz3+OWwkiIiKSICLpqXuFYBbqmNCh\nK4BS7n5NRDcwOx94liBAvuzuj5nZDQQ9di+E2gwDzgd+A65293lm1gB4m2BoNRl4zd0fC7VPBd4A\n6gKrgJ7uftD6JIfqqdu5dydH/e0oFt68kKMqHRXJ14m6X36BU06BF1+E88+PSwkiIiJSBBS0py6S\nUFcW6Ae0DR2aCQwvDosRHyrUAfR5uw+n1zmd/qf3L6SqDjZjBlx2WbAo8VHxyZYiIiISZzEPdcVZ\nJKFu4tKJPDH7CT69Oj4zYTMNHgwzZ8JHH0GpUnEtRUREROIgZs/UmdkboZ8LzOy77K/83rCo6diw\nI9+nf8+6jHVxreP++8Fdz9eJiIhI/uTaU2dmR4WWE6mf03l3XxXTyqIgkp46gKsnXE2Lmi24/czb\nC6Gq3P3yC7RqBY8+GgzHioiISMkRs546d18f+vXm0I4QWS/g5vzesCjqfVJvXvnmFeI9FH3kkfDO\nO3DLLfDNN3EtRURERIqZSJY06ZjDsc7RLiSe2jdoz+59u5m5ema8S6FFi2B/2G7d4Kef4l2NiIiI\nFBd5PVN3k5ktAI7P9jzdf4AFhVdi7JkZ/U/vz7C5w+JdCgA9esCNN8IFF0BGRryrERERkeIgr2fq\nUoCqwKPAwLBTW919cyHUVmCRPlMHkLErg2OeOYYFNy2gTuXs29MWPne4+Wb48UeYNEk7ToiIiCS6\nWD5T96u7ryRYOHhz2PN0e83sjPzesKiqXLYyl590Of/4+h/xLgUI9od97jkoXx6uuy4IeSIiIiK5\nieSZuhHAtrD320LHEk6/Vv144esX2LW3aKyrnJwMY8fCkiXwwAPxrkZERESKskhC3QFjmO6+n2Db\nroTTtHpTmtVoxvhF4+NdSpYKFeC994Jw94+i0YkoIiIiRVAkoW6Fmd1qZqVDr9uAFbEuLF5uOu0m\nnv/q+XiXcYDq1eGDD4JdJ95+O97ViIiISFEUSai7EWgDrAPWAmcA18eyqHi66LiL+HHzj/yQ/kO8\nSzlA48YwcSLcdBO8+Wa8qxEREZGipsTv/ZqTB6Y9wJadWxjaeWgMqiqY776D88+Hxx+HK6+MdzUi\nIiISLQWd/XrIUGdmrwIHNXL3a/J708KS31C3+tfVnPKPU1h9+2oqlqkYg8oKZtEi6NgRBg0KZsaK\niIhI8RezJU3CTAQmhV6fAJU5cDZswqmXUo+z6p7FuB/GxbuUHDVtCtOnw1/+Eix7IiIiInLYw69m\nlgTMcvc2sSkpevLbUwfw/rL3GTx9MHOvmxvlqqJn1Sro0AGuvx7uuive1YiIiEhBFEZPXXZNgBr5\nvWFx0alRJ37e/jOfr/083qXkqn59mDEDRo6E226DffviXZGIiIjEyyFDnZltNbOMzBfwHnB37EuL\nr1JJpbirzV08/OnD8S4lT3XqwOzZ8MMP0LUrbN0a74pEREQkHvIMdWZmwInuXjnsday7F53VeWPo\nmlOuYcHGBcxdV3SHYAGqVAnWsTvqKGjbFpYvj3dFIiIiUtjyDHWhB9ImFVItRU7Z5LIMbDuQh2Y8\nFO9SDql0aXjhhWA2bJs2wS4UIiIiUnJE8kzdPDNrFfNKiqg/nvJH5m+cz5frvox3KYdkBv37wzvv\nwM03w5//rOfsRERESopI1qlbDDQGVgG/AUbQidc89uUVTEFmv4Yb/uVw3l/2PhMvnxiFqgpHejpc\ndhmUKgX/+lew1ZiIiIgUXYUx+7UT0AhoD3QBLgz9LDH+eMofWZC+gNmrZ8e7lIjVqAEffQSnnRa8\nZhef0kVERCQfIgl1j7j7qvAX8EisCytKyiaX5ZFzH+HOj++kOG2rlpwMjz4aLFB8ySVw552wY0e8\nqxIREZFYiCTUnRj+xsxKAS1jU07RdUXzK9i+ZztvL3473qUctosuCvaMXb0aTjkFvvgi3hWJiIhI\ntOUa6szsHjPbCjQPW6duK5AOTCi0CouIJEviiY5PMHDKQPbs2xPvcg5b9eowbhw8/HCwnt3AgbBz\nZ7yrEhERkWjJNdS5+6PuXgl4MmyNukruXs3d7ynEGouM8xqdxzFVjuHFeS/Gu5R869Ej6LVbtgxa\ntoQvi/6kXhEREYlAJMOvE82sIoCZ9TazIWZWP8Z1FVlPdHyCB2c8yOYdm+NdSr7VqAFvvQX33w8X\nXgj33Qe7dsW7KhERESmISELdCGC7mbUA/g9YDoyKaVVF2Mm1TqbHCT2475P74l1KgZgFS57Mnx9s\nMXbaaTBvXryrEhERkfyKJNTtDS321hUY5u5/ByrFtqyi7eFzH+adJe8UiwWJD6VWLXj7bbj7bjj/\nfBg0CHbvjndVIiIicrgiCXVbzeweoDcwycySgNKxLatoq1q+Ko91eIyb37+ZffuL/5YNZtC7N3z7\nbdBbd/rpQQ+eiIiIFB+RhLpLgV3AH919A3A08GRMqyoGrmxxJWVLleWleS/Fu5SoqV0b3n0XBgyA\njh3hoYdgT/Gb6CsiIlIiHTLUufsGdx/i7jND71e7e8TP1JnZ+Wa22MyWmtndubQZambLzOxbMzs5\n27kkM5tnZu+GHRtkZmtDx+eZ2fmR1hMtSZbEiN+P4P5p97N+6/rCvn3MmEHfvkGP3eefQ4sW8PHH\n8a5KREREDuWQoc7MLg4Frl8z16ozs4xILh4aqh1GsNXYiUAvMzs+W5vOQCN3bwLcADyf7TK3AQtz\nuPwQdz819Powknqi7aSaJ3F9y+u55YNb4nH7mDr6aJg0KdiR4sYboVs3WLEi3lWJiIhIbiIZfn0C\nuMjdU8LWqqsc4fVPB5aFthfbA4wlmHARriuh2bTu/gWQYmY1AczsaOACIKcxznxveBtNf273Zxak\nL+Cdxe/Eu5SoMwsWKv7hBzjjDGjVKlj+ZNu2eFcmIiIi2UUS6ja6+6J8Xr8OsCbs/drQsbzarAtr\n8zRwJ5DThqv9Q8O1L5lZSj7rK7ByyeV4scuL9H+/P7/u/DVeZcRUuXJwzz3BosWrVsFxx8HTT8Nv\nv8W7MhEREckUSaj7yszGmVmv0FDsxWZ2cawLM7PfEwTKbwl65cJ75oYDDd39ZGADMCTW9eSlXf12\ndD2uK9e+dy3B6i+JqU4dGDMmmEwxaxY0bAh/+Qts2RLvykRERCQ5gjaVge3AeWHHHPh3BJ9dB9QL\ne3906Fj2NnVzaHMJcJGZXQCUByqZ2Sh37+PuP4e1fxF4L7cCBg8enPV7WloaaWlpEZR9+P7W6W+0\ne7UdT855krvOuism9ygqWraE8eNh4UJ47DFo1AhuuCGYNVu9eryrExERKR6mT5/O9OnTo3Y9i2XP\nkpmVApYAHYD1wFygV/hwbii09XP335vZmcAz7n5mtuucA/yfu18Uel8rtLwKZjYAaOXul+dwfy/M\nnrM1v67hjJfOYGS3kXRs1LHQ7htvK1bAE0/AG29Anz5wxx3BRAsRERGJnJnh7vmeMxDJ7Nejzext\nM0sPvcaHJjAckrvvA/oDHwE/AGPdfZGZ3WBm14favA/8x8x+BP4B3BzBpZ8ws+/M7FvgHGBAJPXE\nWt2Uuvyr+7+48u0rWf3r6niXU2gaNoTnn4cFC6BUKWjeHK67Dn78Md6ViYiIlByH7Kkzs4+BfwGj\nQ4d6A1e4e5HviirsnrpMj858lMnLJ/NJn08olVSq0O8fb7/8As8+CyNGQKdOwSSLZs3iXZWIiEjR\nFvOeOqC6u7/q7ntDr38CenIqD5nP1D05p2RuvHHkkfDww8Gw7Eknwe9+F6xz98UX8a5MREQkcUUS\n6jaZWW8zKxV69QY2xbqw4qxUUilG/2E0Qz4bwlc/fRXvcuKmcmUYODAId+3bw6WXwmmnwUsvaa07\nERGRaItk+LU+8BzQmmDW6xzgVncv8g+NxWv4NdMbP7zBn6f+mbnXzaVKuSpxq6Oo2LcPJk+Gf/wD\nZs6EXr2CWbPNm8e7MhERkfgr6PBrTGe/xlu8Qx3AbR/cxqJfFvH+Fe+TnBTJCjIlw5o18PLLQa9d\nvXpBuOvZE8qXj3dlIiIi8VEYs19HmlmVsPdVzeyV/N6wpPlbp79RKqkUt394e7xLKVLq1oXBg2Hl\nSrj7bhg3Ljh2++2wKL/7l4iIiJRgkTxT19zds/YMcPf/AqfErqTEkpyUzNjuY5m2chrD5g6LdzlF\nTnJysL/s++/DV19BxYrB83ft2sHf/w7rsi9VLSIiIjmKJNQlmVnVzDdmlkpkO1FISEq5FN7r9R6P\nznqUtxe9He9yiqxjjgm2HVu9Gv7v/4LZsiedBGefDRMmwP798a5QRESk6IpkokQf4F7gzdChHsBf\n3H107p8qGorCM3Xh5q2fx/ljzuetnm/Rrn67eJdTLOzeHQS6xx6DnTvhttuge3eoVi3elYmIiERX\noUyUMLMTgPaht1PdfWF+b1iYilqoA5iyYgpX/PsKplw5hZNqnhTvcooNd/j442BixeTJcOaZcNVV\nQcArUybe1YmIiBRcYYW6tkATd3/VzKoDR7j7f/J708JSFEMdwLjvxzFg8gA+uvIjmtXQVguH67ff\nYNKkYGmURYvgxhuD2bM1a8a7MhERkfwrjNmvg4C7gXtCh0oDY/J7Q4FLm13KkE5D6Di6Iws2Loh3\nOcVOxYrB8ieffAIffRRMpjj+eOjTB+bODXr1RERESppInqn7lmC26zx3PyV07Dt3L/JLxhbVnrpM\n474fx+2Tb+fjKz9Wj10Bbd4crHv3/PPBjNrLLgteTZvGuzIREZHIFMber7tDychDN6yY35vJgS5t\ndilDzhtC59c6s2rLqniXU6ylpsKdd8KPP8Lo0bB1a7DnbIsW8Ne/wvLl8a5QREQktiLpqbsDaAJ0\nBB4FrgFed/ehsS+vYIp6T12moV8MZfiXw5l1zSyOrHBkvMtJGPv3w6xZwcLGb70F9esH+8/27Bks\ndCwiIlKUFNZEiY7AeYABk9394/zesDAVl1AHcO8n9/LJfz7ho94fkVIuJd7lJJy9e2HaNBg7Ft55\nB044IQh4PXpogoWIiBQNcdn71cwudfdx+b1pYSlOoc7duf3D25m+ajofXPEBtSvVjndJCWv37mCC\nxdixMHEinHZaEO7at4fGjcHy/V8nERGR/ItZqAs9O3cD0Aj4AXgeuAj4K7DM3bvm96aFpTiFOgiC\n3eOzH+f5r57ngys+oGl1PeUfa9u3B1uUvf02fPop7NkTPIt35ZXBz1Kl4l2hiIiUFLEMdeOBrcBn\nBM/T1QV2Are5+7f5vWFhKm6hLtOo+aO46+O7mHT5JFrWbhnvckoMd1i1KlgDb+TIYKmUiy4KevJO\nOw2aNVPIExGR2IllqMtatsTMSgHrgXruvjO/NytsxTXUAbyz+B2uf+96Jlw2gdZ1W8e7nBJp4UKY\nMgW++irYh/bXX+Hii4NXy5ZQteqhryEiIhKpWIa6ee5+am7vi4PiHOoAPvzxQ/q83Yexl4ylfYP2\nh/6AxNTSpTB+PLz7Lnz/PZQrB82bwx//CJdcou3KRESkYGIZ6vYBv2W+BcoD20O/u7tXzu9NC0tx\nD3UA0/4zjV7je3HTaTdxX7v7SE5KjndJQjBUu2EDzJkDw4cH25X17g1nnw2tW8ORWplGREQOU1xm\nvxYXiRDqAH7a+hN93+nL9j3bee3i1zimyjHxLkmy+f77YC28zz6Dzz8Plklp0yYIeM2bw7HHQrVq\n8a5SRESKMoW6PCRKqAPY7/sZ8tkQnpzzJKO6jaJT407xLklysW9f8DzenDlByFu4EJYsCYZru3YN\nZtaedRYkRbKfi4iIlBgKdXlIpFCXaeaqmVw2/jJubHkj97W7jyRTMigOMmfWjhsXbGP23//C6afD\nqadC27bBsG2yRtZFREo0hbo8JGKog2A4tuebPalSrgqj/zCaquU1DbM4cYcVK+Drr4PXlCmwdi38\n4Q9wzjnQpEnwStHGIiIiJYpCXR4SNdQB7N63mzs/upOJyyYyvud4Tq51crxLkgJYsSKYWfvll7Bs\nWfCqWxfatQuC3jnnQJ068a5SRERiSaEuD4kc6jK9vuB1bv3wVh4+92FuaHkDpj2uEsK+ffDddzBj\nRvCaOROqVAmGaU85BU4+ORi6PeKIeFcqIiLRolCXh5IQ6gAW/7KYXuN70bBqQ17s8iKp5VPjXZJE\n2f79wYSL2bNh/nz45hv44Ycg5HXrBl26QK1a8a5SREQKQqEuDyUl1AHs2ruLgVMG8ubCNxnSaQg9\nTuihXrsEl5EBH3wA77wDH34ITZsGs2s7doQWLbSlmYhIcaNQl4eSFOoyzVw1k37v96NGxRoMu2AY\nxx95fLxLkkKwezdMnw4TJsDUqbBxYzCrtmXLYKi2RQuoXx+U80VEii6FujyUxFAHsHf/XobNHcYj\nnz7Cn1r/iTvb3EnpUqXjXZYUovXrg+fwvv02GK799lvYvj0Id6edFgzbtm2rBZFFRIoShbo8lNRQ\nl2nVllXcMPEGNmzbwMsXvUzL2i3jXZLEUXp6EPC++AI+/TTY+aJevSDgtWsXLIhcr168qxQRKbkU\n6vJQ0kMdgLsz5rsx3PnxnfQ8sScPn/swKeW0AJrA3r1BD97MmUHImz072PXirLP+9zrpJC2KLCJS\nWIp8qDOz84FngCTgZXd/PIc2Q4HOwG/AVe7+bdi5JOArYK27XxQ6VhUYB9QHVgI93f3XHK5b4kNd\npk3bNzFwykA++PEDBqcN5qqTryI5Sf9rLf/jDj/+GIS72bNh1iz46adgmPbcc6FVKzj+eKhRQ8/m\niYjEQpEOdaFAthToAPwEfAlc5u6Lw9p0Bvq7++/N7AzgWXc/M+z8AKAlUDks1D0ObHL3J8zsbqCq\nuw/M4f4Kddl8vvZz7v3kXtZmrOWhcx/i0hMv1SxZyVV6etCLN21asIzKkiXB8iq/+x306gUXXBD0\n7omISMEV9VB3JjDI3TuH3g8EPLy3zsyeB6a5+7jQ+0VAmrtvNLOjgVeBvwB/Cgt1i4FzQm1qAdPd\n/aBpngp1uftkxSfcPeVuypcuz7DOw2hRq0W8S5JiYuNGePddeP31YJuz444LtjU7/ng480w44wyo\nXDneVYqIFD8FDXWx3g2+DrAm7P3a0LG82qwLa/M0cCeQPZnVcPeNAO6+AagRrYJLig4NO/DFtV9w\nZfMrOW/Medw48UYW/rww3mVJMVCzJlx3XbB0yn/+A888A+edB9u2wcMPQ+3awTIqN98MY8bA4sXB\n83siIhJbRfahKjP7PbDR3b81szQgr+Sq7rh8KJVUiutbXk/3pt15+vOn+d2o39GkWhPuaH0HXY7r\nEu/ypBhITYU2bYJXpt27g6HaOXOCdfMGDQqWWDn+eGjWLJh8kfmqXVvP54mIREthDL8OdvfzQ+8j\nGX5dDJwD3Ab0BvYC5YFKwL/dvU+2Idpaoc83zeH+PmjQoKz3aWlppKWlxebLJoA9+/YwYckE/jz1\nzxxb7Vie6/wc9avUj3dZkgC2bQu2OVuw4MDX3r3BHrbnnQedOwehTyFPREqK6dOnM3369Kz3Dz74\nYJF+pq4UsIRgosR6YC7Qy90XhbW5AOgXmihxJvBM+ESJUJtzgP/LNlFis7s/rokS0bdr7y6emvMU\nT3/+NJeeeClXNL+C1ke31oQKibqNG2HuXJg8Odjy7L//DbY7O/74YKHkVq2Codzy5eNdqYhI7BXp\niRKQtaTJs/xvSZPHzOwGgh67F0JthgHnEyxpcrW7z8t2jeyhLhV4A6gLrCJY0mRLDvdWqCuA1b+u\nZvT80by24DV27N1B75N60/fkvjRObRzv0iQBuQezbZcsgUWLgiHcL78Mfq9bNwh64a+mTaFKlXhX\nLSISPUU+1MWTQl10uDvzN85n1PxRvLbgNY6tdmzWc3dJFuu5NlLS7doVrJ+3eHEQ+BYvDl6LFgXP\n5J1xBpxwQjCBo2bNIPA1aKBhXBEpfhTq8qBQF32Zz909Nusxtu/Zzt1n3U2vk3pRplSZeJcmJcze\nvcFzel98AcuWBb18GzbA99/Djh3BHretWv3vZ53s8+5FRIoYhbo8KNTFjrvzyX8+4bFZj7H4l8Xc\ndsZtXN/yem1BJkXC+vXw1VfB8G3mq0yZIOA1axb08NWqBQ0bwoknQtmy8a5YREShLk8KdYVj3vp5\nPDXnKd5f9j4XHnshV598Nec2OFdDs1JkuMPq1UHQW7gwCH3r1wc9fMuXQ+PGQcirVAmqVg12zPj9\n77WIsogULoW6PCjUFa6ff/uZfy34F698+wrb92znptNu4qqTryK1fGq8SxPJ1c6dwfN5P/8MW7cG\nM3InTYKZM4OevQoVICkJqlULnt8788zgGb4yeuJARKJMoS4PCnXx4e58vvZzhn81nPeWvMe5Dc6l\ne9PudDm2i4ZnpdjYsgU+/xz27Al6+tavD95/9hmsWBGEvHr14PTToX17OOecYDFmEZH8UqjLg0Jd\n/G3esZn3lrzH+EXj+XTVp1x47IVc3/J6zq53tta9k2Jr374g5K1cGeycMW0azJoFycnB8iu1awcB\nr2rVIPidcUbQ63fEEfGuXESKMoW6PCjUFS2btm9i9HejeeHrF9i5dyddju3ChcdeSLv67SibrCfV\npXhzh82bYc0a+OmnYCHl//43WI7liy/gu++genU4+uhgJm5OP2vXhtKl4/1NRCReFOryoFBXNLk7\n3238jolLJ/Le0vdY/Mtizmt0HhcddxEXHnshVcppRVlJPLt3B4Fv3brgtXbtwb9v3hwM56alBTtq\nVKsWvI45RrtqiJQECnV5UKgrHjZu28ikZZN4Z/E7zFg1g/YN2nPpiZfSsWFHqlWoFu/yRApNRgbM\nnh0M5y5aBJs2wS+/BGGwYUNo3jzo7atUKRjePfVUaNlSs3RFEoVCXR4U6oqfLTu38Pait3lr0VvM\nXDWTBlUb0KFBB7o37U7ruq21TIqUSLt2BSFvwYIg6G3dGszW/fpr+PbbYNi2bt3gdfTRB/9MTdUO\nGyLFgUJdHhTqirc9+/bw9fqvmfzjZN5a9Babd2yme9Pu9DyxJ23qtlHAEyGYnbt8edCbt3bt/36G\n/75r1/9CXt26wbp8jRtDkybBzxRNShcpEhTq8qBQl1gW/byItxa+9f/t3XlwnPWd5/H3V750S5Zk\n3bZsSxhjbLAgBDNgYzLAAJnAFqnMkmySZXayG5Iw7M7OTg6qtpJJbU0tU5NM7mGZZDKTSShImBxk\nQrhxuDHENja2jCxfkq1b1n1Zx2//+HbTjbEbY0tuqfV5Vf2qu59+JD1NPdX+8P1d/GzPz+ga7uLG\nmvm7OlkAABqDSURBVBvZuGwjm6o2sTx/uWbTipzCwEAs5DU2+uSNhgZffLmhwdfiOzHonXeed/nm\n5/s6fSIy/RTqElCoS117O/fy5IEnea7xOZ49/CwZ8zO4vvp6rq++nmtXXkvuIg0yEjkdIfiCy9GA\nF31saPAKYH+/h77CQt9Sbe1aD3wlJd5WrvRxfiJy9hTqElComxtCCOzu2M0T+5/g0f2P8mLTi2yo\n3MCNNTeyqWoT60vXMz9tfrIvU2RWmpz0Sl97u2+x9sYbHvba2rw1NPgyLGvWeLjLyvKWne2PS5b4\njN6LLtJyLSLvRqEuAYW6uWng+ABPHniSxxoe4/mm5znUc4irll3Fzatu5kPnf4jK3MpkX6JIygjB\n1+Wrq/N1+QYGYHDQ28BAbCeOQ4e8yldTA9XVkJ7u54yMeBWwstIXar74Yu/yFZmLFOoSUKgT8F0t\nnjrwFA/XP8wj+x6hLLuMzcs3s3n5ZmpLa1mev5x5afOSfZkiKa2nB3bvjnXtjo15JS893ZdtOXIE\nDh70Gb6lpR7uqqu9FRTA+LhXDaurobZWe+9KalKoS0ChTk40PjnOjtYdPHPwGZ5tfJadbTvpHOpk\nzZI1bK7azHXV17Fx2UYyFmilV5FkmJiAN9/0cLd/v7eeHt+CzQz27vVQuH69j+crL/cQmJPjITE/\n36t+lZU+q1fzp2Q2UahLQKFOTkf/aD+vt73OUwee4okDT7CtZRurCldxadmlXL38am4+/2btciEy\ng/T1+Rp9jY3e9dvaGuv2PXYstkvHxERsC7YTW/R4UZFm98rMoVCXgEKdnImR8RF2tu3ktebXeGz/\nYzxz8Bk2VW1i47KNXFh8IWuL11KVV6UlVERmuL6+WMCLthNf9/e/cw/e+OCXm+tdxNFJHwqAMp0U\n6hJQqJOp0DfaxyP7HuHVo6+yu2M3u9p3MTYxxobKDWyo3MDa4rWsWbKGFfkrNDZPZJYZHn77/rsn\nBsCBAZ/MMTDgbflyX8A5P9+7d3NzY4/l5d4lvHIlLF6c7E8ms5FCXQIKdTJdjvQd4aWml9h6dCu7\nO3azu2M33cPdXLH0Cq5aehUbKjdwafmlFGQUJPtSRWSKDA76ZI6mJujt9dbXF3t+9CgcOOBt/nwP\nd8uW+fIu6eneMjL8sazMxwWuW+dVQBFQqEtIoU7Opc6hTl5sepHnDj/H1uatbG/ZTmFmIRuXbeTa\nldfygRUfoCKnQt22IikuBN+j98ABH/c3NOTVvuHh2GNTk+/bW1fnlb7oYs6lpf5YXPzOamBenncL\n5+Qk+xPKdFGoS0ChTpJpMkxS31XPlkNbeOrgU2w5tIWR8RFW5K9g5eKV1BTUUFNQw7ridby/4v0s\nmKeVWUXmmvFxX9i5rc0nfEQf29tjFcD4duSIh75167wKWFz89rZkia/7l58P8zQaZNZRqEtAoU5m\nmt6RXg72HORA9wEajjXQcKyB15pfY3/3fjZVbWJN0RpyF+WSl57HpqpNrCtep8qeiLxlYsKXedm1\ny7t729tjra3NH7u7vVu4qMh3+lizxqt8w8Nw/Lh3C9fW+i4fmZk++WPBAk0CmQkU6hJQqJPZomOw\ng6cPPs3BnoP0jfbRNdTF4wceZ0HaAm45/xaW5y9nSdYSKnIqWFeyTnvbikhCExOxnT727PFJHunp\nHt7274dt23wx6NFRP3dy0it8paU+ySM6BrCy0vf8jQbDBQs8CFZW+rhBmVoKdQko1MlsFkJgW8s2\nHtn3CM39zXQMddDY28jujt2U55RTW1rL+tL11JbWsrZ4LZW5larqicgZGR/3nT1aW32tv9FRr+wd\nPuzhr67Og+Hx4z5hpL3du39LS/3Y8LAv9Bzd97e42GcJV1R4SMzJ8V1A2tq8wpiWBtde6xVDVQhj\nFOoSUKiTVDQ+OU59Vz07WnewvWU721u3s6djD32jfawqXEVFbgUlWSVU5FRwWcVlbKjcQFFmUbIv\nW0RSyOioTwRpa4vN6p2cjO3529YWWxomOkt4dNTHA1ZUeAh84gkPkpdc4sfiW3l5LAzm5sKiRcn+\nxOeGQl0CCnUyl/SO9FLfVU9zfzNtg2009jay9ehWXjn6CgUZBazIX0FVfhXLcpdRkVtBeU455xWc\nR3VBNfPT1I9y2r7zHf9X5pOfTPaViMx6jY1eCYxfK/DoUe86jobBvj6v5uXm+jjBmhrfA7ikxKuC\n0epgVpaHwPJy7x5OT0/2p3vvFOoSUKgTgYnJCQ50H+Bw72EO9RyiqbeJo/1HOdJ3hPqueloHWllV\nuIp1Jeu4qPgi1ixZQ3FWMYWZhZRml5K5IDPZH2FmeeYZuOMOH6ik6YUi0y4Er/L193v38P79vv9v\nZ2dse7joY2+vB8LmZg950bGBBQWwapW3zExfZmZoyLuEs7I8MK5e7bOKi4uT91kV6hJQqBN5d4PH\nB9nTsYdd7bvY1baLus46Ooc66Rruom2gjeX5y6ktq2XtkrVvLcOyqnAVWQvn6IqpIcCGDfCFL8Ct\ntyb7akTkJCYnfRbwyIgHwo4OqK+HN9/0Y1lZ3mU8Nubhrrvb/z9t1y4fXzh/vo8RzM72cYMntvJy\n7yYuLY39rvR0/5mzoVCXgEKdyNkZmxijrrOObS3bqOuoo6G7gX1d+2g41kBJdgmri1ZTklVCQUYB\nSzKXsLpoNRcWX0j14urU3jLt5z+He+6Bl18++29xEZkxQvDu3slJb/39sbUDo62lJVYNbG2NVf2O\nH4+NL8zI8HGA6eneTVxV5RNH8vI8BGZmxr46li+HjRv9uUJdAgp1ItNjfHKcg90H2du5l86hTrpH\numnpb6Gus47dHbtpH2xnbfFaLi65mJqCGsqyyyjNLqU4q5iizCKWZC1h4byFyf4YZ25iwtd4uPde\nuOaaZF+NiMwAk5M+ASS6g0h0BnFrq48dbGrywDg46OeAh8jLL4c//3N/rVCXgEKdSHL0jfaxq20X\nO1p3cLDnIC0DLbT0t9A51EnHUAfHho9RnlPO6qLVrCpYxfL85VTlV7EifwU1BTXkLJoF+yB9//vw\n0EPw6KPJvhIRSREzPtSZ2Q3AN4A04AchhHtOcs63gBuBQeD2EMIOM1sEPAssjLRfhRDujpz/ZeC/\nAu2RX3F3COEd36wKdSIzU3ylr76rnsO9h2nsbXxrp43cRbnkLMrh+MRxxifHWZG/gtrSWi4quYjS\n7FKKMosoziqmIreC9PlJmuI2OupL8//4x6rWiciUmNGhzszSgHrgD4Fm4FXgthDC3rhzbgTuDCF8\n0MwuB74ZQtgQeS8zhDBkZvOAF4C/DCG8EAl1/SGEr7/L31eoE5llJsMkzf3NDI8Ns3DeQtIsjX3H\n9rGjdQdvtL9B+2A7nUOdtA600jLQQu6iXEqySshLzyNvUR41BTVvLcxcnlNOQUbB9O2r+8QT8PGP\n+4zYNWum52+IyJxxtqFuuhenej+wL4RwGMDMHgBuAfbGnXML8COAEMIrZpZnZiUhhLYQQqTXmUV4\npa877uc0OlkkBaVZGpW5lW87tjRvKR9Y8YF3nDsZJmkfbKd9sJ3ekV56Rnqo76rnyYNP8rWXvkbb\nYBvdw91kL8ymJLvkrSpf1oIsMhdkUpZdxqrCVawqXMWyvGUUZRa9t105rrsO/u7v4Kab4IUXfDqc\niEiSTHeoqwCa4l4fwYNeonOORo61RSp9vweqgXtDCHvizrvTzD4BvIZX8Hqn+uJFZGZLszRKs0sp\nzS495TmTYZLekV7aBttoHWilc6iTweODDI4N0tzfzC/2/oL6rnqa+poYPD5IWU4Zy/KWUZVXxdLc\npZRkl7w1s7e2rPadf+ATn/DVUm+8EX77WwU7EUmaGb2MfAhhEqg1s1zgcTO7OoTwO+B7wFdDCMHM\n/g/wdeDPknmtIjIzpVkaizMWszhjMauLVic8d3hsmOb+Zhp7Gznce5im3iYajjXwYtOLHOo5dPJQ\nB75mXQhw2WXwk59ojJ2IJMV0h7qjwLK415WRYyeeszTROSGEPjP7DfA+4HchhI64t/8R+PWpLuAr\nX/nKW883b97M5s2bT//qRWROyViQQXVBNdUF1e/tB83gS1/yUPexj8Fdd8HnP68dJ0QkoS1btrBl\ny5Yp+33TPVFiHvAmPlGiBdgKfDSEUBd3zk3A5yITJTYA3wghbDCzImAshNBrZhnAY8BfhxCeMrPS\nEEJr5Of/ArgshPCxk/x9TZQQkXOrqcm7ZCcm4J//2TepFBE5DWc7USJtKi/mRCGECeBO4HFgN/BA\nCKHOzD5tZv8tcs4jwEEzawD+H/DZyI+XAc+Y2XbgZeDhEMJTkff+1sx2mtkO4GrgL6bzc4iInLal\nS+Hpp30LsQ0b4Nvf9oAnIjLNtPiwiMh0qauDz3zGdxn/7nfhD/4g2VckIjPYjK7UiYjMaRdc4GvY\n/dVfwUc+4mvaNTYm+6pEJEUp1ImITCcznzyxd6/vQFFb65MqOjuTfWUikmIU6kREzoWcHPjqV2Hn\nTujqgvPOg099CnbtSvaViUiKUKgTETmXKirgvvugvh5WrIAbboBrr4Xf/AYmJ5N9dSIyi2mihIhI\nMh0/Dj/9Kfz930N/v0+suP12WLw42VcmIufY2U6UUKgTEZkJQoCXXoJ/+Af493+HP/5juO023192\n4cJkX52InAMKdQko1InIrNTRAQ8+CA884BMsPvxh+OQnfUkUO+PvexGZ4RTqElCoE5FZr7ER7r8f\nfvQjGB31pVFuvdW3JFPAE0kpCnUJKNSJSMoIAbZvh4cegn/7Nxgagg99yNs110B6erKvUETOkkJd\nAgp1IpKSQvDdKn79ax9/9/rrcPXVcNNN3qqqkn2FInIGFOoSUKgTkTnh2DF4/HH47W+9lZT4RItr\nr/X9Z7Oykn2FInIaFOoSUKgTkTlnYgJefdUreFu2wI4dsG4dbN7s7corITs7yRcpIiejUJeAQp2I\nzHlDQ/DKKx7wnnkGfv97WLsWNm6Eq67ySl5pabKvUkRQqEtIoU5E5ATDw7B1Kzz3HDz/vD/PzYUr\nrvAq3pVXemVv/vxkX6nInKNQl4BCnYjIuwjBtyx76SV44QUPekeOQG0tXH45vO99cOmlUF2tJVRE\npplCXQIKdSIiZ6Cnx8flbd3q3bWvvQa9vbBmDVx4oVfy1q+Hiy+G/PxkX61IylCoS0ChTkRkinR1\nwe7dsGcP7Nzpy6js3AmFhR7w1q/36t769bBsmap6ImdAoS4BhToRkWk0OQn79/sM2+3bY4+joz4Z\nY+1ar+7V1Hhbvlxj9UQSUKhLQKFORCQJ2trgjTdilb39+2HfPmhthZUr4YIL4Pzz4bzzYNUqf1yy\nRNU9mfMU6hJQqBMRmUFGRnxSRl2dP+7bF3scH/eAt24dXHSRV/iWLoXKSsjJSfaVi5wTCnUJKNSJ\niMwSx47Bm2/Crl0+Xq+uDo4ehaYmWLAAKiq81dR4pe+CC7zCt3QpzJuX7KsXmRIKdQko1ImIzHIh\n+Gzco0d9qZV9+zzw1dVBQwN0dPhet0uX+gSNpUtjz6uq/DEjI9mfQuS0KNQloFAnIpLihofh4EGv\n6MW3xkY4fNifL17s3bhlZVBe7s+jXbvl5d5yczWmT5JOoS4BhToRkTluchJaWrzS19wcq/gdOeKB\nr6XFj09OesiLBr6qKm+lpT6Jo7jYQ+HChcn+RJLCFOoSUKgTEZHT0tcXC3zRKt/hwz6Tt6MD2tv9\neWGhB7/SUm/l5T7WL3qspMQD4IIFyf5EMgsp1CWgUCciIlNmYsKXZTlyxANea6sHwWgYbG31452d\nkJ0NRUVe5YsGwLKyWCWwvNzDX0EBpKUl+5PJDKFQl4BCnYiInHOTkz65o7MzVuFrbfVu3vhu37Y2\nGBjw4FdW5sGvqMirgYWFXvWLVv4KCrzl5ysEpjCFugQU6kREZEY7ftzDXUuLt64ub9FA2Nrq3b/d\n3b7sS3+/B7to6Csr81ZY6BNCFi+GvDyf+LF4cSwkaizgrKBQl4BCnYiIpJTxcQ94nZ0eBpubPQwe\nO+bHu7t9fGBvrz+PhsTs7Fg1MC/PF3TOyfHXFRWxYFhY6BXBvDxVBJNAoS4BhToREZnzQvDQ19rq\nrbfXu317e2PjAuOrhF1dMDjo1b5o0Css9AphXl4sFGZn+7HoRJElSyAzE9LTtTzMGVKoS0ChTkRE\n5AyMj3voO3YsFvR6evxYT4+HvoEBfz86UaSzE4aGYGzMQ19eXqyruKjIK4DZ2ZCV5e9FxwwWFnqA\nzM31n1u0aM6GQoW6BBTqREREzrHxcR/7Fw2A0TGC0QpgtErY1uatu9tf9/b6+2NjHv4KC736F+0O\njobE+Ekjublv707OyfHQOEvHECrUJaBQJyIiMstEQ+GxYx4GOztjoa+nJzZpJHqst9fPj7bBQf89\n0e7h6ASSggIPinl5/l5OTqw6mJvr50ZbXp53I59jMz7UmdkNwDeANOAHIYR7TnLOt4AbgUHg9hDC\nDjNbBDwLLIy0X4UQ7o6cvxh4EKgCDgF/EkLoPcnvVagTERGZa8bGPODFh8DoxJG+Pm/xQTA+NEa7\nmcGrfunp3vLyYl3J0eCXm+vvLVrk5xYVxd7PyvLwmJl52l3KMzrUmVkaUA/8IdAMvArcFkLYG3fO\njcCdIYQPmtnlwDdDCBsi72WGEIbMbB7wAvCXIYQXzOweoCuE8Ldm9gVgcQjhiyf5+wp1clq2bNnC\n5s2bk30ZMkvofpHTpXtlFhsZ8a7i0VF/3tsbqxxGg19fn78/OurnRruao5NRBgZi4wzT0yEjwx+z\nsnz9wZISuO46uOMO4OxD3fwp+/An935gXwjhMICZPQDcAuyNO+cW4EcAIYRXzCzPzEpCCG0hhKHI\nOYvwSl933M9cHXn+L8AW4B2hTuR06YtX3gvdL3K6dK/MYtEK3VSYmPBgODICw8Me9trbvRUWTs3f\nYPpDXQXQFPf6CB70Ep1zNHKsLVLp+z1QDdwbQtgTOac4hNAGEEJoNbPi6bh4ERERkbM2b55X57Ky\nYsdWr57yPzOjVxYMIUyGEGqBSmCTmV19qlPP4WWJiIiIzDjTPaZuA/CVEMINkddfBEL8ZAkzuxd4\nJoTwYOT1XuDqaCUu7rz/DQyFEL5mZnXA5hBCm5mVRn7+gpP8fYU9ERERmTVm8pi6V4EaM6sCWoDb\ngI+ecM7DwOeAByMhsCcS1oqAsRBCr5llANcBfx33M7cD9wD/GfjVyf742fyHEREREZlNpjXUhRAm\nzOxO4HFiS5rUmdmn/e1wXwjhETO7ycwa8CVN/jTy42XAv5iZRX72X0MIT0Xeuwf4qZn9F+Aw8CfT\n+TlEREREZrqUXnxYREREZK6Y0RMlzpSZ3WBme82sPrKOncjbmNkhM3vdzLab2dbIscVm9riZvWlm\nj5lZXrKvU5LDzH5gZm1mtjPu2CnvDzP7kpntM7M6M7s+OVctyXCKe+XLZnbEzLZF2g1x7+lemaPM\nrNLMnjaz3Wa2y8zuihyfsu+WlAt1kWVQvgP8EXAh8FEzm/p5wzLbTeKTbWpDCNFldr4IPBlCOB94\nGvhS0q5Oku2H+HdIvJPeH2a2Bh8CcgG+M873IsNGZG442b0C8PUQwiWR9iiAmV2A7pW5bBz4nyGE\nC4ErgM9F8smUfbekXKgjbsHjEMIYEF3wWCRedKxmvFvwxayJPP6Hc3pFMmOEEJ4ntth51Knuj5uB\nB0II4yGEQ8A+3rkep6SoU9wr4N8xJ7oF3StzVgihNYSwI/J8AKjDl2ybsu+WVAx1J1vwuCJJ1yIz\nVwCeMLNXzexTkWMl8YtaA1rUWuIVn+L+ONUC6jK33WlmO8zs+3HdabpXBAAzWw6sB17m1P/2vOf7\nJRVDncjpuDKEcAlwE14C38g7F7HWLCJJRPeHnMr3gJUhhPVAK/C1JF+PzCBmlg08BPz3SMVuyv7t\nScVQdxRYFve6MnJM5C0hhJbIYwfwS7yk3WZmJQCRRa3bk3eFMgOd6v44CiyNO0/fOXNcCKEjxJaW\n+EdiXWa6V+Y4M5uPB7p/DSFE19idsu+WVAx1by14bGYL8QWPH07yNckMYmaZkf9TwsyygOuBXcQW\ntYYEi1rLnGG8fVzUqe6Ph4HbzGyhma0AaoCt5+oiZUZ4270S+Yc56lbgjchz3SvyT8CeEMI3445N\n2XfLdO8occ6dasHjJF+WzCwlwC8i28jNB34SQnjczF5Di1oLYGb3A5uBQjNrBL4M/F/gZyfeHyGE\nPWb2U2APMAZ8Nq5KIynuFPfKNWa2Hp9lfwj4NOhemevM7ErgPwG7zGw73s16N6fYUOFM7hctPiwi\nIiKSAlKx+1VERERkzlGoExEREUkBCnUiIiIiKUChTkRERCQFKNSJiIiIpACFOhEREZEUoFAnInOK\nmU2Y2TYz2x55/PwU/u4qM9s1Vb9PROS9SLnFh0VE3sVgZN/f6aLFP0UkKVSpE5G5xk560Oygmd1j\nZjvN7GUzWxk5XmVmT5nZDjN7wswqI8eLzeznkePbzWxD5FfNN7P7zOwNM3vUzBZFzr/LzHZHzr//\nnHxSEZlTFOpEZK7JOKH79SNx73WHEC4CvgtE92b8NvDDEMJ64P7Ia4BvAVsixy8BdkeOnwd8O4Sw\nFugFPhw5/gVgfeT8O6brw4nI3KVtwkRkTjGzvhBC7kmOHwSuCSEcMrP5QEsIYYmZdQClkX2l5wPN\nIYRiM2sHKkIIY3G/owp4PIRwfuT154H5IYS/MbNHgEHgl8AvQwiD0/9pRWQuUaVORCQmnOL5ezEa\n93yC2NjlDwLfwat6r5qZvn9FZErpS0VE5pqTjqmL+I+Rx9uAlyLPXwA+Gnn+ceC5yPMngc8CmFma\nmUWrf6f6/ctCCL8DvgjkAtnv/dJFRE5Ns19FZK5JN7NtePgKwKMhhLsj7y02s9eBEWJB7i7gh2b2\nv4AO4E8jx/8HcJ+Z/RkwDnwGaOUkFb5It+2PI8HPgG+GEPqm5dOJyJylMXUiIrw1pu7SEMKxZF+L\niMiZUPeriIjT/+GKyKymSp2IiIhIClClTkRERCQFKNSJiIiIpACFOhEREZEUoFAnIiIikgIU6kRE\nRERSgEKdiIiISAr4/5Ns3nGACRg3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f203b8b3cd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt1, = plt.plot(ae_scratch.history.history['loss'][:200], label='Vanilla AE')\n",
    "plt2, = plt.plot(decoder.history.history['loss'], label='t-SNE Generator')\n",
    "plt3, = plt.plot(range(100, 200), ae.history.history['loss'], label='t-SNE AE')\n",
    "plt.legend(handles=[plt1, plt2, plt3])\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Reconstruction Loss (MSE)')\n",
    "# plt.plot(encoder.history.history['loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ae_scratch_encoder = get_encoder(ae_scratch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ae_scratch_encoded = ae_scratch_encoder(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ae_scratch_decoder = Sequential()\n",
    "ae_scratch_decoder.add(Dense(2000, activation='relu', weights=ae_scratch.layers[4].get_weights(), input_shape=(2,)))\n",
    "ae_scratch_decoder.add(Dense(500, activation='relu', weights=ae_scratch.layers[5].get_weights()))\n",
    "ae_scratch_decoder.add(Dense(500, activation='relu', weights=ae_scratch.layers[6].get_weights()))\n",
    "ae_scratch_decoder.add(Dense(X_train.shape[1], weights=ae_scratch.layers[7].get_weights()))\n",
    "%time ae_scratch_decoder.compile(loss='mse', optimizer='rmsprop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sample_grid(ae_scratch_decoder, ae_scratch_encoded, filename='output-scratch.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}