cifar10_distillation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attempt to train an mnist model and distill it into a smaller model\n",
    "# conclusion, distillation is a scam, at least for mnist\n",
    "# works on CIFAR! at least a little"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import os\n",
    "import time\n",
    "import random\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train, x_test = [x/255.0 for x in [x_train, x_test]]\n",
    "y_train, y_test = [tf.keras.utils.to_categorical(x) for x in [y_train, y_test]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_m(scale=64, loss='categorical_crossentropy'):\n",
    "    x = in1 = Input((32,32,3))\n",
    "\n",
    "    x = Conv2D(scale, (3,3), padding='same')(x)\n",
    "    x = ELU()(x)\n",
    "    x = Conv2D(scale, (3,3), padding='same')(x)\n",
    "    x = ELU()(x)\n",
    "    x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "    x = Conv2D(2*scale, (3,3), padding='same')(x)\n",
    "    x = ELU()(x)\n",
    "    x = Conv2D(2*scale, (3,3), padding='same')(x)\n",
    "    x = ELU()(x)\n",
    "    x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(512)(x)\n",
    "    x = ELU()(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(10, activation='softmax')(x)\n",
    "\n",
    "    m = Model(in1, x)\n",
    "    m.compile('adam', loss, metrics=['accuracy'])\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "elu (ELU)                    (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 32, 32, 32)        9248      \n",
      "_________________________________________________________________\n",
      "elu_1 (ELU)                  (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 16, 16, 64)        18496     \n",
      "_________________________________________________________________\n",
      "elu_2 (ELU)                  (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 16, 16, 64)        36928     \n",
      "_________________________________________________________________\n",
      "elu_3 (ELU)                  (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               2097664   \n",
      "_________________________________________________________________\n",
      "elu_4 (ELU)                  (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 2,168,362\n",
      "Trainable params: 2,168,362\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "m = get_m(32)\n",
    "m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/25\n",
      "50000/50000 [==============================] - 9s 181us/sample - loss: 1.4052 - accuracy: 0.5020 - val_loss: 1.1197 - val_accuracy: 0.6124\n",
      "Epoch 2/25\n",
      "50000/50000 [==============================] - 5s 91us/sample - loss: 1.0535 - accuracy: 0.6350 - val_loss: 1.0033 - val_accuracy: 0.6538\n",
      "Epoch 3/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.9297 - accuracy: 0.6784 - val_loss: 0.9338 - val_accuracy: 0.6824\n",
      "Epoch 4/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.8382 - accuracy: 0.7088 - val_loss: 0.8846 - val_accuracy: 0.6943\n",
      "Epoch 5/25\n",
      "50000/50000 [==============================] - 5s 93us/sample - loss: 0.7549 - accuracy: 0.7354 - val_loss: 0.9046 - val_accuracy: 0.6943\n",
      "Epoch 6/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.6684 - accuracy: 0.7657 - val_loss: 0.8685 - val_accuracy: 0.7118\n",
      "Epoch 7/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.5761 - accuracy: 0.7961 - val_loss: 0.8964 - val_accuracy: 0.7173\n",
      "Epoch 8/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.4948 - accuracy: 0.8267 - val_loss: 0.8809 - val_accuracy: 0.7231\n",
      "Epoch 9/25\n",
      "50000/50000 [==============================] - 5s 91us/sample - loss: 0.4187 - accuracy: 0.8516 - val_loss: 0.9286 - val_accuracy: 0.7262\n",
      "Epoch 10/25\n",
      "50000/50000 [==============================] - 5s 91us/sample - loss: 0.3464 - accuracy: 0.8790 - val_loss: 1.0765 - val_accuracy: 0.7256\n",
      "Epoch 11/25\n",
      "50000/50000 [==============================] - 5s 93us/sample - loss: 0.2938 - accuracy: 0.8958 - val_loss: 1.0707 - val_accuracy: 0.7289\n",
      "Epoch 12/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.2612 - accuracy: 0.9068 - val_loss: 1.0359 - val_accuracy: 0.7344\n",
      "Epoch 13/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.2291 - accuracy: 0.9196 - val_loss: 1.1819 - val_accuracy: 0.7331\n",
      "Epoch 14/25\n",
      "50000/50000 [==============================] - 5s 90us/sample - loss: 0.2032 - accuracy: 0.9280 - val_loss: 1.2287 - val_accuracy: 0.7304\n",
      "Epoch 15/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.1964 - accuracy: 0.9320 - val_loss: 1.2550 - val_accuracy: 0.7315\n",
      "Epoch 16/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.1837 - accuracy: 0.9369 - val_loss: 1.3017 - val_accuracy: 0.7294\n",
      "Epoch 17/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.1686 - accuracy: 0.9431 - val_loss: 1.3787 - val_accuracy: 0.7320\n",
      "Epoch 18/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.1569 - accuracy: 0.9473 - val_loss: 1.4174 - val_accuracy: 0.7331\n",
      "Epoch 19/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.1390 - accuracy: 0.9523 - val_loss: 1.6001 - val_accuracy: 0.7363\n",
      "Epoch 20/25\n",
      "50000/50000 [==============================] - 4s 90us/sample - loss: 0.1475 - accuracy: 0.9517 - val_loss: 1.5125 - val_accuracy: 0.7304\n",
      "Epoch 21/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.1447 - accuracy: 0.9530 - val_loss: 1.4285 - val_accuracy: 0.7378\n",
      "Epoch 22/25\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.1384 - accuracy: 0.9551 - val_loss: 1.4930 - val_accuracy: 0.7307\n",
      "Epoch 23/25\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.1404 - accuracy: 0.9555 - val_loss: 1.6774 - val_accuracy: 0.7331\n",
      "Epoch 24/25\n",
      "50000/50000 [==============================] - 5s 91us/sample - loss: 0.1301 - accuracy: 0.9578 - val_loss: 1.6650 - val_accuracy: 0.7330\n",
      "Epoch 25/25\n",
      "50000/50000 [==============================] - 4s 90us/sample - loss: 0.1338 - accuracy: 0.9563 - val_loss: 1.6164 - val_accuracy: 0.7375\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f94db91bbe0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.fit(x_train, y_train, validation_data=(x_test, y_test), batch_size=128, epochs=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rv = m.predict(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 32, 32, 3)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 32, 32, 16)        448       \n",
      "_________________________________________________________________\n",
      "elu_5 (ELU)                  (None, 32, 32, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 32, 32, 16)        2320      \n",
      "_________________________________________________________________\n",
      "elu_6 (ELU)                  (None, 32, 32, 16)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 16, 16, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 16, 16, 32)        4640      \n",
      "_________________________________________________________________\n",
      "elu_7 (ELU)                  (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, 16, 16, 32)        9248      \n",
      "_________________________________________________________________\n",
      "elu_8 (ELU)                  (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 8, 8, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               1049088   \n",
      "_________________________________________________________________\n",
      "elu_9 (ELU)                  (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,070,874\n",
      "Trainable params: 1,070,874\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sm1 = get_m(16)\n",
    "sm2 = get_m(16)\n",
    "sm1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 6s 125us/sample - loss: 1.4395 - accuracy: 0.4919 - val_loss: 1.1357 - val_accuracy: 0.6114\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 1.1053 - accuracy: 0.6166 - val_loss: 1.0645 - val_accuracy: 0.6317\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 5s 91us/sample - loss: 0.9900 - accuracy: 0.6567 - val_loss: 0.9986 - val_accuracy: 0.6600\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.9267 - accuracy: 0.6835 - val_loss: 0.9554 - val_accuracy: 0.6734\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 5s 93us/sample - loss: 0.8745 - accuracy: 0.6989 - val_loss: 0.9285 - val_accuracy: 0.6810\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 4s 89us/sample - loss: 0.8285 - accuracy: 0.7137 - val_loss: 0.9187 - val_accuracy: 0.6867\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 5s 96us/sample - loss: 0.7715 - accuracy: 0.7341 - val_loss: 0.8944 - val_accuracy: 0.6974\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 5s 93us/sample - loss: 0.7258 - accuracy: 0.7476 - val_loss: 0.8849 - val_accuracy: 0.6974\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 5s 94us/sample - loss: 0.6746 - accuracy: 0.7641 - val_loss: 0.8332 - val_accuracy: 0.7218\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.6188 - accuracy: 0.7840 - val_loss: 0.8367 - val_accuracy: 0.7236\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f95d09744e0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm1.fit(x_train, rv, validation_data=(x_test, y_test), batch_size=128, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 6s 126us/sample - loss: 1.4627 - accuracy: 0.4831 - val_loss: 1.1588 - val_accuracy: 0.6005\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 4s 89us/sample - loss: 1.0950 - accuracy: 0.6196 - val_loss: 1.0425 - val_accuracy: 0.6417\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.9938 - accuracy: 0.6565 - val_loss: 0.9906 - val_accuracy: 0.6576\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 5s 92us/sample - loss: 0.9347 - accuracy: 0.6761 - val_loss: 0.9704 - val_accuracy: 0.6699\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 5s 90us/sample - loss: 0.8806 - accuracy: 0.6935 - val_loss: 0.9296 - val_accuracy: 0.6851\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 5s 93us/sample - loss: 0.8321 - accuracy: 0.7114 - val_loss: 0.9343 - val_accuracy: 0.6793\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 5s 93us/sample - loss: 0.7822 - accuracy: 0.7285 - val_loss: 0.8867 - val_accuracy: 0.7035\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 5s 95us/sample - loss: 0.7350 - accuracy: 0.7414 - val_loss: 0.8631 - val_accuracy: 0.7159\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 4s 89us/sample - loss: 0.6830 - accuracy: 0.7594 - val_loss: 0.8548 - val_accuracy: 0.7195\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 5s 95us/sample - loss: 0.6314 - accuracy: 0.7788 - val_loss: 0.8624 - val_accuracy: 0.7203\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f95b119b588>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm2.fit(x_train, y_train, validation_data=(x_test, y_test), batch_size=128, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}