Pseudo-Labelling-MNIST-1st.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pseudo Labelling - MNIST",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNHsX+B1Kzz6fepM83FAySt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/consequencesunintended/Pseudo-Labelling/blob/master/Pseudo-Labelling-MNIST-1st.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nt86cVf9qy7-",
        "colab_type": "text"
      },
      "source": [
        "# **Pseudo Labelling on MNIST dataset**\n",
        "\n",
        "References:\n",
        "\n",
        "*1 - Pseudo-Label : The Simple and Efficient Semi-Supervised Learning\n",
        "Method for Deep Neural Networks, Dong-Hyun Lee\n",
        "http://deeplearning.net/wp-content/uploads/2013/03/pseudo_label_final.pdf*\n",
        "\n",
        "*2 - Naive semi-supervised deep learning using pseudo-label, Zhun Li, ByungSoo Ko & Ho-Jin Choi\n",
        "https://link.springer.com/article/10.1007/s12083-018-0702-9*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ZyrRGAgyGod",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.layers import Input, Dense, Reshape, Flatten, Softmax, Dropout\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras import losses\n",
        "from tensorflow.keras import Sequential\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t5axBk82yU61",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.datasets import mnist\n",
        "import numpy as np\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htVHk9a5z-4N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier = Sequential([\n",
        "    Flatten(input_shape=(28, 28)),\n",
        "    Dense(128, activation='relu'),\n",
        "    Dropout(0.3),\n",
        "    Dense(10)\n",
        "])\n",
        "classifier.build()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlACEjfzyWTl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train = x_train.astype('float32') / 255.\n",
        "x_test = x_test.astype('float32') / 255.\n",
        "x_train = x_train.reshape( (len(x_train),28,28,1))\n",
        "x_test = x_test.reshape( (len(x_test),28,28,1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxW_OQCcpHiw",
        "colab_type": "text"
      },
      "source": [
        "# **Split the data**\n",
        "The idea here is to have a small labelled dataset and a large unlablled set to help improve increasing the accuracy of the model in scenario were we dont have access to large labelled set. We also set aside some data for evaluation test."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8EBnOFUfyjm_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_0 = [ x for x,y in zip(x_train,y_train) if y == 0 ] \n",
        "data_1 = [ x for x,y in zip(x_train,y_train) if y == 1 ] \n",
        "data_2 = [ x for x,y in zip(x_train,y_train) if y == 2 ] \n",
        "data_3 = [ x for x,y in zip(x_train,y_train) if y == 3 ] \n",
        "data_4 = [ x for x,y in zip(x_train,y_train) if y == 4 ] \n",
        "data_5 = [ x for x,y in zip(x_train,y_train) if y == 5 ] \n",
        "data_6 = [ x for x,y in zip(x_train,y_train) if y == 6 ] \n",
        "data_7 = [ x for x,y in zip(x_train,y_train) if y == 7 ] \n",
        "data_8 = [ x for x,y in zip(x_train,y_train) if y == 8 ] \n",
        "data_9 = [ x for x,y in zip(x_train,y_train) if y == 9 ] "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0PqmlE1M2hQc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "source = 0\n",
        "target = 10\n",
        "x_train_shorten = data_0[source:target] + data_1[source:target] + data_2[source:target] + data_3[source:target] + data_4[source:target] + data_5[source:target] + data_6[source:target] + data_7[source:target] + data_8[source:target] + data_9[source:target]\n",
        "y_train_shorten = list(np.zeros(target)) + list(np.ones(target)) + list( np.ones(target) * 2 ) + list( np.ones(target) * 3 ) + list( np.ones(target) * 4 ) + list( np.ones(target) * 5 ) + list( np.ones(target) * 6 ) + list( np.ones(target) * 7 ) + list( np.ones(target) * 8 ) + list( np.ones(target) * 9 )\n",
        "\n",
        "x_train_shorten = np.array( x_train_shorten )\n",
        "y_train_shorten = np.array( y_train_shorten )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lBGrsleIK70U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "source = 10\n",
        "target = 250\n",
        "x_train_unlabelled = data_0[source:target] + data_1[source:target] + data_2[source:target] + data_3[source:target] + data_4[source:target] + data_5[source:target] + data_6[source:target] + data_7[source:target] + data_8[source:target] + data_9[source:target]\n",
        "y_train_unlabelled = list(np.zeros(target - source)) + list(np.ones(target - source)) + list( np.ones(target - source) * 2 ) + list( np.ones(target - source) * 3 ) + list( np.ones(target - source) * 4 ) + list( np.ones(target - source) * 5 ) + list( np.ones(target - source) * 6 ) + list( np.ones(target - source) * 7 ) + list( np.ones(target - source) * 8 ) + list( np.ones(target - source) * 9 )\n",
        "\n",
        "x_train_unlabelled = np.array( x_train_unlabelled )\n",
        "y_train_unlabelled = np.array( y_train_unlabelled )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WRPcnIQLOST",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "source = 250\n",
        "target = 450\n",
        "x_eval_shorten = data_0[source:target] + data_1[source:target] + data_2[source:target] + data_3[source:target] + data_4[source:target] + data_5[source:target] + data_6[source:target] + data_7[source:target] + data_8[source:target] + data_9[source:target]\n",
        "y_eval_shorten = list(np.zeros(target - source)) + list(np.ones(target - source)) + list( np.ones(target - source) * 2 ) + list( np.ones(target - source) * 3 ) + list( np.ones(target - source) * 4 ) + list( np.ones(target - source) * 5 ) + list( np.ones(target - source) * 6 ) + list( np.ones(target - source) * 7 ) + list( np.ones(target - source) * 8 ) + list( np.ones(target - source) * 9 )\n",
        "\n",
        "x_eval_shorten = np.array( x_eval_shorten )\n",
        "y_eval_shorten = np.array( y_eval_shorten )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5PvmPHds52Ir",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier.compile(optimizer='adam',\n",
        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R7txwYuyog8o",
        "colab_type": "text"
      },
      "source": [
        "# **Define the datasets**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EU40O0A84sSo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 100\n",
        "\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train_shorten,y_train_shorten))\n",
        "train_dataset = train_dataset.shuffle(1000).batch(batch_size)\n",
        "\n",
        "pseudo_dataset = tf.data.Dataset.from_tensor_slices((x_train_unlabelled))\n",
        "pseudo_dataset = pseudo_dataset.shuffle(1000).batch(batch_size)\n",
        "\n",
        "eval_dataset = tf.data.Dataset.from_tensor_slices((x_eval_shorten,y_eval_shorten))\n",
        "eval_dataset = eval_dataset.shuffle(1000).batch(batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ujmYyVPhvNk",
        "colab_type": "text"
      },
      "source": [
        "# **Pre-train the Model with labelled data**\n",
        "Firstly we pre-train the model here with 30 epochs on only labelled data, and achieving a maximum accuracy of around 70 percent."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TX6I91756zE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "classifier.fit(train_dataset, validation_data=eval_dataset, batch_size=100, epochs=30)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6bLYyqHow_a",
        "colab_type": "text"
      },
      "source": [
        "## **Train the model with unlabelled dataset**\n",
        "Train the model with unlabelled dataset and fine tune it by training the model for one epoch on labelled dataset. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBv30SM7c7r9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "avg_main_loss = tf.keras.metrics.Mean(name='avg_main_loss', dtype=tf.float32)\n",
        "pseudo_steps = int(x_train_unlabelled.shape[0] / batch_size )\n",
        "eval_steps = int(x_eval_shorten.shape[0] / batch_size )\n",
        "\n",
        "epoch = 30\n",
        "step = 100\n",
        "\n",
        "images, labels = next(iter(train_dataset))\n",
        "eval_images, eval_labels = next(iter(eval_dataset))\n",
        "pseudo_images = next(iter(pseudo_dataset))\n",
        "pseudo_labels = classifier.predict(pseudo_images)\n",
        "pseudo_labels = np.array( [ np.argmax(x) for x in pseudo_labels])\n",
        "\n",
        "classifier_optimizer = tf.keras.optimizers.Adam()\n",
        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "T1 = 100\n",
        "T2 = 700\n",
        "af = 3\n",
        "step_value = 1.0\n",
        "\n",
        "def alpha_weight(step):\n",
        "    if step < T1:\n",
        "        return 0.0\n",
        "    elif step > T2:\n",
        "        return af\n",
        "    else:\n",
        "         return ((step-T1) / (T2-T1))*af\n",
        "\n",
        "for epoch_idx in range( epoch ):\n",
        "\n",
        "  print_info = True\n",
        "\n",
        "  for batch_idx in range(pseudo_steps):\n",
        "\n",
        "    with tf.GradientTape() as pseudo_tape:\n",
        "        generated_images = classifier(images, training=True)\n",
        "        generated_pesudo_images = classifier(pseudo_images, training=False)\n",
        "\n",
        "\n",
        "        main_loss = loss_fn(labels, generated_images)\n",
        "        psuedo_loss = alpha_weight(step) + loss_fn( pseudo_labels, generated_pesudo_images )\n",
        "        loss = psuedo_loss + main_loss\n",
        "\n",
        "    images, labels = next(iter(train_dataset))\n",
        "    \n",
        "    pseudo_images = next(iter(pseudo_dataset))\n",
        "    pseudo_labels = classifier.predict(pseudo_images)\n",
        "    pseudo_labels = np.array( [ np.argmax(x) for x in pseudo_labels])\n",
        "\n",
        "    gradients_of_classifier = pseudo_tape.gradient(loss, classifier.trainable_variables)\n",
        "    classifier_optimizer.apply_gradients(zip(gradients_of_classifier, classifier.trainable_variables))\n",
        "\n",
        "    if ( print_info ): \n",
        "      print(\"Epoch {} - Step {}\".format( epoch_idx + 1, step ) )\n",
        "      classifier.evaluate(eval_dataset)\n",
        "\n",
        "    avg_main_loss.update_state(loss)\n",
        "    main_loss = avg_main_loss.result()\n",
        "\n",
        "\n",
        "    print_info = False\n",
        "\n",
        "    step += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fZgdYOPJUwQJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}