4_keras_mnist_SHERPA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mnist_mlp.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/peterjsadowski/keras_tutorial/blob/master/4_keras_mnist_SHERPA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "Q1pD2rGo0TGx",
        "colab_type": "code",
        "outputId": "8dbd0007-4d6a-404c-9790-03f3b1e8668b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 436
        }
      },
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "SHERPA is a Python library for hyperparameter tuning of machine learning models.\n",
        "Copyright (C) 2018  Lars Hertel, Peter Sadowski, and Julian Collado.\n",
        "\n",
        "This file is part of SHERPA.\n",
        "\n",
        "SHERPA is free software: you can redistribute it and/or modify\n",
        "it under the terms of the GNU General Public License as published by\n",
        "the Free Software Foundation, either version 3 of the License, or\n",
        "(at your option) any later version.\n",
        "\n",
        "SHERPA is distributed in the hope that it will be useful,\n",
        "but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
        "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n",
        "GNU General Public License for more details.\n",
        "\n",
        "You should have received a copy of the GNU General Public License\n",
        "along with SHERPA.  If not, see <http://www.gnu.org/licenses/>.\n",
        "\n",
        "INSTALLATION: \n",
        "pip install parameter-sherpa\n",
        "pip install flask==0.12.2 # Newer version of flash leads to error: 'io.UnsupportedOperation: not writable'\n",
        "\"\"\"\n",
        "!pip install parameter-sherpa\n",
        "!pip install flask==0.12.2\n",
        "\n",
        "from __future__ import print_function\n",
        "import sherpa\n",
        "import time\n",
        "import keras\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.datasets import mnist"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: parameter-sherpa in /usr/local/lib/python3.6/dist-packages (0.0.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (1.1.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (0.22.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (0.20.2)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (3.7.2)\n",
            "Requirement already satisfied: enum34 in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (1.1.6)\n",
            "Requirement already satisfied: flask in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (0.12.2)\n",
            "Requirement already satisfied: numpy> in /usr/local/lib/python3.6/dist-packages (from parameter-sherpa) (1.14.6)\n",
            "Requirement already satisfied: python-dateutil>=2 in /usr/local/lib/python3.6/dist-packages (from pandas->parameter-sherpa) (2.5.3)\n",
            "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas->parameter-sherpa) (2018.9)\n",
            "Requirement already satisfied: Werkzeug>=0.7 in /usr/local/lib/python3.6/dist-packages (from flask->parameter-sherpa) (0.14.1)\n",
            "Requirement already satisfied: Jinja2>=2.4 in /usr/local/lib/python3.6/dist-packages (from flask->parameter-sherpa) (2.10)\n",
            "Requirement already satisfied: click>=2.0 in /usr/local/lib/python3.6/dist-packages (from flask->parameter-sherpa) (7.0)\n",
            "Requirement already satisfied: itsdangerous>=0.21 in /usr/local/lib/python3.6/dist-packages (from flask->parameter-sherpa) (1.1.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2->pandas->parameter-sherpa) (1.11.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.4->flask->parameter-sherpa) (1.1.1)\n",
            "Requirement already satisfied: flask==0.12.2 in /usr/local/lib/python3.6/dist-packages (0.12.2)\n",
            "Requirement already satisfied: Jinja2>=2.4 in /usr/local/lib/python3.6/dist-packages (from flask==0.12.2) (2.10)\n",
            "Requirement already satisfied: click>=2.0 in /usr/local/lib/python3.6/dist-packages (from flask==0.12.2) (7.0)\n",
            "Requirement already satisfied: itsdangerous>=0.21 in /usr/local/lib/python3.6/dist-packages (from flask==0.12.2) (1.1.0)\n",
            "Requirement already satisfied: Werkzeug>=0.7 in /usr/local/lib/python3.6/dist-packages (from flask==0.12.2) (0.14.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.4->flask==0.12.2) (1.1.1)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "V-LbMKAn0TG6",
        "colab_type": "code",
        "outputId": "b414f7e2-ab40-49ee-8b7a-229cbe6299a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "cell_type": "code",
      "source": [
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "num_classes = 10\n",
        "\n",
        "x_train = x_train.reshape(60000, 784)\n",
        "x_test = x_test.reshape(10000, 784)\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "x_train /= 255\n",
        "x_test /= 255\n",
        "print(x_train.shape[0], 'train samples')\n",
        "print(x_test.shape[0], 'test samples')\n",
        "\n",
        "# convert class vectors to binary class matrices\n",
        "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
        "y_test = keras.utils.to_categorical(y_test, num_classes)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "60000 train samples\n",
            "10000 test samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "jpbYAzUL0THE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "64186963-2a91-4d5b-c05b-d3001e878a84"
      },
      "cell_type": "code",
      "source": [
        "# Setup Sherpa study object to explore learning rate and momentum.\n",
        "parameters = [sherpa.Continuous('lrinit', [0.1, 0.001], 'log'),\n",
        "              sherpa.Continuous('momentum', [0., 0.99])]\n",
        "\n",
        "algorithm = sherpa.algorithms.BayesianOptimization(max_num_trials=10)\n",
        "\n",
        "study = sherpa.Study(parameters=parameters,\n",
        "                     algorithm=algorithm,\n",
        "                     lower_is_better=True)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:sherpa.core:\n",
            "-------------------------------------------------------\n",
            "SHERPA Dashboard running on http://172.28.0.2:8880\n",
            "-------------------------------------------------------\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "3qJGHSN40THM",
        "colab_type": "code",
        "outputId": "627a7553-49c2-4009-df4e-d3b9efef4346",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "cell_type": "code",
      "source": [
        "dbatch_size = 32\n",
        "epochs = 1\n",
        "\n",
        "for trial in study:\n",
        "    print(\"Trial {}:\\t{}\".format(trial.id, trial.parameters))\n",
        "\n",
        "    model = Sequential()\n",
        "    model.add(Dense(units=30, activation='relu', input_dim=784))\n",
        "    model.add(Dense(units=10, activation='softmax'))\n",
        "    \n",
        "    optimizer = keras.optimizers.SGD(lr=trial.parameters['lrinit'], \n",
        "                                     momentum=trial.parameters['momentum'])\n",
        "    model.compile(loss='categorical_crossentropy', \n",
        "                  optimizer=optimizer,\n",
        "                  metrics=['accuracy'])\n",
        "    model.fit(x_train, y_train,\n",
        "              batch_size=batch_size,\n",
        "              epochs=epochs,\n",
        "              verbose=0,\n",
        "              callbacks=[study.keras_callback(trial, objective_name='val_loss')],\n",
        "              validation_data=(x_test, y_test))\n",
        "\n",
        "    study.finalize(trial=trial)\n",
        "    \n",
        "print(study.get_best_result())"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Trial 1:\t{'lrinit': 0.021544346900318846, 'momentum': 0.32999999999999996}\n",
            "Trial 2:\t{'lrinit': 0.021544346900318846, 'momentum': 0.6599999999999999}\n",
            "Trial 3:\t{'lrinit': 0.004641588833612782, 'momentum': 0.32999999999999996}\n",
            "Trial 4:\t{'lrinit': 0.004641588833612782, 'momentum': 0.6599999999999999}\n",
            "Trial 5:\t{'lrinit': 0.1, 'momentum': 0.99}\n",
            "Trial 6:\t{'lrinit': 0.013236310084867153, 'momentum': 0.537857299945472}\n",
            "Trial 7:\t{'lrinit': 0.07655490914789083, 'momentum': 0.0}\n",
            "Trial 8:\t{'lrinit': 0.017150394330641344, 'momentum': 0.0}\n",
            "Trial 9:\t{'lrinit': 0.09605670116060591, 'momentum': 0.2722226589033094}\n",
            "{'Trial-ID': 9, 'Iteration': 0, 'lrinit': 0.09605670116060591, 'momentum': 0.2722226589033094, 'Objective': 0.19574639337956906}\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}