ENAS_Keras_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1773,
     "status": "ok",
     "timestamp": 1531705057855,
     "user": {
      "displayName": "cv usk",
      "photoUrl": "//lh5.googleusercontent.com/--UQyF-cisE4/AAAAAAAAAAI/AAAAAAACcCg/nW1mf571W7o/s50-c-k-no/photo.jpg",
      "userId": "103824322408948109555"
     },
     "user_tz": -540
    },
    "id": "L8PLWMnE1e_5",
    "outputId": "a22a2442-0ba3-4cfd-90d9-8d2103478b64"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import shutil\n",
    "import gc\n",
    "from copy import deepcopy\n",
    "\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.utils import to_categorical\n",
    "from keras.optimizers import Adam,SGD\n",
    "from keras.datasets import mnist\n",
    "from keras.callbacks import EarlyStopping, LearningRateScheduler\n",
    "\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ENAS import EfficientNeuralArchitectureSearch\n",
    "from src.utils import sgdr_learning_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "og9AGLL4AaLc"
   },
   "outputs": [],
   "source": [
    "# Load MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "child_classes = 2\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "if child_classes != 10:\n",
    "    train_part = np.where(y_train<child_classes)\n",
    "    test_part = np.where(y_test<child_classes)\n",
    "    x_train = x_train[train_part]\n",
    "    y_train = y_train[train_part]\n",
    "    x_test = x_test[test_part]\n",
    "    y_test = y_test[test_part]\n",
    "\n",
    "x_train = np.reshape(x_train, (len(x_train), 28,28,1))\n",
    "x_test = np.reshape(x_test, (len(x_test), 28,28,1))\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = to_categorical(y_train, child_classes)\n",
    "y_test = to_categorical(y_test, child_classes)\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nt = sgdr_learning_rate(n_Max=0.05, n_min=0.001, ranges=4, init_cycle=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Efficient neural architecture search\n",
    "## Micro search for CNN cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ENAS = EfficientNeuralArchitectureSearch(x_train=x_train,\n",
    "                               y_train=y_train,\n",
    "                               x_test=x_test,\n",
    "                               y_test=y_test,\n",
    "                               child_network_name=\"mnist-cnn\",\n",
    "                               child_classes=child_classes,\n",
    "                               child_input_shape=(28,28,1),\n",
    "                               num_nodes=6,\n",
    "                               num_opers=5,\n",
    "                               controller_lstm_cell_units = 32,\n",
    "                               controller_baseline_decay = 0.99,\n",
    "                               controller_opt = Adam(lr=0.00035, decay=1e-3, amsgrad=True),\n",
    "                               controller_batch_size = 1,\n",
    "                               controller_epochs = 50,\n",
    "                               controller_callbacks = [EarlyStopping(monitor='val_loss', patience=1, verbose=1, mode='auto')],\n",
    "                               controller_temperature = 5.0,\n",
    "                               controller_tanh_constant = 2.5,\n",
    "                               controller_normal_model_file = \"mnist_normal_controller.hdf5\",\n",
    "                               controller_reduction_model_file = \"mnist_reduction_controller.hdf5\",\n",
    "                               child_init_filters=64,\n",
    "                               child_network_definition=[\"N\",\"R\",\"N\",\"R\"],\n",
    "                               child_weight_directory=\"./mnist_weights\",\n",
    "                               child_opt_loss='categorical_crossentropy',\n",
    "                               child_opt=SGD(lr=0.05, decay=1e-6, nesterov=True),\n",
    "                               child_opt_metrics=['accuracy'],\n",
    "                               child_val_batch_size = 128,\n",
    "                               child_batch_size = 128,\n",
    "                               child_epochs = len(nt),\n",
    "                               child_lr_scedule = nt,\n",
    "                               start_from_record=True,\n",
    "                               run_on_jupyter = True,\n",
    "                               initialize_child_weight_directory=False,\n",
    "                               save_to_disk=False,\n",
    "                               set_from_dict=True,\n",
    "                               data_gen=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "H-aUGKEOBnff",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ENAS.search_neural_architecture()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENAS.train_best_cells()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "ENAS_Keras.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}