Hyperopt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b448194a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "from ax import optimize\n",
    "from omegaconf import DictConfig\n",
    "\n",
    "from eml.Config import Config\n",
    "from eml.Datasets import load_data\n",
    "from eml.Train import train\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from ax.plot.contour import plot_contour\n",
    "from ax.plot.trace import optimization_trace_single_method\n",
    "from ax.service.managed_loop import optimize\n",
    "from ax.utils.notebook.plotting import render, init_notebook_plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba241ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"hyperopt\", exist_ok=True)\n",
    "os.chdir(\"hyperopt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ffc1a00",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cfg = DictConfig(Config(1, 1))\n",
    "cfg.num_train_labels = 500\n",
    "data_loaders = load_data(cfg)\n",
    "\n",
    "def evaluation_function(params) -> None:\n",
    "    cfg.unsupervised_epochs = params[\"unsupervised_epochs\"]\n",
    "    cfg.classifier_epochs = params[\"classifier_epochs\"]\n",
    "    cfg.autoencoder_features = params[\"autoencoder_features\"]\n",
    "    # cfg.autoencoder_lr = params[\"autoencoder_lr\"]\n",
    "    # cfg.classifier_lr = params[\"classifier_lr\"]\n",
    "    # cfg.classifier_lr_autoenc = params[\"classifier_lr_autoenc\"]\n",
    "    cfg.classifier_lr_autoenc = params[\"lr_ratio\"] * cfg.classifier_lr\n",
    "    cfg.auto_encoder_channels = [[16, 32, 64], [32, 64], [16, 32]][params[\"auto_encoder_channels\"]]\n",
    "    cfg.auto_encoder_depth = params[\"auto_encoder_depth\"]\n",
    "    cfg.auto_encoder_fc_layers = params[\"auto_encoder_fc_layers\"]\n",
    "    cfg.classifier_size = params[\"classifier_size\"]\n",
    "    cfg.variational_sigma = params[\"variational_sigma\"]\n",
    "    cfg.dropout_p = params[\"dropout_p\"]\n",
    "    cfg.num_train_labels = 5000\n",
    "    # cfg.classifier_neurons = [[128, 64, 32], [128, 64], [256, 128, 64], [256, 128, 64, 32]][params[\"classifier_neurons\"]]\n",
    "    result = train(cfg, *data_loaders)\n",
    "    return {\n",
    "        \"accuracy\": result[\"classifier/val_acc\"]\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4d8682a",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_parameters, best_values, experiment, model = optimize(\n",
    "    parameters=[\n",
    "        {\n",
    "            \"name\": \"unsupervised_epochs\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [7, 20],\n",
    "            \"value_type\": \"int\",\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"classifier_epochs\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [10, 40],\n",
    "            \"value_type\": \"int\",\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"autoencoder_features\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [32, 1024],\n",
    "            \"value_type\": \"int\",\n",
    "            \"log_scale\": True,\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"lr_ratio\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [0.01, 1.0],\n",
    "            \"value_type\": \"float\",\n",
    "            \"log_scale\": True,\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"variational_sigma\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [0.01, 0.1],\n",
    "            \"value_type\": \"float\",\n",
    "            \"log_scale\": True,\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"auto_encoder_channels\",\n",
    "            \"type\": \"choice\",\n",
    "            \"values\": [0, 1, 2]\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"auto_encoder_depth\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [0, 4],\n",
    "            \"value_type\": \"int\",\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"auto_encoder_fc_layers\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [0, 4],\n",
    "            \"value_type\": \"int\",\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"classifier_size\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [0, 6],\n",
    "            \"value_type\": \"int\",\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"dropout_p\",\n",
    "            \"type\": \"range\",\n",
    "            \"bounds\": [0.001, 0.5],\n",
    "            \"value_type\": \"float\",\n",
    "            \"log_scale\": True,\n",
    "        },\n",
    "    ],\n",
    "    objective_name=\"accuracy\",\n",
    "    evaluation_function=evaluation_function,\n",
    "    minimize=False,\n",
    "    total_trials=10,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55fbcc34",
   "metadata": {},
   "outputs": [],
   "source": [
    "#render(plot_contour(model=model, param_x='unsupervised_epochs', param_y='classifier_epochs', metric_name=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "266a9e46",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_objectives = np.array([[trial.objective_mean*100 for trial in experiment.trials.values() if trial.index <= 10]])\n",
    "best_objective_plot = optimization_trace_single_method(\n",
    "    y=np.maximum.accumulate(best_objectives, axis=1),\n",
    "    title=\"Model performance vs. # of iterations\",\n",
    "    ylabel=\"Classification Accuracy, %\",\n",
    ")\n",
    "render(best_objective_plot)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62af4fad",
   "metadata": {},
   "outputs": [],
   "source": [
    "list(experiment.trials.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41c3d917",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8656706",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment.fetch_data().df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0b3a9e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "488b1df6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}