models_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "13125ff0-8241-4896-bb0b-cfb7c1f7e73b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importa bibliotecas\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random as rd\n",
    "import os\n",
    "\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "from sklearn.metrics import balanced_accuracy_score, make_scorer, roc_auc_score, recall_score\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "from sklearn.impute import KNNImputer\n",
    "\n",
    "import hyperopt\n",
    "from hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK\n",
    "from hyperopt.pyll.base import scope\n",
    "from hyperopt.pyll.stochastic import sample\n",
    "\n",
    "rstate = np.random.default_rng(17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d636bc16-a7ac-40b0-b087-f2d55160cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#algoritmos\n",
    "algorithms = {\n",
    "    \n",
    "    'SVM_linear': (SVC, {'probability': True,\n",
    "                         'C': hp.loguniform('C', np.log(1e-3), np.log(1e3))}),\n",
    "    \n",
    "    'SVM_rbf': (SVC, {'kernel': 'rbf',\n",
    "             'probability': True,\n",
    "             'C': hp.loguniform('C', np.log(1e-3), np.log(1e3)),\n",
    "             'gamma': hp.choice('gamma',['scale', 'auto'])}),\n",
    "    \n",
    "    'RF' : (RandomForestClassifier,{\n",
    "             'n_estimators': hp.uniformint('n_estimators', 2, 200),\n",
    "             'max_depth': hp.uniformint('max_depth', 1, 100),\n",
    "             'criterion': hp.choice('criterion', [\"gini\", \"entropy\"])}),\n",
    "    \n",
    "    \n",
    "    'GB' : (GradientBoostingClassifier, {'learning_rate': hp.lognormal('learning_rate', np.log(0.01), np.log(10.0)),\n",
    "                                     'n_estimators': scope.int(hp.qloguniform('n_estimators', np.log(10.5), np.log(1000.5), 1)),\n",
    "                                     'loss': hp.choice('loss', ['deviance']),\n",
    "                                     }),\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3766c50e-51c9-4e15-ae4b-d4daf2020d05",
   "metadata": {},
   "outputs": [],
   "source": [
    "#choose of the best hyperparameters through balanced accuracy\n",
    "perf = make_scorer(balanced_accuracy_score)\n",
    "\n",
    "#10-fold cross validation \n",
    "kf = StratifiedKFold(n_splits=10, shuffle=True, random_state=17) \n",
    "\n",
    "#define Standard Scaler to standardize the features\n",
    "prep = StandardScaler()\n",
    "\n",
    "#imputa os valores ausentes dos 3 vizinhos mais próximos\n",
    "imputer = KNNImputer(n_neighbors=3, weights = 'distance')\n",
    "\n",
    "# redefine the function usng a wider range of hyperparameters\n",
    "def objective(search_space):\n",
    "    model = clf(**search_space, random_state = 0)\n",
    "    score = cross_val_score(model, x_train, y_train, cv=3, scoring=perf, n_jobs=None).mean()\n",
    "    return {'loss': -score, 'status': STATUS_OK} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d80a5311-4dae-45bc-a04d-1123478106e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#datasets\n",
    "hosp1 = pd.read_csv(\"hosp1_severity_.csv\")\n",
    "hosp2 = pd.read_csv(\"hosp2_severity_.csv\")\n",
    "\n",
    "data = {\n",
    "    \"hosp1\": ((hosp1.drop(columns=['severity'])), hosp1.severity), \n",
    "    \"hosp2\": ((hosp2.drop(columns=['severity'])), hosp2.severity)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a69724f-1db8-4386-a1d3-0d233e6010cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5995939813709099]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5892842926518589]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5862922272642836]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.80trial/s, best loss: -0.5819401321550832]\n",
      "100%|███████| 100/100 [00:58<00:00,  1.71trial/s, best loss: -0.599706764323435]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.5860732956505585]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.56trial/s, best loss: -0.6092004352095108]\n",
      "100%|██████| 100/100 [01:12<00:00,  1.39trial/s, best loss: -0.5745628002016824]\n",
      "100%|████████| 100/100 [01:11<00:00,  1.41trial/s, best loss: -0.59869171775071]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.56trial/s, best loss: -0.5756640925616326]\n",
      "100%|██████| 100/100 [01:12<00:00,  1.37trial/s, best loss: -0.6017434917602101]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.5871082450972588]\n",
      "100%|███████| 100/100 [01:07<00:00,  1.48trial/s, best loss: -0.589317464108484]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.47trial/s, best loss: -0.6038930021495104]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5818207149112332]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.71trial/s, best loss: -0.5735145821723324]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.81trial/s, best loss: -0.5955935037019345]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.80trial/s, best loss: -0.5534392166228803]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.71trial/s, best loss: -0.6049544887615105]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.83trial/s, best loss: -0.5932980389034843]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5670859538784067]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.64trial/s, best loss: -0.5817610062893083]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5712788259958071]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.82trial/s, best loss: -0.5786163522012578]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5786163522012578]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5870020964360587]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.5817610062893082]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5859538784067085]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5922431865828092]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.80trial/s, best loss: -0.5754716981132075]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5755911153570575]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5757105326009075]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.5830016187670833]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.5945320170899344]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.63trial/s, best loss: -0.5799564790489079]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.70trial/s, best loss: -0.5735411193376324]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.63trial/s, best loss: -0.5925350954011093]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.60trial/s, best loss: -0.5840631053790833]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.5913475572539341]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.59trial/s, best loss: -0.5745030915797574]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.68trial/s, best loss: -0.5767852877955577]\n",
      "100%|██████| 100/100 [01:10<00:00,  1.43trial/s, best loss: -0.5840365682137834]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.63trial/s, best loss: -0.5756906297269325]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.5713584374917071]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.80trial/s, best loss: -0.5852174720696336]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5672518111615317]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5933909189820344]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.81trial/s, best loss: -0.5639147626250564]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.5882228060398589]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5797176445612079]\n",
      "100%|███████| 100/100 [00:54<00:00,  1.83trial/s, best loss: -0.589383807021734]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.5735013135896824]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5789215296022079]\n",
      "100%|███████| 100/100 [00:54<00:00,  1.83trial/s, best loss: -0.590259533476634]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5819268635724332]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5764469389379827]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.6103614361913859]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5776610142504578]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.79trial/s, best loss: -0.5987779635379349]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.5838906138046333]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5840498367964334]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.79trial/s, best loss: -0.5798171589310829]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.80trial/s, best loss: -0.5766857734256826]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.75trial/s, best loss: -0.5798635989703579]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5860467584852586]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.71trial/s, best loss: -0.5766459676777326]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5735543879202823]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.5882626117878088]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5693482472202319]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5745694344930073]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5986850834593849]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5883156861184088]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.72trial/s, best loss: -0.5819865721943581]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.5819003264071332]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5923692381179843]\n",
      "100%|████████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.60086776530531]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.81trial/s, best loss: -0.5965952816920096]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.83trial/s, best loss: -0.5724132898123823]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.5829684473104583]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.5786362550752328]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.5882228060398589]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.5734814107157074]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.5946315314598095]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.56trial/s, best loss: -0.5860998328158585]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.59trial/s, best loss: -0.5755712124830826]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.61trial/s, best loss: -0.5745495316190324]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.55trial/s, best loss: -0.6029575670726852]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.51trial/s, best loss: -0.5745893373669824]\n",
      "100%|███████| 100/100 [01:19<00:00,  1.25trial/s, best loss: -0.579850330387708]\n",
      "100%|██████| 100/100 [01:19<00:00,  1.26trial/s, best loss: -0.5725327070562323]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.46trial/s, best loss: -0.5986718148767348]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.56trial/s, best loss: -0.5830812302629833]\n",
      "100%|██████| 100/100 [01:12<00:00,  1.39trial/s, best loss: -0.5977297455085847]\n",
      "100%|██████| 100/100 [01:20<00:00,  1.25trial/s, best loss: -0.6008345938486851]\n",
      "100%|███████| 100/100 [01:12<00:00,  1.38trial/s, best loss: -0.589118435368734]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.60trial/s, best loss: -0.5998062786933099]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.50trial/s, best loss: -0.5966417217312846]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.54trial/s, best loss: -0.5924090438659342]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.62trial/s, best loss: -0.5924090438659343]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.5819666693203832]\n",
      "100%|███████| 100/100 [00:59<00:00,  1.67trial/s, best loss: -0.599600615662235]\n",
      "100%|███████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.590332510681209]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.5852042034869834]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.5819401321550832]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5986585462940849]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.81trial/s, best loss: -0.5860666613592336]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.72trial/s, best loss: -0.6092004352095108]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.5735145821723323]\n",
      "100%|████████| 100/100 [00:54<00:00,  1.82trial/s, best loss: -0.59869171775071]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.5756707268529576]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.61trial/s, best loss: -0.6017434917602101]\n",
      "100%|██████| 100/100 [01:09<00:00,  1.45trial/s, best loss: -0.5871016108059337]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.61trial/s, best loss: -0.5882360746225088]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.68trial/s, best loss: -0.6039394421887854]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.62trial/s, best loss: -0.5828622986492583]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.5735145821723324]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.5955935037019345]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.59trial/s, best loss: -0.5534392166228803]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.64trial/s, best loss: -0.6049478544701855]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.78trial/s, best loss: -0.5932980389034843]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.67trial/s, best loss: -0.5660377358490566]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5817610062893083]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.72trial/s, best loss: -0.5723270440251572]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5796645702306079]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.71trial/s, best loss: -0.5786163522012578]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.76trial/s, best loss: -0.5880503144654088]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.67trial/s, best loss: -0.5817610062893082]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.60trial/s, best loss: -0.5849056603773585]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.48trial/s, best loss: -0.5911949685534591]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.57trial/s, best loss: -0.5733752620545073]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.54trial/s, best loss: -0.5755911153570575]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.55trial/s, best loss: -0.5778069686596078]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.51trial/s, best loss: -0.5830016187670833]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.53trial/s, best loss: -0.5945320170899344]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.59trial/s, best loss: -0.5778932144468327]\n",
      "100%|██████| 100/100 [01:02<00:00,  1.59trial/s, best loss: -0.5745893373669825]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.51trial/s, best loss: -0.5925417296924341]\n",
      "100%|██████| 100/100 [01:11<00:00,  1.39trial/s, best loss: -0.5840631053790833]\n",
      "100%|██████| 100/100 [01:16<00:00,  1.30trial/s, best loss: -0.5913475572539341]\n",
      "100%|██████| 100/100 [01:17<00:00,  1.30trial/s, best loss: -0.5745030915797574]\n",
      "100%|██████| 100/100 [01:20<00:00,  1.24trial/s, best loss: -0.5767852877955577]\n",
      "100%|██████| 100/100 [01:24<00:00,  1.19trial/s, best loss: -0.5840365682137834]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.46trial/s, best loss: -0.5756839954356076]\n",
      "100%|██████| 100/100 [01:14<00:00,  1.34trial/s, best loss: -0.5713584374917071]\n",
      "100%|██████| 100/100 [01:21<00:00,  1.22trial/s, best loss: -0.5862656900989837]\n",
      "100%|██████| 100/100 [01:17<00:00,  1.29trial/s, best loss: -0.5672518111615317]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.63trial/s, best loss: -0.5933909189820344]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.5639147626250564]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5882228060398589]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.85trial/s, best loss: -0.5797176445612079]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5872807366717088]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.5735013135896824]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.5789215296022079]\n",
      "100%|███████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.590259533476634]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.93trial/s, best loss: -0.5819268635724332]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.92trial/s, best loss: -0.5744765544144574]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.6103614361913859]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.90trial/s, best loss: -0.5766127962211077]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.91trial/s, best loss: -0.5987779635379349]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5849321975426585]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.91trial/s, best loss: -0.5840498367964334]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.5798171589310829]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.90trial/s, best loss: -0.5766924077170076]\n",
      "100%|███████| 100/100 [00:53<00:00,  1.85trial/s, best loss: -0.579870233261683]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.93trial/s, best loss: -0.5839503224265584]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.5755911153570575]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5735543879202823]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5882626117878088]\n",
      "100%|███████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.570363293792957]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.83trial/s, best loss: -0.5745694344930073]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.89trial/s, best loss: -0.5944789427593343]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5883156861184088]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.81trial/s, best loss: -0.5819799379030332]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5819069606984582]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5923692381179843]\n",
      "100%|████████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.59981954727596]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.90trial/s, best loss: -0.5965952816920096]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.90trial/s, best loss: -0.5734681421330573]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.85trial/s, best loss: -0.5829684473104583]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.90trial/s, best loss: -0.5786362550752328]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5882228060398589]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5734814107157074]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.90trial/s, best loss: -0.5956797494891596]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5860998328158585]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.91trial/s, best loss: -0.5755712124830826]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5745495316190324]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.6019093490433352]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.84trial/s, best loss: -0.5745893373669824]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5809250855823581]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.84trial/s, best loss: -0.5746291431149324]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.5986718148767348]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.83trial/s, best loss: -0.5841294482923334]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5977297455085847]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.6008345938486851]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.5891184353687339]\n",
      "100%|███████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.599799644401985]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5966417217312846]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.5934572618952842]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.5924090438659343]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.92trial/s, best loss: -0.5830082530584083]\n",
      "100%|██████| 100/100 [01:27<00:00,  1.14trial/s, best loss: -0.6891237428017939]\n",
      " 69%|██████▏  | 69/100 [01:30<00:40,  1.31s/trial, best loss: -0.69959928880397]\n",
      " 74%|██████▋  | 74/100 [01:30<00:31,  1.22s/trial, best loss: -0.70072711832922]\n",
      " 66%|████▌  | 66/100 [01:30<00:46,  1.37s/trial, best loss: -0.7216317702942971]\n",
      "100%|██████| 100/100 [01:17<00:00,  1.30trial/s, best loss: -0.7142345354669214]\n",
      " 80%|█████▌ | 80/100 [01:31<00:22,  1.14s/trial, best loss: -0.6774805615264178]\n",
      " 68%|████▊  | 68/100 [01:30<00:42,  1.33s/trial, best loss: -0.6921821511026192]\n",
      " 80%|█████▌ | 80/100 [01:31<00:22,  1.14s/trial, best loss: -0.7017023591539951]\n",
      " 79%|█████▌ | 79/100 [01:30<00:23,  1.14s/trial, best loss: -0.7017421649019452]\n",
      " 76%|█████▎ | 76/100 [01:30<00:28,  1.19s/trial, best loss: -0.7006077010853701]\n",
      " 96%|██████▋| 96/100 [01:30<00:03,  1.06trial/s, best loss: -0.7122906881086962]\n",
      " 81%|█████▋ | 81/100 [01:31<00:21,  1.12s/trial, best loss: -0.6984979964440199]\n",
      " 94%|██████▌| 94/100 [01:30<00:05,  1.04trial/s, best loss: -0.6838295783244434]\n",
      " 87%|██████ | 87/100 [01:30<00:13,  1.04s/trial, best loss: -0.6953931481039195]\n",
      " 79%|███████  | 79/100 [01:30<00:24,  1.15s/trial, best loss: -0.70059443250272]\n",
      " 91%|███████▎| 91/100 [01:30<00:08,  1.01trial/s, best loss: -0.711176127166096]\n",
      " 70%|████▉  | 70/100 [01:30<00:38,  1.30s/trial, best loss: -0.6922352254332192]\n",
      " 91%|██████▎| 91/100 [01:31<00:09,  1.01s/trial, best loss: -0.6964280975506196]\n",
      "100%|██████| 100/100 [01:10<00:00,  1.42trial/s, best loss: -0.6984979964440198]\n",
      " 78%|█████▍ | 78/100 [01:31<00:25,  1.17s/trial, best loss: -0.7036860122601704]\n",
      "100%|██████| 100/100 [01:25<00:00,  1.17trial/s, best loss: -0.7044025157232704]\n",
      " 88%|██████▏| 88/100 [01:30<00:12,  1.03s/trial, best loss: -0.7075471698113206]\n",
      " 76%|██████  | 76/100 [01:30<00:28,  1.19s/trial, best loss: -0.709643605870021]\n",
      " 69%|████▊  | 69/100 [01:30<00:40,  1.31s/trial, best loss: -0.7054507337526205]\n",
      " 72%|█████  | 72/100 [01:30<00:35,  1.26s/trial, best loss: -0.7033542976939202]\n",
      " 97%|██████▊| 97/100 [01:30<00:02,  1.07trial/s, best loss: -0.6907756813417191]\n",
      "100%|████████| 100/100 [01:27<00:00,  1.14trial/s, best loss: -0.69916142557652]\n",
      " 66%|████▌  | 66/100 [01:30<00:46,  1.37s/trial, best loss: -0.6939203354297693]\n",
      " 82%|█████▋ | 82/100 [01:30<00:19,  1.10s/trial, best loss: -0.6844863731656184]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.46trial/s, best loss: -0.7085953878406709]\n",
      " 73%|█████  | 73/100 [01:31<00:33,  1.25s/trial, best loss: -0.6870007695777938]\n",
      " 83%|█████▊ | 83/100 [01:30<00:18,  1.09s/trial, best loss: -0.6974298755406947]\n",
      " 70%|████▉  | 70/100 [01:30<00:38,  1.30s/trial, best loss: -0.6838892869463683]\n",
      "100%|██████| 100/100 [01:09<00:00,  1.44trial/s, best loss: -0.6975028527452697]\n",
      " 73%|█████  | 73/100 [01:30<00:33,  1.23s/trial, best loss: -0.7090531539420959]\n",
      " 83%|██████▋ | 83/100 [01:30<00:18,  1.09s/trial, best loss: -0.691140567364594]\n",
      " 90%|██████▎| 90/100 [01:31<00:10,  1.01s/trial, best loss: -0.7058620598147707]\n",
      " 69%|█████▌  | 69/100 [01:31<00:41,  1.33s/trial, best loss: -0.691266618899769]\n",
      " 65%|████▌  | 65/100 [01:30<00:48,  1.39s/trial, best loss: -0.7079783987474458]\n",
      " 66%|████▌  | 66/100 [01:31<00:47,  1.39s/trial, best loss: -0.6828078974603932]\n",
      " 66%|████▌  | 66/100 [01:30<00:46,  1.37s/trial, best loss: -0.7007536554945201]\n",
      " 86%|██████ | 86/100 [01:30<00:14,  1.06s/trial, best loss: -0.6953599766472945]\n",
      " 92%|██████▍| 92/100 [01:30<00:07,  1.02trial/s, best loss: -0.6806318499057932]\n",
      " 85%|█████▉ | 85/100 [01:30<00:15,  1.06s/trial, best loss: -0.6648356022609665]\n",
      " 73%|█████  | 73/100 [01:30<00:33,  1.24s/trial, best loss: -0.7143274155454714]\n",
      " 71%|████▉  | 71/100 [01:31<00:37,  1.28s/trial, best loss: -0.6849375049757184]\n",
      " 75%|█████▎ | 75/100 [01:31<00:30,  1.21s/trial, best loss: -0.7152495820396464]\n",
      " 69%|████▊  | 69/100 [01:31<00:41,  1.33s/trial, best loss: -0.6838362126157683]\n",
      "100%|██████| 100/100 [01:23<00:00,  1.20trial/s, best loss: -0.6964148289679697]\n",
      "100%|██████| 100/100 [01:25<00:00,  1.17trial/s, best loss: -0.7143008783801715]\n",
      "100%|██████| 100/100 [01:26<00:00,  1.16trial/s, best loss: -0.7016492848233952]\n",
      " 68%|████▊  | 68/100 [01:31<00:42,  1.34s/trial, best loss: -0.6943449300745694]\n",
      " 81%|█████▋ | 81/100 [01:30<00:21,  1.12s/trial, best loss: -0.6881684048509937]\n",
      "100%|██████| 100/100 [01:21<00:00,  1.23trial/s, best loss: -0.7017156277366451]\n",
      "100%|██████| 100/100 [01:20<00:00,  1.24trial/s, best loss: -0.7101146405540959]\n",
      " 66%|████▌  | 66/100 [01:30<00:46,  1.37s/trial, best loss: -0.7059084998540456]\n",
      " 75%|█████▎ | 75/100 [01:30<00:30,  1.21s/trial, best loss: -0.7163508743995965]\n",
      " 80%|█████▌ | 80/100 [01:30<00:22,  1.13s/trial, best loss: -0.7058885969800706]\n",
      " 72%|█████  | 72/100 [01:30<00:35,  1.25s/trial, best loss: -0.6932701748799194]\n",
      " 77%|█████▍ | 77/100 [01:30<00:26,  1.17s/trial, best loss: -0.6985643393572699]\n",
      " 69%|████▊  | 69/100 [01:30<00:40,  1.31s/trial, best loss: -0.6964944404638697]\n",
      " 80%|█████▌ | 80/100 [01:30<00:22,  1.13s/trial, best loss: -0.7143075126714965]\n",
      " 69%|████▊  | 69/100 [01:30<00:40,  1.31s/trial, best loss: -0.6775933444789427]\n",
      "100%|██████| 100/100 [01:23<00:00,  1.20trial/s, best loss: -0.7028169200965952]\n",
      " 76%|█████▎ | 76/100 [01:31<00:28,  1.20s/trial, best loss: -0.6807379985669929]\n",
      " 85%|██████▊ | 85/100 [01:31<00:16,  1.07s/trial, best loss: -0.679663243372343]\n",
      " 85%|█████▉ | 85/100 [01:30<00:15,  1.06s/trial, best loss: -0.6953666109386195]\n",
      "100%|██████| 100/100 [01:10<00:00,  1.41trial/s, best loss: -0.6964679032985698]\n",
      " 77%|█████▍ | 77/100 [01:30<00:27,  1.18s/trial, best loss: -0.6838362126157683]\n",
      " 84%|███████▌ | 84/100 [01:30<00:17,  1.08s/trial, best loss: -0.70062096966802]\n",
      " 82%|█████▋ | 82/100 [01:30<00:19,  1.10s/trial, best loss: -0.7059151341453705]\n",
      " 88%|██████▏| 88/100 [01:31<00:12,  1.04s/trial, best loss: -0.7028036515139453]\n",
      " 75%|█████▎ | 75/100 [01:30<00:30,  1.20s/trial, best loss: -0.7079850330387707]\n",
      "100%|██████| 100/100 [01:25<00:00,  1.17trial/s, best loss: -0.6817132393917681]\n",
      " 88%|██████▏| 88/100 [01:30<00:12,  1.03s/trial, best loss: -0.6964546347159196]\n",
      " 79%|█████▌ | 79/100 [01:30<00:24,  1.14s/trial, best loss: -0.6943382957832444]\n",
      " 86%|██████ | 86/100 [01:30<00:14,  1.05s/trial, best loss: -0.7142677069235465]\n",
      " 80%|█████▌ | 80/100 [01:30<00:22,  1.14s/trial, best loss: -0.7048072074940954]\n",
      " 67%|████▋  | 67/100 [01:31<00:45,  1.37s/trial, best loss: -0.7090995939813709]\n",
      " 74%|█████▏ | 74/100 [01:30<00:31,  1.23s/trial, best loss: -0.7069169121354456]\n",
      "100%|██████| 100/100 [01:24<00:00,  1.18trial/s, best loss: -0.6827614574211182]\n",
      "100%|███████| 100/100 [01:18<00:00,  1.27trial/s, best loss: -0.690158692248494]\n",
      " 72%|█████  | 72/100 [01:30<00:35,  1.26s/trial, best loss: -0.6953533423559696]\n",
      " 71%|████▉  | 71/100 [01:30<00:36,  1.27s/trial, best loss: -0.6922551283071942]\n",
      " 81%|█████▋ | 81/100 [01:30<00:21,  1.11s/trial, best loss: -0.6870472096170687]\n",
      " 90%|██████▎| 90/100 [01:31<00:10,  1.01s/trial, best loss: -0.6648886765915665]\n",
      " 87%|██████ | 87/100 [01:30<00:13,  1.04s/trial, best loss: -0.6922551283071942]\n",
      "100%|██████| 100/100 [01:09<00:00,  1.45trial/s, best loss: -0.6828078974603932]\n",
      " 90%|██████▎| 90/100 [01:30<00:10,  1.00s/trial, best loss: -0.7092123769338959]\n",
      " 76%|█████▎ | 76/100 [01:31<00:28,  1.20s/trial, best loss: -0.7069567178833956]\n",
      " 88%|██████▏| 88/100 [01:30<00:12,  1.02s/trial, best loss: -0.6985046307353447]\n",
      " 93%|██████▌| 93/100 [01:30<00:06,  1.03trial/s, best loss: -0.6954130509778945]\n",
      " 69%|████▊  | 69/100 [01:31<00:40,  1.32s/trial, best loss: -0.7080712788259959]\n",
      "100%|██████| 100/100 [01:27<00:00,  1.14trial/s, best loss: -0.6943117586179444]\n",
      " 91%|████████▏| 91/100 [01:30<00:08,  1.01trial/s, best loss: -0.69953294589072]\n",
      " 87%|██████ | 87/100 [01:31<00:13,  1.05s/trial, best loss: -0.6838561154897432]\n",
      " 63%|████▍  | 63/100 [01:31<00:54,  1.46s/trial, best loss: -0.6943382957832446]\n",
      " 87%|██████ | 87/100 [01:30<00:13,  1.04s/trial, best loss: -0.6953865138125946]\n",
      " 98%|██████▊| 98/100 [01:31<00:01,  1.07trial/s, best loss: -0.6975625613671946]\n",
      " 68%|████▊  | 68/100 [01:30<00:42,  1.32s/trial, best loss: -0.7017089934453201]\n",
      " 25%|█▊     | 25/100 [01:30<04:32,  3.64s/trial, best loss: -0.6764721492450176]\n",
      " 39%|██▋    | 39/100 [01:33<02:25,  2.39s/trial, best loss: -0.6859790887137436]\n",
      " 27%|█▉     | 27/100 [01:34<04:15,  3.50s/trial, best loss: -0.6912732531910941]\n",
      " 30%|██     | 30/100 [01:31<03:32,  3.03s/trial, best loss: -0.6869476952471937]\n",
      " 45%|███▏   | 45/100 [01:33<01:53,  2.07s/trial, best loss: -0.6879561075285938]\n",
      " 36%|██▌    | 36/100 [01:33<02:45,  2.59s/trial, best loss: -0.6690815487089669]\n",
      " 42%|██▉    | 42/100 [01:32<02:07,  2.20s/trial, best loss: -0.6628121434068412]\n",
      " 36%|██▉     | 36/100 [01:30<02:40,  2.50s/trial, best loss: -0.689103839927819]\n",
      " 32%|██▏    | 32/100 [01:31<03:14,  2.86s/trial, best loss: -0.6860188944616935]\n",
      " 34%|██▋     | 34/100 [01:34<03:02,  2.77s/trial, best loss: -0.680618581323143]\n",
      " 29%|██     | 29/100 [01:31<03:43,  3.15s/trial, best loss: -0.6933763235411193]\n",
      " 27%|█▉     | 27/100 [01:36<04:21,  3.59s/trial, best loss: -0.6922484940158693]\n",
      " 33%|██▎    | 33/100 [01:33<03:10,  2.85s/trial, best loss: -0.6575644189687657]\n",
      " 37%|██▌    | 37/100 [01:32<02:38,  2.51s/trial, best loss: -0.6880158161505188]\n",
      " 39%|██▋    | 39/100 [01:33<02:26,  2.41s/trial, best loss: -0.6785685853037178]\n",
      " 42%|██▉    | 42/100 [01:34<02:10,  2.25s/trial, best loss: -0.6890773027625189]\n",
      " 31%|██▍     | 31/100 [01:32<03:24,  2.97s/trial, best loss: -0.680678289945068]\n",
      " 33%|██▎    | 33/100 [01:34<03:11,  2.85s/trial, best loss: -0.6974829498712948]\n",
      " 27%|█▉     | 27/100 [01:31<04:07,  3.40s/trial, best loss: -0.6869344266645436]\n",
      " 63%|████▍  | 63/100 [01:30<00:53,  1.44s/trial, best loss: -0.6953069023166946]\n",
      " 30%|██     | 30/100 [01:33<03:37,  3.11s/trial, best loss: -0.6865828092243186]\n",
      " 35%|██▍    | 35/100 [01:34<02:55,  2.70s/trial, best loss: -0.6949685534591193]\n",
      " 30%|██     | 30/100 [01:33<03:38,  3.12s/trial, best loss: -0.6886792452830188]\n",
      " 35%|██▍    | 35/100 [01:37<03:00,  2.77s/trial, best loss: -0.6897274633123689]\n",
      " 28%|█▉     | 28/100 [01:32<03:57,  3.29s/trial, best loss: -0.6981132075471698]\n",
      " 36%|██▌    | 36/100 [01:30<02:41,  2.53s/trial, best loss: -0.6792452830188679]\n",
      " 47%|███▎   | 47/100 [01:33<01:45,  2.00s/trial, best loss: -0.6907756813417191]\n",
      " 63%|████▍  | 63/100 [01:33<00:55,  1.49s/trial, best loss: -0.6771488469601676]\n",
      " 44%|███    | 44/100 [01:32<01:57,  2.09s/trial, best loss: -0.6876310272536688]\n",
      " 40%|██▊    | 40/100 [01:37<02:25,  2.43s/trial, best loss: -0.6928721174004192]\n",
      " 39%|██▋    | 39/100 [01:30<02:21,  2.32s/trial, best loss: -0.6670116498155666]\n",
      " 67%|████▋  | 67/100 [01:30<00:44,  1.35s/trial, best loss: -0.6816800679351432]\n",
      " 82%|█████▋ | 82/100 [01:33<00:20,  1.14s/trial, best loss: -0.6744221532255924]\n",
      " 49%|███▉    | 49/100 [01:30<01:34,  1.85s/trial, best loss: -0.680711461401693]\n",
      " 32%|██▏    | 32/100 [01:30<03:12,  2.84s/trial, best loss: -0.6797295862855929]\n",
      " 48%|███▎   | 48/100 [01:32<01:39,  1.92s/trial, best loss: -0.6742828331077675]\n",
      " 47%|███▎   | 47/100 [01:31<01:42,  1.94s/trial, best loss: -0.6870538439083936]\n",
      " 35%|██▍    | 35/100 [01:31<02:50,  2.63s/trial, best loss: -0.6839025555290185]\n",
      " 28%|█▉     | 28/100 [01:32<03:56,  3.29s/trial, best loss: -0.6932303691319692]\n",
      " 37%|██▌    | 37/100 [01:32<02:37,  2.50s/trial, best loss: -0.6795769975851179]\n",
      " 39%|██▋    | 39/100 [01:31<02:23,  2.35s/trial, best loss: -0.6964546347159196]\n",
      " 41%|███▎    | 41/100 [01:34<02:16,  2.31s/trial, best loss: -0.691173738821219]\n",
      " 30%|██     | 30/100 [01:35<03:41,  3.17s/trial, best loss: -0.6732279807870922]\n",
      " 29%|██     | 29/100 [01:30<03:41,  3.11s/trial, best loss: -0.6532720324814904]\n",
      " 34%|██▍    | 34/100 [01:31<02:58,  2.70s/trial, best loss: -0.6848512591884934]\n",
      " 36%|██▌    | 36/100 [01:33<02:46,  2.60s/trial, best loss: -0.6880158161505188]\n",
      " 33%|██▎    | 33/100 [01:31<03:05,  2.76s/trial, best loss: -0.6879959132765437]\n",
      " 38%|██▋    | 38/100 [01:31<02:28,  2.40s/trial, best loss: -0.6628718520287663]\n",
      " 39%|██▋    | 39/100 [01:30<02:21,  2.33s/trial, best loss: -0.6732611522437173]\n",
      " 43%|███    | 43/100 [01:37<02:09,  2.27s/trial, best loss: -0.6848711620624685]\n",
      " 86%|██████ | 86/100 [01:30<00:14,  1.05s/trial, best loss: -0.6827481888384682]\n",
      " 48%|███▎   | 48/100 [01:30<01:38,  1.89s/trial, best loss: -0.6817862165963432]\n",
      " 30%|██     | 30/100 [01:34<03:40,  3.15s/trial, best loss: -0.6838627497810683]\n",
      " 32%|██▏    | 32/100 [01:33<03:19,  2.93s/trial, best loss: -0.6932900777538943]\n",
      " 50%|███▌   | 50/100 [01:31<01:31,  1.82s/trial, best loss: -0.6912069102778441]\n",
      " 31%|██▏    | 31/100 [01:35<03:31,  3.07s/trial, best loss: -0.6785818538863678]\n",
      " 39%|██▋    | 39/100 [01:32<02:24,  2.36s/trial, best loss: -0.6942387814133694]\n",
      " 33%|██▎    | 33/100 [01:30<03:04,  2.75s/trial, best loss: -0.6975625613671946]\n",
      " 30%|██     | 30/100 [01:33<03:37,  3.10s/trial, best loss: -0.6848379906058435]\n",
      " 35%|██▍    | 35/100 [01:32<02:52,  2.65s/trial, best loss: -0.6870604781997187]\n",
      " 35%|██▍    | 35/100 [01:31<02:50,  2.62s/trial, best loss: -0.6754703712549425]\n",
      " 27%|██▏     | 27/100 [01:31<04:07,  3.38s/trial, best loss: -0.711196030040071]\n",
      " 45%|███▌    | 45/100 [01:37<01:59,  2.17s/trial, best loss: -0.669108085874267]\n",
      " 42%|███▎    | 42/100 [01:30<02:04,  2.15s/trial, best loss: -0.689130377093119]\n",
      " 42%|██▉    | 42/100 [01:34<02:10,  2.26s/trial, best loss: -0.6670978956027916]\n",
      " 45%|███▏   | 45/100 [01:31<01:51,  2.03s/trial, best loss: -0.6691810630788418]\n",
      " 53%|███▋   | 53/100 [01:38<01:27,  1.85s/trial, best loss: -0.6953997823952444]\n",
      " 32%|██▌     | 32/100 [01:32<03:17,  2.90s/trial, best loss: -0.680791072897593]\n",
      " 38%|██▋    | 38/100 [01:30<02:27,  2.38s/trial, best loss: -0.6659567974948916]\n",
      " 31%|██▏    | 31/100 [01:35<03:32,  3.08s/trial, best loss: -0.6848645277711434]\n",
      " 46%|███▏   | 46/100 [01:31<01:46,  1.98s/trial, best loss: -0.6900923493352439]\n",
      " 47%|███▎   | 47/100 [01:35<01:47,  2.03s/trial, best loss: -0.6880489876071438]\n",
      " 36%|██▌    | 36/100 [01:33<02:46,  2.61s/trial, best loss: -0.6954329538518694]\n",
      " 75%|█████▎ | 75/100 [01:30<00:30,  1.21s/trial, best loss: -0.6670116498155667]\n",
      " 50%|███▌   | 50/100 [01:35<01:35,  1.92s/trial, best loss: -0.6817596794310431]\n",
      " 32%|██▏    | 32/100 [01:31<03:14,  2.86s/trial, best loss: -0.6932635405885943]\n",
      " 45%|███▏   | 45/100 [01:30<01:50,  2.01s/trial, best loss: -0.6985112650266698]\n",
      " 49%|███▍   | 49/100 [01:31<01:35,  1.86s/trial, best loss: -0.6921622482286441]\n",
      " 61%|████▎  | 61/100 [01:30<00:57,  1.48s/trial, best loss: -0.7028102858052702]\n",
      " 36%|██▌    | 36/100 [01:31<02:42,  2.53s/trial, best loss: -0.6848645277711434]\n",
      " 40%|██▊    | 40/100 [01:30<02:15,  2.26s/trial, best loss: -0.6817198736830932]\n",
      " 38%|██▋    | 38/100 [01:36<02:36,  2.53s/trial, best loss: -0.6722859114189422]\n",
      " 58%|████   | 58/100 [01:32<01:07,  1.60s/trial, best loss: -0.6858994772178436]\n",
      " 44%|███    | 44/100 [01:34<02:00,  2.15s/trial, best loss: -0.6901918637051191]\n",
      " 57%|███▉   | 57/100 [01:32<01:09,  1.62s/trial, best loss: -0.6839025555290185]\n",
      " 34%|██▋     | 34/100 [01:37<03:10,  2.88s/trial, best loss: -0.660675901600191]\n",
      " 43%|███▍    | 43/100 [01:31<02:01,  2.14s/trial, best loss: -0.690158692248494]\n",
      " 30%|██     | 30/100 [01:33<03:37,  3.10s/trial, best loss: -0.6870737467823688]\n",
      " 34%|██▋     | 34/100 [01:31<02:56,  2.68s/trial, best loss: -0.690304646657644]\n",
      " 27%|█▉     | 27/100 [01:34<04:15,  3.49s/trial, best loss: -0.6818326566356182]\n",
      " 50%|████    | 50/100 [01:31<01:31,  1.82s/trial, best loss: -0.690132155083194]\n",
      " 35%|██▍    | 35/100 [01:37<03:00,  2.78s/trial, best loss: -0.6837897725764934]\n",
      " 30%|██     | 30/100 [01:30<03:32,  3.03s/trial, best loss: -0.6891569142584189]\n",
      " 40%|██▊    | 40/100 [01:35<02:23,  2.39s/trial, best loss: -0.6733473980309425]\n",
      " 31%|██▏    | 31/100 [01:37<03:37,  3.15s/trial, best loss: -0.6858994772178435]\n",
      " 51%|███▌   | 51/100 [01:30<01:27,  1.78s/trial, best loss: -0.6869941352864687]\n",
      " 32%|██▏    | 32/100 [01:30<03:12,  2.83s/trial, best loss: -0.6965209776291695]\n",
      " 47%|███▎   | 47/100 [01:30<01:41,  1.92s/trial, best loss: -0.6775800758962927]\n",
      " 58%|████   | 58/100 [01:31<01:06,  1.57s/trial, best loss: -0.6733872037788923]\n",
      " 35%|██▍    | 35/100 [01:31<02:49,  2.60s/trial, best loss: -0.6890972056364939]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.25trial/s, best loss: -0.5666666666666668]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 38.76trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.63trial/s, best loss: -0.5583333333333335]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.83trial/s, best loss: -0.5750000000000001]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 42.92trial/s, best loss: -0.525]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.94trial/s, best loss: -0.5083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.68trial/s, best loss: -0.5416666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.56trial/s, best loss: -0.5750000000000001]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 42.61trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.21trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.56trial/s, best loss: -0.5499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.96trial/s, best loss: -0.5166666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.25trial/s, best loss: -0.5666666666666668]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.87trial/s, best loss: -0.5499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.83trial/s, best loss: -0.5583333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.61trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.41trial/s, best loss: -0.5333333333333333]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 46.56trial/s, best loss: -0.525]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.73trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.26trial/s, best loss: -0.5583333333333332]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.20trial/s, best loss: -0.5666666666666668]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.24trial/s, best loss: -0.5916666666666667]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 45.93trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.93trial/s, best loss: -0.6249999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.54trial/s, best loss: -0.5583333333333335]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.44trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.98trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 44.15trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.10trial/s, best loss: -0.6083333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.06trial/s, best loss: -0.6250000000000001]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.21trial/s, best loss: -0.5666666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.96trial/s, best loss: -0.6083333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.47trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.12trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.92trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.59trial/s, best loss: -0.5416666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.25trial/s, best loss: -0.6166666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.03trial/s, best loss: -0.6083333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.35trial/s, best loss: -0.6499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.71trial/s, best loss: -0.6166666666666667]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 42.49trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.91trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.54trial/s, best loss: -0.5499999999999999]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 40.45trial/s, best loss: -0.625]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.27trial/s, best loss: -0.5583333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.49trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.04trial/s, best loss: -0.6416666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.74trial/s, best loss: -0.5499999999999999]\n",
      "100%|████████████████████| 100/100 [00:02<00:00, 40.37trial/s, best loss: -0.65]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.39trial/s, best loss: -0.6166666666666667]\n",
      "100%|███████| 100/100 [00:02<00:00, 37.69trial/s, best loss: -0.557142857142857]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.12trial/s, best loss: -0.5083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.86trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.80trial/s, best loss: -0.5436507936507936]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.98trial/s, best loss: -0.5904761904761905]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.94trial/s, best loss: -0.6138888888888888]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.14trial/s, best loss: -0.5603174603174603]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.60trial/s, best loss: -0.5396825396825397]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.84trial/s, best loss: -0.5444444444444444]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.65trial/s, best loss: -0.6301587301587303]\n",
      "100%|███████| 100/100 [00:02<00:00, 40.07trial/s, best loss: -0.638095238095238]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.45trial/s, best loss: -0.6230158730158731]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.36trial/s, best loss: -0.5992063492063492]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.15trial/s, best loss: -0.5992063492063492]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.21trial/s, best loss: -0.6126984126984127]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.34trial/s, best loss: -0.5599206349206349]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.50trial/s, best loss: -0.6067460317460317]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.62trial/s, best loss: -0.5571428571428572]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.23trial/s, best loss: -0.6404761904761905]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.09trial/s, best loss: -0.5753968253968255]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.88trial/s, best loss: -0.5928571428571429]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.36trial/s, best loss: -0.5742063492063492]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.70trial/s, best loss: -0.6162698412698413]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.79trial/s, best loss: -0.5841269841269842]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.42trial/s, best loss: -0.5904761904761905]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.31trial/s, best loss: -0.6396825396825397]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.83trial/s, best loss: -0.5083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.46trial/s, best loss: -0.5603174603174602]\n",
      "100%|███████| 100/100 [00:02<00:00, 39.57trial/s, best loss: -0.623015873015873]\n",
      "100%|███████| 100/100 [00:02<00:00, 40.89trial/s, best loss: -0.573015873015873]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.34trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.19trial/s, best loss: -0.6333333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.37trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.79trial/s, best loss: -0.5833333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.34trial/s, best loss: -0.5916666666666667]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 46.20trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.84trial/s, best loss: -0.5666666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.33trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 46.98trial/s, best loss: -0.5666666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.99trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.15trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.75trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 39.93trial/s, best loss: -0.6]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 39.19trial/s, best loss: -0.625]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.38trial/s, best loss: -0.5750000000000001]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 41.41trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.84trial/s, best loss: -0.5416666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.37trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.37trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 39.80trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.67trial/s, best loss: -0.5750000000000001]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.49trial/s, best loss: -0.5583333333333335]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.59trial/s, best loss: -0.5583333333333335]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.04trial/s, best loss: -0.5750000000000001]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.42trial/s, best loss: -0.5333333333333333]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 47.79trial/s, best loss: -0.5]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.70trial/s, best loss: -0.5416666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.74trial/s, best loss: -0.5750000000000001]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 42.32trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.41trial/s, best loss: -0.5833333333333334]\n",
      "100%|████████████████████| 100/100 [00:02<00:00, 42.74trial/s, best loss: -0.55]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.80trial/s, best loss: -0.5166666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.00trial/s, best loss: -0.5666666666666668]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.19trial/s, best loss: -0.5499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.57trial/s, best loss: -0.5583333333333335]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 41.49trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.00trial/s, best loss: -0.5416666666666666]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 44.91trial/s, best loss: -0.525]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.63trial/s, best loss: -0.5916666666666667]\n",
      "100%|████████████████████| 100/100 [00:02<00:00, 44.94trial/s, best loss: -0.55]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.74trial/s, best loss: -0.5750000000000001]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.38trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 43.76trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.83trial/s, best loss: -0.6249999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.04trial/s, best loss: -0.5583333333333335]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.07trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.01trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 40.78trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.95trial/s, best loss: -0.6083333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.12trial/s, best loss: -0.6250000000000001]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.32trial/s, best loss: -0.5666666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.10trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.74trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 38.93trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.64trial/s, best loss: -0.6166666666666667]\n",
      "100%|████████████████████| 100/100 [00:02<00:00, 42.87trial/s, best loss: -0.55]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.29trial/s, best loss: -0.6166666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.65trial/s, best loss: -0.6083333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.59trial/s, best loss: -0.6499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.86trial/s, best loss: -0.6166666666666667]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 42.42trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.28trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.75trial/s, best loss: -0.5499999999999999]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 39.76trial/s, best loss: -0.625]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.58trial/s, best loss: -0.5499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.80trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.94trial/s, best loss: -0.6416666666666666]\n",
      "100%|████████████████████| 100/100 [00:02<00:00, 43.52trial/s, best loss: -0.55]\n",
      "100%|████████████████████| 100/100 [00:02<00:00, 40.71trial/s, best loss: -0.65]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.71trial/s, best loss: -0.6166666666666667]\n",
      "100%|███████| 100/100 [00:02<00:00, 38.40trial/s, best loss: -0.557142857142857]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.77trial/s, best loss: -0.5083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.56trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.01trial/s, best loss: -0.5436507936507936]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.08trial/s, best loss: -0.5869047619047619]\n",
      "100%|██████| 100/100 [00:02<00:00, 38.97trial/s, best loss: -0.6138888888888888]\n",
      "100%|██████| 100/100 [00:02<00:00, 38.98trial/s, best loss: -0.5603174603174603]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.23trial/s, best loss: -0.5313492063492063]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.00trial/s, best loss: -0.5361111111111111]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.01trial/s, best loss: -0.6301587301587303]\n",
      "100%|███████| 100/100 [00:02<00:00, 40.50trial/s, best loss: -0.638095238095238]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.08trial/s, best loss: -0.6313492063492064]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.82trial/s, best loss: -0.5992063492063492]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.96trial/s, best loss: -0.6003968253968254]\n",
      "100%|██████| 100/100 [00:02<00:00, 38.92trial/s, best loss: -0.6130952380952381]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.02trial/s, best loss: -0.5599206349206349]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 39.06trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.70trial/s, best loss: -0.5571428571428572]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.34trial/s, best loss: -0.6638888888888889]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.57trial/s, best loss: -0.5753968253968255]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.61trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 38.28trial/s, best loss: -0.5662698412698413]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.28trial/s, best loss: -0.6162698412698413]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.81trial/s, best loss: -0.5928571428571429]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.78trial/s, best loss: -0.5904761904761905]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.11trial/s, best loss: -0.6301587301587303]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.53trial/s, best loss: -0.5083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.86trial/s, best loss: -0.5603174603174602]\n",
      "100%|███████| 100/100 [00:02<00:00, 38.83trial/s, best loss: -0.623015873015873]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.72trial/s, best loss: -0.5753968253968255]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.36trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 41.91trial/s, best loss: -0.6333333333333334]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.62trial/s, best loss: -0.6166666666666667]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 43.92trial/s, best loss: -0.575]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 44.08trial/s, best loss: -0.6]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 44.94trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.19trial/s, best loss: -0.5499999999999999]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.50trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 44.31trial/s, best loss: -0.5666666666666667]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 39.25trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.67trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:02<00:00, 39.02trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 39.61trial/s, best loss: -0.6]\n",
      "100%|███████████████████| 100/100 [00:02<00:00, 39.46trial/s, best loss: -0.625]\n",
      "100%|██████| 100/100 [00:02<00:00, 42.34trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 41.76trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:02<00:00, 40.23trial/s, best loss: -0.5416666666666666]\n",
      "100%|██████| 100/100 [00:02<00:00, 45.17trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [00:02<00:00, 43.26trial/s, best loss: -0.5833333333333334]\n",
      "100%|█████████████████████| 100/100 [00:02<00:00, 39.09trial/s, best loss: -0.6]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:42<00:00,  2.35trial/s, best loss: -0.6333333333333333]\n",
      "100%|██████| 100/100 [00:58<00:00,  1.71trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.52trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.89trial/s, best loss: -0.7416666666666667]\n",
      "100%|████████████████████| 100/100 [00:33<00:00,  3.02trial/s, best loss: -0.65]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.6583333333333333]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.46trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.62trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.87trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.48trial/s, best loss: -0.6583333333333333]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.60trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:33<00:00,  3.00trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:49<00:00,  2.03trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.89trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.04trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.57trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.05trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [00:56<00:00,  1.77trial/s, best loss: -0.6749999999999999]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.49trial/s, best loss: -0.6749999999999999]\n",
      "100%|██████| 100/100 [00:46<00:00,  2.14trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:47<00:00,  2.11trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.63trial/s, best loss: -0.7250000000000001]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.49trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.7416666666666667]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.95trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.89trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.69trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [00:41<00:00,  2.39trial/s, best loss: -0.7416666666666667]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.45trial/s, best loss: -0.7000000000000001]\n",
      "100%|████████████████████| 100/100 [00:53<00:00,  1.88trial/s, best loss: -0.65]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.49trial/s, best loss: -0.7666666666666666]\n",
      "100%|██████| 100/100 [00:42<00:00,  2.35trial/s, best loss: -0.7333333333333334]\n",
      "100%|███████████████████| 100/100 [00:35<00:00,  2.81trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.63trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.54trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [01:11<00:00,  1.41trial/s, best loss: -0.7083333333333334]\n",
      "100%|████████████████████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.75]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.48trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.45trial/s, best loss: -0.6750000000000002]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.87trial/s, best loss: -0.6833333333333332]\n",
      "100%|████████████████████| 100/100 [00:51<00:00,  1.95trial/s, best loss: -0.75]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.63trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.79trial/s, best loss: -0.7166666666666668]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.91trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.48trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.58trial/s, best loss: -0.7583333333333333]\n",
      "100%|███████████████████| 100/100 [01:07<00:00,  1.49trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.93trial/s, best loss: -0.6749999999999999]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.06trial/s, best loss: -0.6892857142857144]\n",
      "100%|██████| 100/100 [01:11<00:00,  1.39trial/s, best loss: -0.6714285714285714]\n",
      "100%|███████| 100/100 [00:34<00:00,  2.89trial/s, best loss: -0.703968253968254]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.89trial/s, best loss: -0.7067460317460318]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.07trial/s, best loss: -0.6976190476190477]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.56trial/s, best loss: -0.7384920634920635]\n",
      "100%|██████| 100/100 [00:41<00:00,  2.41trial/s, best loss: -0.7123015873015873]\n",
      "100%|██████| 100/100 [00:59<00:00,  1.67trial/s, best loss: -0.7218253968253968]\n",
      "100%|██████| 100/100 [00:37<00:00,  2.66trial/s, best loss: -0.7043650793650794]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.47trial/s, best loss: -0.6797619047619047]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.48trial/s, best loss: -0.6952380952380953]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.74trial/s, best loss: -0.7055555555555556]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.74trial/s, best loss: -0.7440476190476191]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.71trial/s, best loss: -0.7686507936507937]\n",
      "100%|██████| 100/100 [00:35<00:00,  2.81trial/s, best loss: -0.7297619047619047]\n",
      "100%|███████| 100/100 [00:46<00:00,  2.17trial/s, best loss: -0.736904761904762]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.7047619047619048]\n",
      "100%|██████| 100/100 [00:43<00:00,  2.27trial/s, best loss: -0.7123015873015873]\n",
      "100%|██████| 100/100 [00:41<00:00,  2.41trial/s, best loss: -0.6888888888888888]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.7206349206349207]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.7111111111111111]\n",
      "100%|██████| 100/100 [00:47<00:00,  2.11trial/s, best loss: -0.7043650793650794]\n",
      "100%|████████| 100/100 [00:42<00:00,  2.33trial/s, best loss: -0.71984126984127]\n",
      "100%|██████| 100/100 [01:09<00:00,  1.44trial/s, best loss: -0.7293650793650793]\n",
      "100%|██████| 100/100 [00:45<00:00,  2.21trial/s, best loss: -0.7289682539682539]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.7365079365079366]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.94trial/s, best loss: -0.7369047619047618]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.51trial/s, best loss: -0.7293650793650793]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.75trial/s, best loss: -0.7126984126984127]\n",
      "100%|██████| 100/100 [00:35<00:00,  2.80trial/s, best loss: -0.7063492063492062]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.6583333333333333]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.6583333333333333]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.71trial/s, best loss: -0.7583333333333334]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.73trial/s, best loss: -0.6583333333333333]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.87trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:14<00:00,  1.34trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:13<00:00,  1.37trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.62trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:19<00:00,  1.25trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.76trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.54trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.79trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [00:42<00:00,  2.36trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [00:54<00:00,  1.84trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.7416666666666667]\n",
      "100%|███████████████████| 100/100 [00:54<00:00,  1.82trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.54trial/s, best loss: -0.7000000000000001]\n",
      "100%|███████████████████| 100/100 [01:02<00:00,  1.59trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.44trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:42<00:00,  2.34trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.59trial/s, best loss: -0.7000000000000001]\n",
      " 85%|█████▉ | 85/100 [01:30<00:15,  1.06s/trial, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.88trial/s, best loss: -0.6333333333333334]\n",
      "100%|██████| 100/100 [00:42<00:00,  2.33trial/s, best loss: -0.6416666666666667]\n",
      "100%|███████████████████| 100/100 [01:14<00:00,  1.34trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [00:35<00:00,  2.79trial/s, best loss: -0.6083333333333334]\n",
      "100%|████████████████████| 100/100 [00:38<00:00,  2.63trial/s, best loss: -0.65]\n",
      "100%|█████████████████████| 100/100 [00:36<00:00,  2.75trial/s, best loss: -0.6]\n",
      "100%|████████████████████| 100/100 [00:32<00:00,  3.11trial/s, best loss: -0.65]\n",
      "100%|██████| 100/100 [01:01<00:00,  1.63trial/s, best loss: -0.6750000000000002]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.57trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.59trial/s, best loss: -0.6083333333333334]\n",
      "100%|████████████████████| 100/100 [00:28<00:00,  3.49trial/s, best loss: -0.65]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.59trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [00:45<00:00,  2.19trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [01:20<00:00,  1.24trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.47trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.88trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:40<00:00,  2.47trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [01:15<00:00,  1.33trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:47<00:00,  2.10trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.57trial/s, best loss: -0.7083333333333334]\n",
      "100%|██████| 100/100 [01:24<00:00,  1.19trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.79trial/s, best loss: -0.6999999999999998]\n",
      "100%|██████| 100/100 [01:25<00:00,  1.17trial/s, best loss: -0.6416666666666666]\n",
      "100%|███████████████████| 100/100 [01:05<00:00,  1.53trial/s, best loss: -0.625]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.91trial/s, best loss: -0.7166666666666667]\n",
      "100%|██████| 100/100 [01:10<00:00,  1.42trial/s, best loss: -0.7333333333333333]\n",
      " 97%|██████▊| 97/100 [01:30<00:02,  1.07trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:49<00:00,  2.01trial/s, best loss: -0.6750000000000002]\n",
      "100%|███████████████████| 100/100 [01:08<00:00,  1.46trial/s, best loss: -0.775]\n",
      "100%|████████████████████| 100/100 [00:51<00:00,  1.93trial/s, best loss: -0.75]\n",
      "100%|███████████████████| 100/100 [01:12<00:00,  1.38trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [01:19<00:00,  1.26trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [01:21<00:00,  1.22trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [00:33<00:00,  2.98trial/s, best loss: -0.7083333333333334]\n",
      "100%|████████████████████| 100/100 [00:39<00:00,  2.50trial/s, best loss: -0.75]\n",
      "100%|██████| 100/100 [00:30<00:00,  3.33trial/s, best loss: -0.7416666666666667]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.79trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [01:19<00:00,  1.26trial/s, best loss: -0.6083333333333333]\n",
      "100%|██████| 100/100 [01:00<00:00,  1.65trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:52<00:00,  1.89trial/s, best loss: -0.6750000000000002]\n",
      "100%|██████| 100/100 [00:49<00:00,  2.03trial/s, best loss: -0.6666666666666666]\n",
      " 93%|██████▌| 93/100 [01:30<00:06,  1.03trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:37<00:00,  2.69trial/s, best loss: -0.6833333333333332]\n",
      "100%|██████| 100/100 [01:04<00:00,  1.56trial/s, best loss: -0.6749999999999999]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.81trial/s, best loss: -0.7166666666666668]\n",
      "100%|██████| 100/100 [01:28<00:00,  1.12trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:44<00:00,  2.25trial/s, best loss: -0.6646825396825397]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.77trial/s, best loss: -0.6166666666666667]\n",
      "100%|██████| 100/100 [00:46<00:00,  2.14trial/s, best loss: -0.6634920634920635]\n",
      "100%|██████| 100/100 [00:30<00:00,  3.27trial/s, best loss: -0.6817460317460317]\n",
      "100%|██████| 100/100 [00:28<00:00,  3.54trial/s, best loss: -0.7373015873015873]\n",
      "100%|██████| 100/100 [00:53<00:00,  1.86trial/s, best loss: -0.7555555555555555]\n",
      "100%|██████| 100/100 [00:46<00:00,  2.14trial/s, best loss: -0.6638888888888889]\n",
      " 98%|██████▊| 98/100 [01:31<00:01,  1.07trial/s, best loss: -0.7222222222222222]\n",
      "100%|██████| 100/100 [00:36<00:00,  2.77trial/s, best loss: -0.6976190476190477]\n",
      "100%|██████| 100/100 [01:14<00:00,  1.34trial/s, best loss: -0.6718253968253968]\n",
      "100%|██████| 100/100 [00:28<00:00,  3.49trial/s, best loss: -0.6626984126984127]\n",
      "100%|██████| 100/100 [01:21<00:00,  1.22trial/s, best loss: -0.7055555555555554]\n",
      "100%|██████| 100/100 [00:55<00:00,  1.80trial/s, best loss: -0.7456349206349207]\n",
      "100%|██████| 100/100 [00:50<00:00,  1.99trial/s, best loss: -0.7543650793650793]\n",
      "100%|██████| 100/100 [01:27<00:00,  1.14trial/s, best loss: -0.7051587301587302]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.08trial/s, best loss: -0.6952380952380951]\n",
      "100%|██████| 100/100 [00:35<00:00,  2.80trial/s, best loss: -0.6718253968253968]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.91trial/s, best loss: -0.6634920634920635]\n",
      "100%|██████| 100/100 [00:46<00:00,  2.13trial/s, best loss: -0.7055555555555556]\n",
      "100%|██████| 100/100 [01:30<00:00,  1.11trial/s, best loss: -0.6722222222222222]\n",
      "100%|██████| 100/100 [00:47<00:00,  2.10trial/s, best loss: -0.6884920634920636]\n",
      "100%|██████| 100/100 [01:11<00:00,  1.40trial/s, best loss: -0.7055555555555556]\n",
      " 97%|██████▊| 97/100 [01:30<00:02,  1.07trial/s, best loss: -0.7365079365079366]\n",
      "100%|██████| 100/100 [00:45<00:00,  2.19trial/s, best loss: -0.7543650793650795]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.95trial/s, best loss: -0.7115079365079365]\n",
      "100%|██████| 100/100 [00:43<00:00,  2.30trial/s, best loss: -0.7293650793650794]\n",
      "100%|██████| 100/100 [01:21<00:00,  1.22trial/s, best loss: -0.7146825396825397]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.95trial/s, best loss: -0.7222222222222222]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.58trial/s, best loss: -0.6964285714285715]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.07trial/s, best loss: -0.6896825396825396]\n",
      "100%|██████| 100/100 [00:50<00:00,  1.99trial/s, best loss: -0.6416666666666667]\n",
      "100%|██████| 100/100 [00:23<00:00,  4.27trial/s, best loss: -0.6916666666666665]\n",
      "100%|██████| 100/100 [00:49<00:00,  2.03trial/s, best loss: -0.7250000000000001]\n",
      "100%|██████| 100/100 [00:26<00:00,  3.77trial/s, best loss: -0.6749999999999999]\n",
      "100%|██████| 100/100 [00:35<00:00,  2.79trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:57<00:00,  1.75trial/s, best loss: -0.6749999999999999]\n",
      "100%|██████| 100/100 [00:49<00:00,  2.04trial/s, best loss: -0.5916666666666667]\n",
      "100%|██████| 100/100 [00:27<00:00,  3.58trial/s, best loss: -0.6666666666666666]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.93trial/s, best loss: -0.6833333333333332]\n",
      "100%|███████████████████| 100/100 [01:00<00:00,  1.66trial/s, best loss: -0.725]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.50trial/s, best loss: -0.7333333333333334]\n",
      "100%|██████| 100/100 [00:34<00:00,  2.87trial/s, best loss: -0.7000000000000001]\n",
      "100%|██████| 100/100 [00:51<00:00,  1.93trial/s, best loss: -0.7416666666666666]\n",
      "100%|██████| 100/100 [00:33<00:00,  2.99trial/s, best loss: -0.6666666666666666]\n",
      " 96%|████████████████████▏| 96/100 [01:30<00:03,  1.07trial/s, best loss: -0.75]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.58trial/s, best loss: -0.6916666666666668]\n",
      "100%|██████| 100/100 [00:48<00:00,  2.06trial/s, best loss: -0.7416666666666667]\n",
      "100%|██████| 100/100 [00:38<00:00,  2.61trial/s, best loss: -0.7333333333333334]\n",
      "100%|██████| 100/100 [00:37<00:00,  2.70trial/s, best loss: -0.7083333333333334]\n",
      "100%|███████████████████| 100/100 [00:28<00:00,  3.47trial/s, best loss: -0.725]\n"
     ]
    }
   ],
   "source": [
    "for name, (X,y) in data.items(): \n",
    "\n",
    "    #store recall values for each algorithm\n",
    "    score = {}\n",
    "    for algorithm in algorithms.keys():\n",
    "        score[algorithm] = []\n",
    "\n",
    "    #store auc values for each algorithm\n",
    "    auc_score = {}\n",
    "    for algorithm in algorithms.keys():\n",
    "        auc_score[algorithm] = []\n",
    "    \n",
    "    #for each algorithm and its respective search space\n",
    "    for algorithm, (clf, search_space) in algorithms.items():\n",
    "\n",
    "        for train, test in kf.split(X, y):\n",
    "            \n",
    "            #train and test alocation\n",
    "            X_train, X_test = X.iloc[train], X.iloc[test]\n",
    "            Y_train, y_test = y.iloc[train], y.iloc[test]\n",
    "            \n",
    "            y_ = pd.DataFrame.from_dict(y)\n",
    "            #if the classes are unbalanced\n",
    "            if (((y_[y_.severity == 1].shape[0])*1.5) < (y_[y_.severity == 0].shape[0])):\n",
    "                \n",
    "                for j in range(0,10):\n",
    "                    \n",
    "                    #vectors to store y_pred e y_true\n",
    "                    y_pred = [] \n",
    "                    y_true = [] \n",
    "\n",
    "                    #undersampling of the majority class\n",
    "                    #undersampling the majority class when classes are umbalanced\n",
    "                    under = RandomUnderSampler(sampling_strategy='majority', random_state = j)\n",
    "                    x_train, y_train = under.fit_resample(X_train, Y_train)\n",
    "                    x_test = X_test\n",
    "                                    \n",
    "                    imputer.fit(x_train)\n",
    "                    x_train = imputer.transform(x_train)\n",
    "                    x_test = imputer.transform(x_test)   \n",
    "\n",
    "                    #standardize the features                \n",
    "                    prep.fit(x_train)\n",
    "\n",
    "                    best = fmin(\n",
    "                      fn=objective,\n",
    "                      space=search_space,\n",
    "                      algo=tpe.suggest,\n",
    "                      max_evals=100,\n",
    "                      timeout= 90,\n",
    "                      rstate= rstate)        \n",
    "\n",
    "                    best = space_eval(search_space, best)\n",
    "                    best = clf(**best)\n",
    "\n",
    "                    #search for the best hyperparameters\n",
    "                    best.fit(prep.transform(x_train), y_train)\n",
    "\n",
    "                    #store the results\n",
    "                    y_pred = [*y_pred, *(best.predict(prep.transform(x_test)))] \n",
    "                    y_true =  [*y_true, *y_test] \n",
    "\n",
    "                    #calculate the recall\n",
    "                    score[algorithm].append(recall_score(y_true, y_pred, labels = [0,1], average = None))\n",
    "\n",
    "                    #calculate the area under roc curve\n",
    "                    aucscore = roc_auc_score(y_test, (best.predict_proba(prep.transform(x_test)))[:, 1])\n",
    "                    auc_score[algorithm].append(aucscore)\n",
    "           \n",
    "            else:\n",
    "                 #vectors to store y_pred e y_true\n",
    "                y_pred = [] \n",
    "                y_true = [] \n",
    "\n",
    "                x_train = X_train\n",
    "                y_train = Y_train\n",
    "                x_test = X_test\n",
    "                \n",
    "                imputer.fit(x_train)\n",
    "                x_train = imputer.transform(x_train)\n",
    "                x_test = imputer.transform(x_test)   \n",
    "\n",
    "                #standardize the features                \n",
    "                prep.fit(x_train)\n",
    "\n",
    "                best = fmin(\n",
    "                  fn=objective,\n",
    "                  space=search_space,\n",
    "                  algo=tpe.suggest,\n",
    "                  max_evals=100,\n",
    "                  timeout= 90,\n",
    "                  rstate=rstate)       \n",
    "\n",
    "                best = space_eval(search_space, best)\n",
    "                best = clf(**best)\n",
    "\n",
    "                #search for the best hyperparameters\n",
    "                best.fit(prep.transform(x_train), y_train)\n",
    "\n",
    "                #store the results\n",
    "                y_pred = [*y_pred, *(best.predict(prep.transform(x_test)))] \n",
    "                y_true =  [*y_true, *y_test] \n",
    "\n",
    "                #calculate the recall\n",
    "                score[algorithm].append(recall_score(y_true, y_pred, labels = [0,1], average = None))\n",
    "\n",
    "                #calculate the area under roc curve\n",
    "                aucscore = roc_auc_score(y_test, (best.predict_proba(prep.transform(x_test)))[:, 1])\n",
    "                auc_score[algorithm].append(aucscore)\n",
    "            \n",
    "    auc_score = pd.DataFrame.from_dict(auc_score)  \n",
    "    auc_score.to_csv(name + '_auc.csv')\n",
    "\n",
    "    #write a csv with the recall of class '0' - specificity \n",
    "    #and another csv with the recall of class '1' - sensitivity\n",
    "    recall_svm_linear = pd.DataFrame(np.vstack(score['SVM_linear']))\n",
    "    recall_svm_rbf = pd.DataFrame(np.vstack(score['SVM_rbf']))\n",
    "    recall_gb = pd.DataFrame(np.vstack(score['GB']))\n",
    "    recall_rf = pd.DataFrame(np.vstack(score['RF']))\n",
    "\n",
    "    esp = pd.concat([recall_svm_linear[[0]], recall_svm_rbf[[0]], recall_rf[[0]], recall_gb[[0]]], axis=1)\n",
    "    sen = pd.concat([recall_svm_linear[[1]], recall_svm_rbf[[1]], recall_rf[[1]], recall_gb[[1]]], axis=1)\n",
    "\n",
    "    esp.columns = ['SVM_rbf', 'SVM_linear', 'RF', 'GB']\n",
    "    sen.columns = ['SVM_rbf', 'SVM_linear', 'RF', 'GB']\n",
    "\n",
    "    esp.to_csv(name + '_spe.csv')\n",
    "    sen.to_csv(name + '_sen.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7c847d3b-ca78-4f7f-b9b9-6eb1a29fc87a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svm_linear</th>\n",
       "      <th>svm_rbf</th>\n",
       "      <th>gb</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auc</th>\n",
       "      <td>0.656</td>\n",
       "      <td>0.656</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>0.608</td>\n",
       "      <td>0.608</td>\n",
       "      <td>0.691</td>\n",
       "      <td>0.693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spe</th>\n",
       "      <td>0.615</td>\n",
       "      <td>0.616</td>\n",
       "      <td>0.664</td>\n",
       "      <td>0.679</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     svm_linear  svm_rbf     gb     rf\n",
       "auc       0.656    0.656  0.749  0.759\n",
       "sen       0.608    0.608  0.691  0.693\n",
       "spe       0.615    0.616  0.664  0.679"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#hosp1\n",
    "auc = pd.read_csv('hosp1_auc.csv')\n",
    "\n",
    "# recall\n",
    "sen = pd.read_csv('hosp1_sen.csv')\n",
    "spe = pd.read_csv('hosp1_spe.csv')\n",
    "\n",
    "column_names = [\"svm_linear\", \"svm_rbf\", \"gb\", \"rf\"]\n",
    "\n",
    "df = pd.DataFrame(columns = column_names)\n",
    "df.loc['auc'] = [auc['SVM_linear'].mean(), auc['SVM_rbf'].mean(), auc['GB'].mean(), auc['RF'].mean()]\n",
    "df.loc['sen'] = [sen['SVM_linear'].mean(), sen['SVM_rbf'].mean(), sen['GB'].mean(), sen['RF'].mean()]\n",
    "df.loc['spe'] = [spe['SVM_linear'].mean(), spe['SVM_rbf'].mean(), spe['GB'].mean(), spe['RF'].mean()]\n",
    "\n",
    "\n",
    "df = df.round(decimals=3)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e2706bad-0cf4-45c6-a25f-4ccc40c06224",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svm_linear</th>\n",
       "      <th>svm_rbf</th>\n",
       "      <th>gb</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auc</th>\n",
       "      <td>0.045</td>\n",
       "      <td>0.046</td>\n",
       "      <td>0.028</td>\n",
       "      <td>0.026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>0.070</td>\n",
       "      <td>0.071</td>\n",
       "      <td>0.057</td>\n",
       "      <td>0.051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spe</th>\n",
       "      <td>0.044</td>\n",
       "      <td>0.043</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.050</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     svm_linear  svm_rbf     gb     rf\n",
       "auc       0.045    0.046  0.028  0.026\n",
       "sen       0.070    0.071  0.057  0.051\n",
       "spe       0.044    0.043  0.042  0.050"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns = column_names)\n",
    "df.loc['auc'] = [auc['SVM_linear'].std(), auc['SVM_rbf'].std(), auc['GB'].std(), auc['RF'].std()]\n",
    "df.loc['sen'] = [sen['SVM_linear'].std(), sen['SVM_rbf'].std(), sen['GB'].std(), sen['RF'].std()]\n",
    "df.loc['spe'] = [spe['SVM_linear'].std(), spe['SVM_rbf'].std(), spe['GB'].std(), spe['RF'].std()]\n",
    "\n",
    "\n",
    "df = df.round(decimals=3)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0535cc39-0973-47ad-83e7-47fbebc963d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svm_linear</th>\n",
       "      <th>svm_rbf</th>\n",
       "      <th>gb</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auc</th>\n",
       "      <td>0.757</td>\n",
       "      <td>0.740</td>\n",
       "      <td>0.696</td>\n",
       "      <td>0.738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>0.666</td>\n",
       "      <td>0.665</td>\n",
       "      <td>0.613</td>\n",
       "      <td>0.617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spe</th>\n",
       "      <td>0.710</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.661</td>\n",
       "      <td>0.701</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     svm_linear  svm_rbf     gb     rf\n",
       "auc       0.757    0.740  0.696  0.738\n",
       "sen       0.666    0.665  0.613  0.617\n",
       "spe       0.710    0.714  0.661  0.701"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#hosp2\n",
    "auc = pd.read_csv('hosp2_auc.csv')\n",
    "\n",
    "# recall\n",
    "sen = pd.read_csv('hosp2_sen.csv')\n",
    "spe = pd.read_csv('hosp2_spe.csv')\n",
    "\n",
    "column_names = [\"svm_linear\", \"svm_rbf\", \"gb\", \"rf\"]\n",
    "\n",
    "df = pd.DataFrame(columns = column_names)\n",
    "df.loc['auc'] = [auc['SVM_linear'].mean(), auc['SVM_rbf'].mean(), auc['GB'].mean(), auc['RF'].mean()]\n",
    "df.loc['sen'] = [sen['SVM_linear'].mean(), sen['SVM_rbf'].mean(), sen['GB'].mean(), sen['RF'].mean()]\n",
    "df.loc['spe'] = [spe['SVM_linear'].mean(), spe['SVM_rbf'].mean(), spe['GB'].mean(), spe['RF'].mean()]\n",
    "\n",
    "\n",
    "df = df.round(decimals=3)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "010487cc-6b18-4998-b864-e7176e578211",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svm_linear</th>\n",
       "      <th>svm_rbf</th>\n",
       "      <th>gb</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auc</th>\n",
       "      <td>0.121</td>\n",
       "      <td>0.154</td>\n",
       "      <td>0.119</td>\n",
       "      <td>0.124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>0.192</td>\n",
       "      <td>0.188</td>\n",
       "      <td>0.202</td>\n",
       "      <td>0.210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spe</th>\n",
       "      <td>0.110</td>\n",
       "      <td>0.103</td>\n",
       "      <td>0.146</td>\n",
       "      <td>0.131</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     svm_linear  svm_rbf     gb     rf\n",
       "auc       0.121    0.154  0.119  0.124\n",
       "sen       0.192    0.188  0.202  0.210\n",
       "spe       0.110    0.103  0.146  0.131"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = pd.DataFrame(columns = column_names)\n",
    "df.loc['auc'] = [auc['SVM_linear'].std(), auc['SVM_rbf'].std(), auc['GB'].std(), auc['RF'].std()]\n",
    "df.loc['sen'] = [sen['SVM_linear'].std(), sen['SVM_rbf'].std(), sen['GB'].std(), sen['RF'].std()]\n",
    "df.loc['spe'] = [spe['SVM_linear'].std(), spe['SVM_rbf'].std(), spe['GB'].std(), spe['RF'].std()]\n",
    "\n",
    "\n",
    "df = df.round(decimals=3)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9a69a376-2156-4f7f-a2bb-7471680f7651",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████| 100/100 [01:05<00:00,  1.53trial/s, best loss: -0.5877685755863722]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.50trial/s, best loss: -0.5840716486902927]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.47trial/s, best loss: -0.5773679592535524]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.47trial/s, best loss: -0.5850721195000856]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.47trial/s, best loss: -0.5935627461051189]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.49trial/s, best loss: -0.5888760486218113]\n",
      "100%|██████| 100/100 [01:08<00:00,  1.46trial/s, best loss: -0.5934717942133196]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.52trial/s, best loss: -0.5576581492895052]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.49trial/s, best loss: -0.5888492980653998]\n",
      "100%|██████| 100/100 [01:07<00:00,  1.49trial/s, best loss: -0.5699955059065228]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.58trial/s, best loss: -0.5877685755863722]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.51trial/s, best loss: -0.5840716486902927]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.52trial/s, best loss: -0.5773679592535524]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.53trial/s, best loss: -0.5841304999143982]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.52trial/s, best loss: -0.5935627461051189]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.52trial/s, best loss: -0.5879504793699709]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.51trial/s, best loss: -0.5934717942133196]\n",
      "100%|██████| 100/100 [01:03<00:00,  1.56trial/s, best loss: -0.5586051189864749]\n",
      "100%|██████| 100/100 [01:05<00:00,  1.52trial/s, best loss: -0.5888546481766821]\n",
      "100%|██████| 100/100 [01:06<00:00,  1.50trial/s, best loss: -0.5690538863208355]\n",
      " 67%|████▋  | 67/100 [01:31<00:44,  1.36s/trial, best loss: -0.7022823574730355]\n",
      " 79%|█████▌ | 79/100 [01:30<00:24,  1.15s/trial, best loss: -0.7126348228043143]\n",
      "100%|██████| 100/100 [01:17<00:00,  1.29trial/s, best loss: -0.6956214689265536]\n",
      " 79%|█████▌ | 79/100 [01:30<00:23,  1.14s/trial, best loss: -0.6880564115733607]\n",
      "100%|███████| 100/100 [01:20<00:00,  1.24trial/s, best loss: -0.692753809279233]\n",
      " 77%|█████▍ | 77/100 [01:30<00:26,  1.17s/trial, best loss: -0.7013139873309365]\n",
      " 70%|████▉  | 70/100 [01:30<00:38,  1.30s/trial, best loss: -0.7013781886663243]\n",
      "100%|██████| 100/100 [01:10<00:00,  1.42trial/s, best loss: -0.6766392740969013]\n",
      "100%|██████| 100/100 [01:24<00:00,  1.18trial/s, best loss: -0.6918710409176511]\n",
      " 70%|████▉  | 70/100 [01:30<00:38,  1.29s/trial, best loss: -0.6918817411402157]\n",
      " 36%|██▌    | 36/100 [01:31<02:42,  2.54s/trial, best loss: -0.6928180106146208]\n",
      " 50%|███▌   | 50/100 [01:30<01:30,  1.81s/trial, best loss: -0.6965523882896765]\n",
      " 36%|██▌    | 36/100 [01:30<02:40,  2.51s/trial, best loss: -0.6880671117959253]\n",
      " 34%|██▍    | 34/100 [01:33<03:02,  2.76s/trial, best loss: -0.6823638931689779]\n",
      " 40%|██▊    | 40/100 [01:30<02:15,  2.27s/trial, best loss: -0.6758153569594247]\n",
      " 37%|██▌    | 37/100 [01:37<02:46,  2.64s/trial, best loss: -0.6814757746961136]\n",
      " 38%|██▋    | 38/100 [01:32<02:30,  2.43s/trial, best loss: -0.6909989727786338]\n",
      " 28%|█▉     | 28/100 [01:33<04:00,  3.35s/trial, best loss: -0.6794694829652457]\n",
      " 35%|██▍    | 35/100 [01:33<02:52,  2.66s/trial, best loss: -0.6890675826057183]\n",
      " 32%|██▏    | 32/100 [01:30<03:11,  2.82s/trial, best loss: -0.6852476031501454]\n"
     ]
    }
   ],
   "source": [
    "#armazena o recall de cada algoritmo\n",
    "score = {}\n",
    "for algorithm in algorithms.keys():\n",
    "    score[algorithm] = []\n",
    "\n",
    "#armazena auc de cada algoritmo    \n",
    "auc_score = {}\n",
    "for algorithm in algorithms.keys():\n",
    "    auc_score[algorithm] = []\n",
    "\n",
    "#for each algorithm and its respective search space\n",
    "for algorithm, (clf, search_space) in algorithms.items():\n",
    "\n",
    "    for j in range(0,10):\n",
    "        \n",
    "        X_train = hosp1.drop(columns=['severity'])\n",
    "        Y_train = hosp1.severity\n",
    "        X_test = hosp2.drop(columns=['severity'])\n",
    "        y_test = hosp2.severity\n",
    "\n",
    "        #vectors to store y_pred e y_true\n",
    "        y_pred = [] \n",
    "        y_true = [] \n",
    "\n",
    "        #undersampling of the majority class\n",
    "        #undersampling the majority class when classes are umbalanced\n",
    "        under = RandomUnderSampler(sampling_strategy='majority', random_state = j)\n",
    "        x_train, y_train = under.fit_resample(X_train, Y_train)\n",
    "        x_test = X_test\n",
    "\n",
    "        imputer.fit(x_train)\n",
    "        x_train = imputer.transform(x_train)\n",
    "        x_test = imputer.transform(x_test)   \n",
    "\n",
    "\n",
    "        #standardize the features                \n",
    "        prep.fit(x_train)\n",
    "\n",
    "        best = fmin(\n",
    "          fn=objective,\n",
    "          space=search_space,\n",
    "          algo=tpe.suggest,\n",
    "          max_evals=100,\n",
    "          timeout= 90,\n",
    "          rstate=rstate)       \n",
    "\n",
    "        best = space_eval(search_space, best)\n",
    "        best = clf(**best)\n",
    "\n",
    "        #search for the best hyperparameters\n",
    "        best.fit(prep.transform(x_train), y_train)\n",
    "\n",
    "        #store the results\n",
    "        y_pred = [*y_pred, *(best.predict(prep.transform(x_test)))] \n",
    "        y_true =  [*y_true, *y_test] \n",
    "\n",
    "        #calculate the recall\n",
    "        score[algorithm].append(recall_score(y_true, y_pred, labels = [0,1], average = None))\n",
    "\n",
    "        #calculate the area under roc curve\n",
    "        aucscore = roc_auc_score(y_test, (best.predict_proba(prep.transform(x_test)))[:, 1])\n",
    "        auc_score[algorithm].append(aucscore)\n",
    "\n",
    "auc_score = pd.DataFrame.from_dict(auc_score)  \n",
    "auc_score.to_csv('crossmodels_auc.csv')\n",
    "\n",
    "#write a csv with the recall of class '0' - specificity \n",
    "#and another csv with the recall of class '1' - sensitivity\n",
    "recall_svm_linear = pd.DataFrame(np.vstack(score['SVM_linear']))\n",
    "recall_svm_rbf = pd.DataFrame(np.vstack(score['SVM_rbf']))\n",
    "recall_gb = pd.DataFrame(np.vstack(score['GB']))\n",
    "recall_rf = pd.DataFrame(np.vstack(score['RF']))\n",
    "\n",
    "esp = pd.concat([recall_svm_linear[[0]], recall_svm_rbf[[0]], recall_rf[[0]], recall_gb[[0]]], axis=1)\n",
    "sen = pd.concat([recall_svm_linear[[1]], recall_svm_rbf[[1]], recall_rf[[1]], recall_gb[[1]]], axis=1)\n",
    "\n",
    "esp.columns = ['SVM_rbf', 'SVM_linear', 'RF', 'GB']\n",
    "sen.columns = ['SVM_rbf', 'SVM_linear', 'RF', 'GB']\n",
    "\n",
    "esp.to_csv('crossmodels_spe.csv')\n",
    "sen.to_csv('crossmodels_sen.csv')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e3d9c495-28c1-4f72-92b7-2c0e4aa01157",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svm_linear</th>\n",
       "      <th>svm_rbf</th>\n",
       "      <th>gb</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auc</th>\n",
       "      <td>0.662</td>\n",
       "      <td>0.660</td>\n",
       "      <td>0.769</td>\n",
       "      <td>0.779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>0.581</td>\n",
       "      <td>0.581</td>\n",
       "      <td>0.660</td>\n",
       "      <td>0.663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spe</th>\n",
       "      <td>0.644</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.742</td>\n",
       "      <td>0.727</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     svm_linear  svm_rbf     gb     rf\n",
       "auc       0.662    0.660  0.769  0.779\n",
       "sen       0.581    0.581  0.660  0.663\n",
       "spe       0.644    0.647  0.742  0.727"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cruzados\n",
    "auc = pd.read_csv('crossmodels_auc.csv')\n",
    "\n",
    "# recall\n",
    "sen = pd.read_csv('crossmodels_sen.csv')\n",
    "spe = pd.read_csv('crossmodels_spe.csv')\n",
    "\n",
    "column_names = [\"svm_linear\", \"svm_rbf\", \"gb\", \"rf\"]\n",
    "\n",
    "df = pd.DataFrame(columns = column_names)\n",
    "df.loc['auc'] = [auc['SVM_linear'].mean(), auc['SVM_rbf'].mean(), auc['GB'].mean(), auc['RF'].mean()]\n",
    "df.loc['sen'] = [sen['SVM_linear'].mean(), sen['SVM_rbf'].mean(), sen['GB'].mean(), sen['RF'].mean()]\n",
    "df.loc['spe'] = [spe['SVM_linear'].mean(), spe['SVM_rbf'].mean(), spe['GB'].mean(), spe['RF'].mean()]\n",
    "\n",
    "\n",
    "df = df.round(decimals=3)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "18c9f416-fcf0-40ec-b0c4-95b46dba339e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svm_linear</th>\n",
       "      <th>svm_rbf</th>\n",
       "      <th>gb</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>auc</th>\n",
       "      <td>0.024</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sen</th>\n",
       "      <td>0.039</td>\n",
       "      <td>0.039</td>\n",
       "      <td>0.026</td>\n",
       "      <td>0.018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spe</th>\n",
       "      <td>0.037</td>\n",
       "      <td>0.036</td>\n",
       "      <td>0.024</td>\n",
       "      <td>0.030</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     svm_linear  svm_rbf     gb     rf\n",
       "auc       0.024    0.023  0.013  0.011\n",
       "sen       0.039    0.039  0.026  0.018\n",
       "spe       0.037    0.036  0.024  0.030"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc['auc'] = [auc['SVM_linear'].std(), auc['SVM_rbf'].std(), auc['GB'].std(), auc['RF'].std()]\n",
    "df.loc['sen'] = [sen['SVM_linear'].std(), sen['SVM_rbf'].std(), sen['GB'].std(), sen['RF'].std()]\n",
    "df.loc['spe'] = [spe['SVM_linear'].std(), spe['SVM_rbf'].std(), spe['GB'].std(), spe['RF'].std()]\n",
    "\n",
    "\n",
    "df = df.round(decimals=3)\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}