predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "208beb4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, precision_recall_curve\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, precision_recall_curve\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "042310bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting xgboost\n",
      "  Downloading xgboost-2.1.0-py3-none-win_amd64.whl (124.9 MB)\n",
      "     -------------------------------------- 124.9/124.9 MB 6.5 MB/s eta 0:00:00\n",
      "Requirement already satisfied: numpy in c:\\users\\idriss\\anaconda3\\lib\\site-packages (from xgboost) (1.23.5)\n",
      "Requirement already satisfied: scipy in c:\\users\\idriss\\anaconda3\\lib\\site-packages (from xgboost) (1.10.0)\n",
      "Installing collected packages: xgboost\n",
      "Successfully installed xgboost-2.1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -rotobuf (c:\\users\\idriss\\anaconda3\\lib\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "!pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "01d71e4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>acq_date</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>is_holiday</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>day_of_year</th>\n",
       "      <th>is_weekend</th>\n",
       "      <th>NDVI</th>\n",
       "      <th>SoilMoisture</th>\n",
       "      <th>...</th>\n",
       "      <th>wind_gust_quarterly_mean</th>\n",
       "      <th>dew_point_quarterly_mean</th>\n",
       "      <th>average_temperature_yearly_mean</th>\n",
       "      <th>maximum_temperature_yearly_mean</th>\n",
       "      <th>minimum_temperature_yearly_mean</th>\n",
       "      <th>precipitation_yearly_mean</th>\n",
       "      <th>snow_depth_yearly_mean</th>\n",
       "      <th>wind_gust_yearly_mean</th>\n",
       "      <th>dew_point_yearly_mean</th>\n",
       "      <th>is_fire</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>710465</th>\n",
       "      <td>710465</td>\n",
       "      <td>2015-01-23</td>\n",
       "      <td>34.591408</td>\n",
       "      <td>-2.343653</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2071.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>...</td>\n",
       "      <td>768.53480</td>\n",
       "      <td>48.344566</td>\n",
       "      <td>65.561370</td>\n",
       "      <td>78.095894</td>\n",
       "      <td>54.025204</td>\n",
       "      <td>0.306986</td>\n",
       "      <td>999.9</td>\n",
       "      <td>877.81647</td>\n",
       "      <td>49.801643</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>628007</th>\n",
       "      <td>628007</td>\n",
       "      <td>2018-03-12</td>\n",
       "      <td>33.911026</td>\n",
       "      <td>-5.495805</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6013.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>...</td>\n",
       "      <td>957.65546</td>\n",
       "      <td>41.272827</td>\n",
       "      <td>66.815480</td>\n",
       "      <td>80.186850</td>\n",
       "      <td>54.799725</td>\n",
       "      <td>1.941233</td>\n",
       "      <td>999.9</td>\n",
       "      <td>949.30770</td>\n",
       "      <td>48.057945</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280977</th>\n",
       "      <td>280977</td>\n",
       "      <td>2022-11-24</td>\n",
       "      <td>31.678123</td>\n",
       "      <td>-4.733578</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>328.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1382.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>...</td>\n",
       "      <td>695.23914</td>\n",
       "      <td>34.810870</td>\n",
       "      <td>71.861916</td>\n",
       "      <td>83.816850</td>\n",
       "      <td>59.490410</td>\n",
       "      <td>0.007699</td>\n",
       "      <td>999.9</td>\n",
       "      <td>806.69070</td>\n",
       "      <td>28.587397</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345642</th>\n",
       "      <td>345642</td>\n",
       "      <td>2015-04-02</td>\n",
       "      <td>34.592340</td>\n",
       "      <td>-2.347825</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3008.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>...</td>\n",
       "      <td>817.34220</td>\n",
       "      <td>41.308887</td>\n",
       "      <td>65.561370</td>\n",
       "      <td>78.095894</td>\n",
       "      <td>54.025204</td>\n",
       "      <td>0.306986</td>\n",
       "      <td>999.9</td>\n",
       "      <td>877.81647</td>\n",
       "      <td>49.801643</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138561</th>\n",
       "      <td>138561</td>\n",
       "      <td>2021-03-13</td>\n",
       "      <td>27.534670</td>\n",
       "      <td>-9.859253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1066.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>989.29346</td>\n",
       "      <td>50.739132</td>\n",
       "      <td>69.074860</td>\n",
       "      <td>76.311200</td>\n",
       "      <td>60.608470</td>\n",
       "      <td>0.004590</td>\n",
       "      <td>999.9</td>\n",
       "      <td>984.04920</td>\n",
       "      <td>53.070217</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 279 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0    acq_date   latitude  longitude  is_holiday  day_of_week  \\\n",
       "710465      710465  2015-01-23  34.591408  -2.343653         0.0          4.0   \n",
       "628007      628007  2018-03-12  33.911026  -5.495805         0.0          0.0   \n",
       "280977      280977  2022-11-24  31.678123  -4.733578         0.0          3.0   \n",
       "345642      345642  2015-04-02  34.592340  -2.347825         0.0          3.0   \n",
       "138561      138561  2021-03-13  27.534670  -9.859253         0.0          5.0   \n",
       "\n",
       "        day_of_year  is_weekend    NDVI  SoilMoisture  ...  \\\n",
       "710465         23.0         0.0  2071.0          40.0  ...   \n",
       "628007         71.0         0.0  6013.0          49.0  ...   \n",
       "280977        328.0         0.0  1382.0          11.0  ...   \n",
       "345642         92.0         0.0  3008.0          21.0  ...   \n",
       "138561         72.0         1.0  1066.0           4.0  ...   \n",
       "\n",
       "        wind_gust_quarterly_mean  dew_point_quarterly_mean  \\\n",
       "710465                 768.53480                 48.344566   \n",
       "628007                 957.65546                 41.272827   \n",
       "280977                 695.23914                 34.810870   \n",
       "345642                 817.34220                 41.308887   \n",
       "138561                 989.29346                 50.739132   \n",
       "\n",
       "        average_temperature_yearly_mean  maximum_temperature_yearly_mean  \\\n",
       "710465                        65.561370                        78.095894   \n",
       "628007                        66.815480                        80.186850   \n",
       "280977                        71.861916                        83.816850   \n",
       "345642                        65.561370                        78.095894   \n",
       "138561                        69.074860                        76.311200   \n",
       "\n",
       "        minimum_temperature_yearly_mean  precipitation_yearly_mean  \\\n",
       "710465                        54.025204                   0.306986   \n",
       "628007                        54.799725                   1.941233   \n",
       "280977                        59.490410                   0.007699   \n",
       "345642                        54.025204                   0.306986   \n",
       "138561                        60.608470                   0.004590   \n",
       "\n",
       "        snow_depth_yearly_mean  wind_gust_yearly_mean  dew_point_yearly_mean  \\\n",
       "710465                   999.9              877.81647              49.801643   \n",
       "628007                   999.9              949.30770              48.057945   \n",
       "280977                   999.9              806.69070              28.587397   \n",
       "345642                   999.9              877.81647              49.801643   \n",
       "138561                   999.9              984.04920              53.070217   \n",
       "\n",
       "        is_fire  \n",
       "710465      1.0  \n",
       "628007      1.0  \n",
       "280977      0.0  \n",
       "345642      1.0  \n",
       "138561      0.0  \n",
       "\n",
       "[5 rows x 279 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1 = pd.read_csv(\"morocco.csv\")\n",
    "# Sample 100,000 rows\n",
    "df = df_1.sample(n=100000, random_state=42)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d9675101",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_train=df[df.acq_date<'2022-01-01']\n",
    "df_valid=df[df.acq_date>='2022-01-01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3cc3dafa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Idriss\\AppData\\Local\\Temp\\ipykernel_42712\\4275945084.py:7: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  df_balanced = df.groupby(target_column).apply(lambda x: x.sample(min_samples)).reset_index(drop=True)\n",
      "C:\\Users\\Idriss\\AppData\\Local\\Temp\\ipykernel_42712\\4275945084.py:7: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  df_balanced = df.groupby(target_column).apply(lambda x: x.sample(min_samples)).reset_index(drop=True)\n"
     ]
    }
   ],
   "source": [
    "# Balance the training and validation datasets by taking the same number of positive and negative samples\n",
    "def balance_dataset(df, target_column):\n",
    "    # Get the minimum number of samples in each class\n",
    "    min_samples = df[target_column].value_counts().min()\n",
    "    \n",
    "    # Balance the dataset\n",
    "    df_balanced = df.groupby(target_column).apply(lambda x: x.sample(min_samples)).reset_index(drop=True)\n",
    "    \n",
    "    return df_balanced\n",
    "\n",
    "# Balance training and validation datasets\n",
    "df_train_balanced = balance_dataset(df_train, 'is_fire')\n",
    "df_valid_balanced = balance_dataset(df_valid, 'is_fire')\n",
    "\n",
    "# Shuffle the balanced datasets\n",
    "df_train_balanced = df_train_balanced.sample(frac=1).reset_index(drop=True)\n",
    "df_valid_balanced = df_valid_balanced.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "# Remove 'acq_date' column\n",
    "df_train_balanced.drop(columns='acq_date', inplace=True)\n",
    "df_valid_balanced.drop(columns='acq_date', inplace=True)\n",
    "\n",
    "# Convert all values to float32\n",
    "df_train_balanced = df_train_balanced.astype('float32')\n",
    "df_valid_balanced = df_valid_balanced.astype('float32')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "09b764dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the training dataset into features and target\n",
    "X_train = df_train_balanced.drop(columns=['is_fire'])\n",
    "y_train = df_train_balanced['is_fire']\n",
    "\n",
    "# Split the validation dataset into features and target\n",
    "X_valid = df_valid_balanced.drop(columns=['is_fire'])\n",
    "y_valid = df_valid_balanced['is_fire']\n",
    "\n",
    "# Initialize a StandardScaler\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Fit the scaler on the training features and transform both the training and validation features\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f0e1e92e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating Logistic Regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Idriss\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression: Accuracy=0.8373, AUC=0.9344, Precision=0.8092, Recall=0.7172\n",
      "Evaluating Random Forest\n",
      "Random Forest: Accuracy=0.6122, AUC=0.9376, Precision=0.8222, Recall=0.5898\n",
      "                 Model  Accuracy       AUC  Average Precision  Average Recall\n",
      "0  Logistic Regression  0.837261  0.934436           0.809166        0.717165\n",
      "1        Random Forest  0.612175  0.937553           0.822155        0.589758\n"
     ]
    }
   ],
   "source": [
    "models = [\n",
    "    (\"Logistic Regression\", LogisticRegression()),\n",
    "    (\"Random Forest\", RandomForestClassifier()),\n",
    "]\n",
    "\n",
    "# Evaluate each model\n",
    "results = []\n",
    "for name, model in models:\n",
    "    print(f\"Evaluating {name}\")\n",
    "\n",
    "    # Fit the model on the training data\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # Make predictions on the validation data\n",
    "    y_pred = model.predict(X_valid)\n",
    "\n",
    "    # Calculate accuracy and AUC\n",
    "    accuracy = accuracy_score(y_valid, y_pred)\n",
    "    y_proba = model.predict_proba(X_valid)\n",
    "    auc = roc_auc_score(y_valid, y_proba[:, 1])\n",
    "\n",
    "    # Calculate precision-recall curve\n",
    "    precisions, recalls, _ = precision_recall_curve(y_valid, y_proba[:, 1])\n",
    "\n",
    "    # Store the results\n",
    "    results.append((name, accuracy, auc, precisions.mean(), recalls.mean()))\n",
    "\n",
    "    # Print the results for this model\n",
    "    print(f\"{name}: Accuracy={accuracy:.4f}, AUC={auc:.4f}, Precision={precisions.mean():.4f}, Recall={recalls.mean():.4f}\")\n",
    "\n",
    "# Create a DataFrame to display the results\n",
    "results_df = pd.DataFrame(results, columns=['Model', 'Accuracy', 'AUC', 'Average Precision', 'Average Recall'])\n",
    "print(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8e1500a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "xgb_model = xgb.XGBClassifier(n_jobs=1)\n",
    "xgb_model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the validation data\n",
    "y_pred_xgb = xgb_model.predict(X_valid)\n",
    "y_proba_xgb = xgb_model.predict_proba(X_valid)\n",
    "\n",
    "# Calculate accuracy and AUC\n",
    "accuracy_xgb = accuracy_score(y_valid, y_pred_xgb)\n",
    "auc_xgb = roc_auc_score(y_valid, y_proba_xgb[:, 1])\n",
    "\n",
    "# Calculate precision-recall curve\n",
    "precisions_xgb, recalls_xgb, _ = precision_recall_curve(y_valid, y_proba_xgb[:, 1])\n",
    "\n",
    "# Create a new DataFrame with the results\n",
    "new_results = pd.DataFrame([{\n",
    "    \"Model\": \"XGBoost\",\n",
    "    \"Accuracy\": accuracy_xgb,\n",
    "    \"AUC\": auc_xgb,\n",
    "    \"Average Precision\": precisions_xgb.mean(),\n",
    "    \"Average Recall\": recalls_xgb.mean()\n",
    "}])\n",
    "\n",
    "# Concatenate the new results to the existing results_df\n",
    "results_df = pd.concat([results_df, new_results], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e942dffd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Model  Accuracy       AUC  Average Precision  Average Recall\n",
      "0  Logistic Regression  0.837261  0.934436           0.809166        0.717165\n",
      "1        Random Forest  0.612175  0.937553           0.822155        0.589758\n",
      "2              XGBoost  0.749877  0.929739           0.706266        0.885898\n"
     ]
    }
   ],
   "source": [
    "print(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65ee7205",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with 3 layers, 256 units, 0.6 dropout, and relu activation...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Idriss\\AppData\\Local\\Temp\\ipykernel_42712\\3867126322.py:77: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
      "  dl_results = pd.concat([dl_results, current_result], ignore_index=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 3, Units: 256, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8567, AUC-PR: 0.8991, Precision: 0.8088, Recall: 0.9342\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8449, AUC-PR: 0.8186, Precision: 0.8037, Recall: 0.9126\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8355, AUC-PR: 0.8985, Precision: 0.7757, Recall: 0.9440\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8886, AUC-PR: 0.9364, Precision: 0.9583, Recall: 0.8125\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8449, AUC-PR: 0.9556, Precision: 0.7873, Recall: 0.9450\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8422, AUC-PR: 0.9512, Precision: 0.7883, Recall: 0.9357\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8567, AUC-PR: 0.9300, Precision: 0.8239, Recall: 0.9072\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8486, AUC-PR: 0.8876, Precision: 0.8082, Recall: 0.9141\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8694, AUC-PR: 0.9090, Precision: 0.8302, Recall: 0.9288\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8721, AUC-PR: 0.8295, Precision: 0.8586, Recall: 0.8910\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8589, AUC-PR: 0.8714, Precision: 0.8209, Recall: 0.9180\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8314, AUC-PR: 0.9202, Precision: 0.8032, Recall: 0.8778\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9399, Precision: 0.8047, Recall: 0.9283\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8530, AUC-PR: 0.9472, Precision: 0.8070, Recall: 0.9278\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8765, AUC-PR: 0.8807, Precision: 0.8486, Recall: 0.9165\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8478, AUC-PR: 0.9393, Precision: 0.8084, Recall: 0.9116\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8451, AUC-PR: 0.9461, Precision: 0.8007, Recall: 0.9190\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8358, AUC-PR: 0.9397, Precision: 0.7801, Recall: 0.9352\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8731, AUC-PR: 0.8370, Precision: 0.8532, Recall: 0.9013\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8787, AUC-PR: 0.8507, Precision: 0.8614, Recall: 0.9028\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8508, AUC-PR: 0.8753, Precision: 0.8185, Recall: 0.9013\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8552, AUC-PR: 0.9281, Precision: 0.8142, Recall: 0.9205\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8493, AUC-PR: 0.8929, Precision: 0.8166, Recall: 0.9008\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8483, AUC-PR: 0.8589, Precision: 0.8226, Recall: 0.8881\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8486, AUC-PR: 0.9140, Precision: 0.8001, Recall: 0.9293\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8456, AUC-PR: 0.8861, Precision: 0.8118, Recall: 0.8999\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8486, AUC-PR: 0.9228, Precision: 0.7924, Recall: 0.9445\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8490, AUC-PR: 0.9357, Precision: 0.7990, Recall: 0.9327\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8395, AUC-PR: 0.9084, Precision: 0.8206, Recall: 0.8689\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8569, AUC-PR: 0.8938, Precision: 0.8078, Recall: 0.9367\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8557, AUC-PR: 0.9234, Precision: 0.8116, Recall: 0.9264\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8783, AUC-PR: 0.8918, Precision: 0.8510, Recall: 0.9170\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8569, AUC-PR: 0.9293, Precision: 0.8305, Recall: 0.8969\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8530, AUC-PR: 0.8843, Precision: 0.8259, Recall: 0.8945\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8596, AUC-PR: 0.8576, Precision: 0.8280, Recall: 0.9077\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8854, AUC-PR: 0.9271, Precision: 0.9123, Recall: 0.8527\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8571, AUC-PR: 0.9200, Precision: 0.8159, Recall: 0.9224\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.6 dropout, and relu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 3, Units: 64, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8483, AUC-PR: 0.9195, Precision: 0.7955, Recall: 0.9377\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8488, AUC-PR: 0.9269, Precision: 0.7999, Recall: 0.9303\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8370, AUC-PR: 0.9233, Precision: 0.7817, Recall: 0.9352\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8544, AUC-PR: 0.9497, Precision: 0.8091, Recall: 0.9278\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8387, AUC-PR: 0.9416, Precision: 0.7861, Recall: 0.9308\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8790, AUC-PR: 0.9277, Precision: 0.8490, Recall: 0.9219\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8463, AUC-PR: 0.8964, Precision: 0.8115, Recall: 0.9023\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8591, AUC-PR: 0.9394, Precision: 0.8235, Recall: 0.9141\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8522, AUC-PR: 0.9202, Precision: 0.8057, Recall: 0.9283\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8547, AUC-PR: 0.8512, Precision: 0.8195, Recall: 0.9097\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8490, AUC-PR: 0.9418, Precision: 0.8026, Recall: 0.9259\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8454, AUC-PR: 0.9490, Precision: 0.8003, Recall: 0.9205\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8444, AUC-PR: 0.9236, Precision: 0.7997, Recall: 0.9190\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9358, Precision: 0.8021, Recall: 0.9332\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8731, AUC-PR: 0.8014, Precision: 0.8654, Recall: 0.8837\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8424, AUC-PR: 0.9237, Precision: 0.7910, Recall: 0.9308\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8481, AUC-PR: 0.8667, Precision: 0.8004, Recall: 0.9273\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8208, AUC-PR: 0.8502, Precision: 0.7530, Recall: 0.9548\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8643, AUC-PR: 0.9225, Precision: 0.8407, Recall: 0.8989\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8643, AUC-PR: 0.8838, Precision: 0.8243, Recall: 0.9259\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8436, AUC-PR: 0.9462, Precision: 0.7994, Recall: 0.9175\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8628, AUC-PR: 0.9081, Precision: 0.8399, Recall: 0.8964\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8493, AUC-PR: 0.9488, Precision: 0.8117, Recall: 0.9097\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8405, AUC-PR: 0.9082, Precision: 0.8106, Recall: 0.8886\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8522, AUC-PR: 0.9147, Precision: 0.8105, Recall: 0.9195\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8083, AUC-PR: 0.7056, Precision: 0.7335, Recall: 0.9686\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8598, AUC-PR: 0.9457, Precision: 0.8116, Recall: 0.9372\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8542, AUC-PR: 0.9364, Precision: 0.8171, Recall: 0.9126\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8920, AUC-PR: 0.8928, Precision: 0.9419, Recall: 0.8355\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8306, AUC-PR: 0.9573, Precision: 0.7775, Recall: 0.9264\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8709, AUC-PR: 0.8818, Precision: 0.8429, Recall: 0.9116\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8733, AUC-PR: 0.9311, Precision: 0.8418, Recall: 0.9195\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8716, AUC-PR: 0.8909, Precision: 0.8438, Recall: 0.9121\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8621, AUC-PR: 0.8993, Precision: 0.8303, Recall: 0.9102\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8591, AUC-PR: 0.9450, Precision: 0.8190, Recall: 0.9219\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8574, AUC-PR: 0.9202, Precision: 0.8114, Recall: 0.9313\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8770, AUC-PR: 0.8629, Precision: 0.8668, Recall: 0.8910\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.5 dropout, and leakyrelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 3, Units: 256, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8562, AUC-PR: 0.8509, Precision: 0.8235, Recall: 0.9067\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8684, AUC-PR: 0.9350, Precision: 0.8296, Recall: 0.9273\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8692, AUC-PR: 0.9194, Precision: 0.8333, Recall: 0.9229\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8400, AUC-PR: 0.8693, Precision: 0.7837, Recall: 0.9391\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8692, AUC-PR: 0.8300, Precision: 0.8478, Recall: 0.8999\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8424, AUC-PR: 0.9404, Precision: 0.7949, Recall: 0.9229\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8792, AUC-PR: 0.9069, Precision: 0.8463, Recall: 0.9269\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8375, AUC-PR: 0.9530, Precision: 0.7852, Recall: 0.9293\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8648, AUC-PR: 0.8607, Precision: 0.8411, Recall: 0.8994\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8439, AUC-PR: 0.9427, Precision: 0.8023, Recall: 0.9126\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8633, AUC-PR: 0.8663, Precision: 0.8283, Recall: 0.9165\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8689, AUC-PR: 0.9115, Precision: 0.8443, Recall: 0.9048\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8547, AUC-PR: 0.9229, Precision: 0.8137, Recall: 0.9200\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8493, AUC-PR: 0.9405, Precision: 0.7922, Recall: 0.9470\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8667, AUC-PR: 0.8739, Precision: 0.8458, Recall: 0.8969\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8741, AUC-PR: 0.8410, Precision: 0.8765, Recall: 0.8709\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8596, AUC-PR: 0.9206, Precision: 0.8156, Recall: 0.9293\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8567, AUC-PR: 0.8914, Precision: 0.8163, Recall: 0.9205\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8446, AUC-PR: 0.8484, Precision: 0.8052, Recall: 0.9092\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8738, AUC-PR: 0.9051, Precision: 0.8344, Recall: 0.9327\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8454, AUC-PR: 0.8826, Precision: 0.7935, Recall: 0.9337\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8621, AUC-PR: 0.9342, Precision: 0.8200, Recall: 0.9278\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8964, AUC-PR: 0.9446, Precision: 0.9410, Recall: 0.8459\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8400, AUC-PR: 0.9362, Precision: 0.7948, Recall: 0.9165\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8419, AUC-PR: 0.9541, Precision: 0.7898, Recall: 0.9318\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8594, AUC-PR: 0.9136, Precision: 0.8364, Recall: 0.8935\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8591, AUC-PR: 0.8847, Precision: 0.8300, Recall: 0.9033\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8549, AUC-PR: 0.8577, Precision: 0.8301, Recall: 0.8925\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8535, AUC-PR: 0.9517, Precision: 0.8056, Recall: 0.9318\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8709, AUC-PR: 0.9410, Precision: 0.8433, Recall: 0.9111\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8520, AUC-PR: 0.9296, Precision: 0.8000, Recall: 0.9386\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8704, AUC-PR: 0.8763, Precision: 0.8364, Recall: 0.9210\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8665, AUC-PR: 0.9424, Precision: 0.8305, Recall: 0.9210\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8193, AUC-PR: 0.8456, Precision: 0.7601, Recall: 0.9332\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8601, AUC-PR: 0.8880, Precision: 0.8163, Recall: 0.9293\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8581, AUC-PR: 0.9027, Precision: 0.8255, Recall: 0.9082\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8606, AUC-PR: 0.9420, Precision: 0.8200, Recall: 0.9239\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.5 dropout, and softplus activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 4, Units: 16, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8525, AUC-PR: 0.9362, Precision: 0.7994, Recall: 0.9411\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8574, AUC-PR: 0.9201, Precision: 0.8133, Recall: 0.9278\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8758, AUC-PR: 0.9128, Precision: 0.8526, Recall: 0.9087\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8441, AUC-PR: 0.9319, Precision: 0.7873, Recall: 0.9431\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8441, AUC-PR: 0.9496, Precision: 0.7859, Recall: 0.9460\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8495, AUC-PR: 0.9543, Precision: 0.8002, Recall: 0.9318\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8702, AUC-PR: 0.8834, Precision: 0.8412, Recall: 0.9126\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8461, AUC-PR: 0.8482, Precision: 0.8114, Recall: 0.9018\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8346, AUC-PR: 0.9149, Precision: 0.7862, Recall: 0.9190\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8810, AUC-PR: 0.8222, Precision: 0.8589, Recall: 0.9116\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8751, AUC-PR: 0.8939, Precision: 0.9108, Recall: 0.8316\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8682, AUC-PR: 0.9575, Precision: 0.8324, Recall: 0.9219\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8643, AUC-PR: 0.9318, Precision: 0.8254, Recall: 0.9239\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8719, AUC-PR: 0.8739, Precision: 0.8588, Recall: 0.8900\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8483, AUC-PR: 0.8162, Precision: 0.8486, Recall: 0.8478\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8476, AUC-PR: 0.9438, Precision: 0.7933, Recall: 0.9401\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8601, AUC-PR: 0.8940, Precision: 0.8270, Recall: 0.9107\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8758, AUC-PR: 0.8991, Precision: 0.8740, Recall: 0.8783\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8510, AUC-PR: 0.9140, Precision: 0.8027, Recall: 0.9308\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8589, AUC-PR: 0.8810, Precision: 0.8302, Recall: 0.9023\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8503, AUC-PR: 0.8882, Precision: 0.8027, Recall: 0.9288\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8365, AUC-PR: 0.9155, Precision: 0.7899, Recall: 0.9170\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8085, AUC-PR: 0.8140, Precision: 0.7332, Recall: 0.9701\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8787, AUC-PR: 0.9264, Precision: 0.9713, Recall: 0.7806\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8537, AUC-PR: 0.9125, Precision: 0.8059, Recall: 0.9318\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8633, AUC-PR: 0.8017, Precision: 0.8330, Recall: 0.9087\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8429, AUC-PR: 0.9611, Precision: 0.7862, Recall: 0.9421\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8657, AUC-PR: 0.8904, Precision: 0.8329, Recall: 0.9151\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8606, AUC-PR: 0.8788, Precision: 0.8159, Recall: 0.9313\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8785, AUC-PR: 0.8800, Precision: 0.8560, Recall: 0.9102\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8400, AUC-PR: 0.9529, Precision: 0.7814, Recall: 0.9440\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8596, AUC-PR: 0.9546, Precision: 0.8181, Recall: 0.9249\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8679, AUC-PR: 0.8972, Precision: 0.8356, Recall: 0.9161\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8449, AUC-PR: 0.9574, Precision: 0.7943, Recall: 0.9308\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8444, AUC-PR: 0.9116, Precision: 0.8012, Recall: 0.9161\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8441, AUC-PR: 0.8535, Precision: 0.8129, Recall: 0.8940\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8422, AUC-PR: 0.8545, Precision: 0.8251, Recall: 0.8684\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.4 dropout, and leakyrelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 4, Units: 300, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8616, AUC-PR: 0.8888, Precision: 0.8526, Recall: 0.8743\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8586, AUC-PR: 0.9504, Precision: 0.8131, Recall: 0.9313\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8621, AUC-PR: 0.8857, Precision: 0.8451, Recall: 0.8866\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8679, AUC-PR: 0.8935, Precision: 0.8475, Recall: 0.8974\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8611, AUC-PR: 0.8903, Precision: 0.8244, Recall: 0.9175\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8247, AUC-PR: 0.8425, Precision: 0.7673, Recall: 0.9323\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8517, AUC-PR: 0.9119, Precision: 0.8090, Recall: 0.9210\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8756, AUC-PR: 0.8572, Precision: 0.8545, Recall: 0.9053\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8495, AUC-PR: 0.9424, Precision: 0.7999, Recall: 0.9323\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8414, AUC-PR: 0.9191, Precision: 0.7968, Recall: 0.9165\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8704, AUC-PR: 0.8775, Precision: 0.8619, Recall: 0.8822\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8270, AUC-PR: 0.8237, Precision: 0.7651, Recall: 0.9435\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8412, AUC-PR: 0.9391, Precision: 0.7828, Recall: 0.9445\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8716, AUC-PR: 0.9190, Precision: 0.8501, Recall: 0.9023\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8704, AUC-PR: 0.8551, Precision: 0.8501, Recall: 0.8994\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8552, AUC-PR: 0.9385, Precision: 0.8064, Recall: 0.9347\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8621, AUC-PR: 0.8991, Precision: 0.8297, Recall: 0.9111\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8709, AUC-PR: 0.9220, Precision: 0.8448, Recall: 0.9087\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8513, AUC-PR: 0.9234, Precision: 0.7955, Recall: 0.9455\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8616, AUC-PR: 0.9235, Precision: 0.8232, Recall: 0.9210\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8657, AUC-PR: 0.8164, Precision: 0.8393, Recall: 0.9048\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8733, AUC-PR: 0.8680, Precision: 0.8572, Recall: 0.8959\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8503, AUC-PR: 0.9081, Precision: 0.8043, Recall: 0.9259\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8373, AUC-PR: 0.9216, Precision: 0.7818, Recall: 0.9357\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8444, AUC-PR: 0.9395, Precision: 0.7949, Recall: 0.9283\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8419, AUC-PR: 0.9047, Precision: 0.8006, Recall: 0.9107\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8471, AUC-PR: 0.9428, Precision: 0.8090, Recall: 0.9087\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8446, AUC-PR: 0.9360, Precision: 0.7962, Recall: 0.9264\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8594, AUC-PR: 0.9207, Precision: 0.8197, Recall: 0.9215\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8606, AUC-PR: 0.8928, Precision: 0.8307, Recall: 0.9057\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8370, AUC-PR: 0.9393, Precision: 0.7774, Recall: 0.9445\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8623, AUC-PR: 0.9071, Precision: 0.8248, Recall: 0.9200\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8336, AUC-PR: 0.9551, Precision: 0.7772, Recall: 0.9352\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8542, AUC-PR: 0.9256, Precision: 0.8169, Recall: 0.9131\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8682, AUC-PR: 0.8629, Precision: 0.8511, Recall: 0.8925\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8385, AUC-PR: 0.9371, Precision: 0.7855, Recall: 0.9313\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.7712, AUC-PR: 0.8448, Precision: 0.6916, Recall: 0.9789\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.5 dropout, and gelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 6, Units: 16, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8670, AUC-PR: 0.8738, Precision: 0.9343, Recall: 0.7894\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8456, AUC-PR: 0.9317, Precision: 0.7960, Recall: 0.9293\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8495, AUC-PR: 0.9319, Precision: 0.8056, Recall: 0.9215\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8574, AUC-PR: 0.9005, Precision: 0.8187, Recall: 0.9180\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8679, AUC-PR: 0.9030, Precision: 0.8312, Recall: 0.9234\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8537, AUC-PR: 0.9235, Precision: 0.8301, Recall: 0.8895\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9389, Precision: 0.8036, Recall: 0.9303\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8365, AUC-PR: 0.9319, Precision: 0.7857, Recall: 0.9254\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8594, AUC-PR: 0.9498, Precision: 0.8185, Recall: 0.9234\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8822, AUC-PR: 0.8769, Precision: 0.9332, Recall: 0.8233\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8444, AUC-PR: 0.9278, Precision: 0.7927, Recall: 0.9327\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8419, AUC-PR: 0.9446, Precision: 0.7975, Recall: 0.9165\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8500, AUC-PR: 0.9103, Precision: 0.8310, Recall: 0.8787\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8640, AUC-PR: 0.9060, Precision: 0.8366, Recall: 0.9048\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8461, AUC-PR: 0.9269, Precision: 0.8015, Recall: 0.9200\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8719, AUC-PR: 0.8729, Precision: 0.8602, Recall: 0.8881\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8594, AUC-PR: 0.8710, Precision: 0.8194, Recall: 0.9219\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8542, AUC-PR: 0.9315, Precision: 0.8177, Recall: 0.9116\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8468, AUC-PR: 0.9419, Precision: 0.7977, Recall: 0.9293\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8648, AUC-PR: 0.8608, Precision: 0.8329, Recall: 0.9126\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8429, AUC-PR: 0.9193, Precision: 0.8002, Recall: 0.9141\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8549, AUC-PR: 0.9195, Precision: 0.8146, Recall: 0.9190\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8709, AUC-PR: 0.8872, Precision: 0.8392, Recall: 0.9175\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8527, AUC-PR: 0.8686, Precision: 0.8136, Recall: 0.9151\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9550, Precision: 0.8026, Recall: 0.9323\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8510, AUC-PR: 0.9209, Precision: 0.8007, Recall: 0.9347\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8567, AUC-PR: 0.9276, Precision: 0.8103, Recall: 0.9313\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8630, AUC-PR: 0.8925, Precision: 0.8360, Recall: 0.9033\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8301, AUC-PR: 0.9410, Precision: 0.7689, Recall: 0.9440\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.7609, AUC-PR: 0.9013, Precision: 0.7744, Recall: 0.7364\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8233, AUC-PR: 0.9143, Precision: 0.7612, Recall: 0.9421\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8613, AUC-PR: 0.8880, Precision: 0.8265, Recall: 0.9146\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8466, AUC-PR: 0.9331, Precision: 0.7858, Recall: 0.9529\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8554, AUC-PR: 0.9143, Precision: 0.8224, Recall: 0.9067\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9102, Precision: 0.8196, Recall: 0.9013\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8463, AUC-PR: 0.9459, Precision: 0.7971, Recall: 0.9293\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8773, AUC-PR: 0.8566, Precision: 0.8445, Recall: 0.9249\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.5 dropout, and softplus activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 5, Units: 64, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8385, AUC-PR: 0.9484, Precision: 0.7822, Recall: 0.9381\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8400, AUC-PR: 0.9241, Precision: 0.7887, Recall: 0.9288\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8441, AUC-PR: 0.9547, Precision: 0.7928, Recall: 0.9318\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8402, AUC-PR: 0.9583, Precision: 0.7885, Recall: 0.9298\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8564, AUC-PR: 0.9357, Precision: 0.8157, Recall: 0.9210\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8736, AUC-PR: 0.8316, Precision: 0.8566, Recall: 0.8974\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8351, AUC-PR: 0.9212, Precision: 0.7901, Recall: 0.9126\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8643, AUC-PR: 0.9428, Precision: 0.8215, Recall: 0.9308\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8724, AUC-PR: 0.8798, Precision: 0.8382, Recall: 0.9229\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8584, AUC-PR: 0.8642, Precision: 0.8352, Recall: 0.8930\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8706, AUC-PR: 0.8021, Precision: 0.8571, Recall: 0.8895\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8628, AUC-PR: 0.8561, Precision: 0.8326, Recall: 0.9082\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8495, AUC-PR: 0.9179, Precision: 0.8056, Recall: 0.9215\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8628, AUC-PR: 0.9419, Precision: 0.8207, Recall: 0.9283\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8628, AUC-PR: 0.9323, Precision: 0.8213, Recall: 0.9273\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8459, AUC-PR: 0.9404, Precision: 0.7966, Recall: 0.9288\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8640, AUC-PR: 0.8959, Precision: 0.8457, Recall: 0.8905\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8147, AUC-PR: 0.8277, Precision: 0.7448, Recall: 0.9573\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8694, AUC-PR: 0.8905, Precision: 0.8319, Recall: 0.9259\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8638, AUC-PR: 0.9118, Precision: 0.8335, Recall: 0.9092\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8949, AUC-PR: 0.9325, Precision: 0.9507, Recall: 0.8331\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8429, AUC-PR: 0.9553, Precision: 0.7916, Recall: 0.9308\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8338, AUC-PR: 0.8852, Precision: 0.7766, Recall: 0.9372\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8468, AUC-PR: 0.8903, Precision: 0.8150, Recall: 0.8974\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8719, AUC-PR: 0.8988, Precision: 0.8392, Recall: 0.9200\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8211, AUC-PR: 0.8165, Precision: 0.7601, Recall: 0.9381\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8540, AUC-PR: 0.8873, Precision: 0.8063, Recall: 0.9318\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8603, AUC-PR: 0.8752, Precision: 0.8370, Recall: 0.8949\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8100, AUC-PR: 0.8591, Precision: 0.7487, Recall: 0.9332\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8765, AUC-PR: 0.8878, Precision: 0.8564, Recall: 0.9048\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8569, AUC-PR: 0.9553, Precision: 0.8164, Recall: 0.9210\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8301, AUC-PR: 0.9040, Precision: 0.8081, Recall: 0.8660\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8405, AUC-PR: 0.9167, Precision: 0.7846, Recall: 0.9386\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8395, AUC-PR: 0.8741, Precision: 0.7859, Recall: 0.9332\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8679, AUC-PR: 0.9244, Precision: 0.8289, Recall: 0.9273\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8554, AUC-PR: 0.9209, Precision: 0.8252, Recall: 0.9018\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8378, AUC-PR: 0.9279, Precision: 0.7860, Recall: 0.9283\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.4 dropout, and leakyrelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 3, Units: 64, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8481, AUC-PR: 0.9265, Precision: 0.8056, Recall: 0.9175\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8822, AUC-PR: 0.8627, Precision: 0.8811, Recall: 0.8837\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8657, AUC-PR: 0.9436, Precision: 0.8256, Recall: 0.9273\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8667, AUC-PR: 0.8794, Precision: 0.8474, Recall: 0.8945\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8702, AUC-PR: 0.9077, Precision: 0.8409, Recall: 0.9131\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8704, AUC-PR: 0.8700, Precision: 0.8308, Recall: 0.9303\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8702, AUC-PR: 0.8677, Precision: 0.8327, Recall: 0.9264\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8829, AUC-PR: 0.8962, Precision: 0.9488, Recall: 0.8095\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8385, AUC-PR: 0.9261, Precision: 0.7874, Recall: 0.9273\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8419, AUC-PR: 0.9262, Precision: 0.7875, Recall: 0.9367\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8540, AUC-PR: 0.8757, Precision: 0.8182, Recall: 0.9102\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8473, AUC-PR: 0.8941, Precision: 0.8286, Recall: 0.8758\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8657, AUC-PR: 0.9323, Precision: 0.8291, Recall: 0.9215\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8562, AUC-PR: 0.9288, Precision: 0.8148, Recall: 0.9219\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8689, AUC-PR: 0.8710, Precision: 0.8424, Recall: 0.9077\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8611, AUC-PR: 0.9380, Precision: 0.8158, Recall: 0.9327\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8611, AUC-PR: 0.9475, Precision: 0.8169, Recall: 0.9308\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8569, AUC-PR: 0.9177, Precision: 0.8314, Recall: 0.8954\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8351, AUC-PR: 0.9182, Precision: 0.7807, Recall: 0.9318\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8370, AUC-PR: 0.9172, Precision: 0.7874, Recall: 0.9234\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8498, AUC-PR: 0.9433, Precision: 0.8025, Recall: 0.9278\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8795, AUC-PR: 0.8722, Precision: 0.8514, Recall: 0.9195\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8719, AUC-PR: 0.8605, Precision: 0.8350, Recall: 0.9269\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8881, AUC-PR: 0.9078, Precision: 0.9081, Recall: 0.8635\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8621, AUC-PR: 0.8543, Precision: 0.8357, Recall: 0.9013\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8748, AUC-PR: 0.8239, Precision: 0.8697, Recall: 0.8817\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.6983, AUC-PR: 0.8272, Precision: 0.6267, Recall: 0.9809\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8594, AUC-PR: 0.8809, Precision: 0.8150, Recall: 0.9298\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8468, AUC-PR: 0.9279, Precision: 0.7930, Recall: 0.9386\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8400, AUC-PR: 0.9485, Precision: 0.7860, Recall: 0.9342\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8657, AUC-PR: 0.8574, Precision: 0.8234, Recall: 0.9313\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8567, AUC-PR: 0.9533, Precision: 0.8177, Recall: 0.9180\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8513, AUC-PR: 0.8984, Precision: 0.8131, Recall: 0.9121\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8650, AUC-PR: 0.9313, Precision: 0.8198, Recall: 0.9357\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8360, AUC-PR: 0.9225, Precision: 0.7816, Recall: 0.9327\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.5047, AUC-PR: 0.4973, Precision: 0.5023, Recall: 1.0000\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8756, AUC-PR: 0.9224, Precision: 0.8487, Recall: 0.9141\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.3 dropout, and leakyrelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 3, Units: 32, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8601, AUC-PR: 0.9055, Precision: 0.8219, Recall: 0.9195\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8459, AUC-PR: 0.9129, Precision: 0.7922, Recall: 0.9377\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8441, AUC-PR: 0.9401, Precision: 0.8027, Recall: 0.9126\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8382, AUC-PR: 0.9134, Precision: 0.7927, Recall: 0.9161\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8424, AUC-PR: 0.9456, Precision: 0.7846, Recall: 0.9440\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8743, AUC-PR: 0.9164, Precision: 0.8412, Recall: 0.9229\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8461, AUC-PR: 0.9224, Precision: 0.8106, Recall: 0.9033\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8672, AUC-PR: 0.8604, Precision: 0.8298, Recall: 0.9239\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8314, AUC-PR: 0.9433, Precision: 0.7726, Recall: 0.9391\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8505, AUC-PR: 0.8881, Precision: 0.8010, Recall: 0.9327\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8621, AUC-PR: 0.8907, Precision: 0.8200, Recall: 0.9278\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8468, AUC-PR: 0.9039, Precision: 0.8097, Recall: 0.9067\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8419, AUC-PR: 0.9352, Precision: 0.7975, Recall: 0.9165\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8471, AUC-PR: 0.9347, Precision: 0.7914, Recall: 0.9426\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8648, AUC-PR: 0.9422, Precision: 0.8208, Recall: 0.9332\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8414, AUC-PR: 0.9587, Precision: 0.7822, Recall: 0.9465\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8444, AUC-PR: 0.9488, Precision: 0.7924, Recall: 0.9332\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8748, AUC-PR: 0.8619, Precision: 0.8627, Recall: 0.8915\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8471, AUC-PR: 0.9174, Precision: 0.8001, Recall: 0.9254\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8471, AUC-PR: 0.9110, Precision: 0.8137, Recall: 0.9003\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8640, AUC-PR: 0.9270, Precision: 0.8192, Recall: 0.9342\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8436, AUC-PR: 0.9446, Precision: 0.7778, Recall: 0.9622\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8503, AUC-PR: 0.8756, Precision: 0.8201, Recall: 0.8974\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8542, AUC-PR: 0.8831, Precision: 0.8114, Recall: 0.9229\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8535, AUC-PR: 0.9343, Precision: 0.8090, Recall: 0.9254\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8471, AUC-PR: 0.9317, Precision: 0.8069, Recall: 0.9126\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8633, AUC-PR: 0.8445, Precision: 0.8301, Recall: 0.9136\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8459, AUC-PR: 0.8778, Precision: 0.7959, Recall: 0.9303\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8419, AUC-PR: 0.8904, Precision: 0.8032, Recall: 0.9057\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8409, AUC-PR: 0.9291, Precision: 0.7862, Recall: 0.9367\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8510, AUC-PR: 0.9098, Precision: 0.8139, Recall: 0.9102\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8571, AUC-PR: 0.8551, Precision: 0.8134, Recall: 0.9269\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8375, AUC-PR: 0.9044, Precision: 0.7756, Recall: 0.9499\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8328, AUC-PR: 0.8995, Precision: 0.7703, Recall: 0.9485\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8397, AUC-PR: 0.9363, Precision: 0.7932, Recall: 0.9190\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8483, AUC-PR: 0.9290, Precision: 0.8218, Recall: 0.8895\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8677, AUC-PR: 0.9381, Precision: 0.8383, Recall: 0.9111\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.3 dropout, and leakyrelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 6, Units: 64, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8677, AUC-PR: 0.9175, Precision: 0.8182, Recall: 0.9455\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8461, AUC-PR: 0.9345, Precision: 0.8010, Recall: 0.9210\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8397, AUC-PR: 0.9469, Precision: 0.7820, Recall: 0.9421\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8427, AUC-PR: 0.9480, Precision: 0.7918, Recall: 0.9298\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8439, AUC-PR: 0.9351, Precision: 0.7872, Recall: 0.9426\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8473, AUC-PR: 0.9278, Precision: 0.7997, Recall: 0.9269\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8596, AUC-PR: 0.8616, Precision: 0.8126, Recall: 0.9347\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8530, AUC-PR: 0.9282, Precision: 0.8054, Recall: 0.9308\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8503, AUC-PR: 0.9219, Precision: 0.8002, Recall: 0.9337\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8493, AUC-PR: 0.8976, Precision: 0.8133, Recall: 0.9067\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8206, AUC-PR: 0.8040, Precision: 0.7498, Recall: 0.9622\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8387, AUC-PR: 0.8092, Precision: 0.7926, Recall: 0.9175\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8144, AUC-PR: 0.8363, Precision: 0.7425, Recall: 0.9627\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8665, AUC-PR: 0.8841, Precision: 0.8461, Recall: 0.8959\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8581, AUC-PR: 0.8934, Precision: 0.8311, Recall: 0.8989\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9140, Precision: 0.8309, Recall: 0.8827\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8665, AUC-PR: 0.8720, Precision: 0.8407, Recall: 0.9043\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8471, AUC-PR: 0.8741, Precision: 0.8128, Recall: 0.9018\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8692, AUC-PR: 0.8842, Precision: 0.8281, Recall: 0.9318\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8459, AUC-PR: 0.9151, Precision: 0.7922, Recall: 0.9377\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8667, AUC-PR: 0.8869, Precision: 0.8362, Recall: 0.9121\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8778, AUC-PR: 0.8725, Precision: 0.8597, Recall: 0.9028\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8554, AUC-PR: 0.9265, Precision: 0.8118, Recall: 0.9254\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8238, AUC-PR: 0.9032, Precision: 0.7503, Recall: 0.9705\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8449, AUC-PR: 0.9407, Precision: 0.7973, Recall: 0.9249\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8451, AUC-PR: 0.9095, Precision: 0.8002, Recall: 0.9200\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8451, AUC-PR: 0.9341, Precision: 0.7999, Recall: 0.9205\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8819, AUC-PR: 0.8554, Precision: 0.8708, Recall: 0.8969\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8633, AUC-PR: 0.8946, Precision: 0.8487, Recall: 0.8841\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8589, AUC-PR: 0.9164, Precision: 0.8127, Recall: 0.9327\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8736, AUC-PR: 0.9351, Precision: 0.8422, Recall: 0.9195\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8515, AUC-PR: 0.8459, Precision: 0.8097, Recall: 0.9190\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8051, AUC-PR: 0.8318, Precision: 0.7987, Recall: 0.8159\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8657, AUC-PR: 0.8941, Precision: 0.8350, Recall: 0.9116\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8601, AUC-PR: 0.9395, Precision: 0.8204, Recall: 0.9219\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8625, AUC-PR: 0.9031, Precision: 0.8328, Recall: 0.9072\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8382, AUC-PR: 0.8682, Precision: 0.7821, Recall: 0.9377\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.0 dropout, and leakyrelu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 6, Units: 128, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8648, AUC-PR: 0.9508, Precision: 0.8341, Recall: 0.9107\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8432, AUC-PR: 0.9337, Precision: 0.7862, Recall: 0.9426\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8368, AUC-PR: 0.9122, Precision: 0.7782, Recall: 0.9421\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8378, AUC-PR: 0.9372, Precision: 0.7713, Recall: 0.9602\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8466, AUC-PR: 0.8954, Precision: 0.7939, Recall: 0.9362\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8687, AUC-PR: 0.9393, Precision: 0.8305, Recall: 0.9264\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8751, AUC-PR: 0.9365, Precision: 0.8492, Recall: 0.9121\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8417, AUC-PR: 0.9441, Precision: 0.7883, Recall: 0.9342\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8652, AUC-PR: 0.8952, Precision: 0.8403, Recall: 0.9018\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8451, AUC-PR: 0.8885, Precision: 0.8161, Recall: 0.8910\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8414, AUC-PR: 0.8998, Precision: 0.7958, Recall: 0.9185\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8675, AUC-PR: 0.9233, Precision: 0.8334, Recall: 0.9185\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8584, AUC-PR: 0.8616, Precision: 0.8621, Recall: 0.8532\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8748, AUC-PR: 0.8792, Precision: 0.8644, Recall: 0.8891\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.6 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.6, Activation: relu\n",
      "Accuracy: 0.8586, AUC-PR: 0.9224, Precision: 0.8105, Recall: 0.9362\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8601, AUC-PR: 0.9011, Precision: 0.8174, Recall: 0.9273\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.7587, AUC-PR: 0.7887, Precision: 0.6795, Recall: 0.9794\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8510, AUC-PR: 0.8970, Precision: 0.7982, Recall: 0.9396\n",
      "\n",
      "Training model with 4 layers, 16 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 16, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8667, AUC-PR: 0.9030, Precision: 0.9029, Recall: 0.8218\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8571, AUC-PR: 0.9119, Precision: 0.8134, Recall: 0.9269\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.4 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.4, Activation: leakyrelu\n",
      "Accuracy: 0.8601, AUC-PR: 0.9233, Precision: 0.8233, Recall: 0.9170\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8380, AUC-PR: 0.9097, Precision: 0.7931, Recall: 0.9146\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8436, AUC-PR: 0.8774, Precision: 0.7969, Recall: 0.9224\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8520, AUC-PR: 0.9450, Precision: 0.8028, Recall: 0.9332\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8660, AUC-PR: 0.8937, Precision: 0.8306, Recall: 0.9195\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8441, AUC-PR: 0.9484, Precision: 0.8022, Recall: 0.9136\n",
      "\n",
      "Training model with 6 layers, 16 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 16, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8297, AUC-PR: 0.8833, Precision: 0.7702, Recall: 0.9396\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8557, AUC-PR: 0.9065, Precision: 0.8224, Recall: 0.9072\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8800, AUC-PR: 0.8463, Precision: 0.8455, Recall: 0.9298\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8439, AUC-PR: 0.9470, Precision: 0.7922, Recall: 0.9323\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8422, AUC-PR: 0.9587, Precision: 0.7907, Recall: 0.9308\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8603, AUC-PR: 0.9074, Precision: 0.8137, Recall: 0.9347\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8414, AUC-PR: 0.9150, Precision: 0.7943, Recall: 0.9215\n",
      "\n",
      "Training model with 3 layers, 32 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 32, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8520, AUC-PR: 0.9288, Precision: 0.8109, Recall: 0.9180\n",
      "\n",
      "Training model with 4 layers, 256 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 256, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8613, AUC-PR: 0.8863, Precision: 0.8295, Recall: 0.9097\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8866, AUC-PR: 0.9200, Precision: 0.9482, Recall: 0.8179\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8530, AUC-PR: 0.8692, Precision: 0.8021, Recall: 0.9372\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.0 dropout, and relu activation...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for model - Layers: 6, Units: 64, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8797, AUC-PR: 0.9027, Precision: 0.8505, Recall: 0.9215\n",
      "\n",
      "Training model with 6 layers, 64 units, 0.5 dropout, and softplus activation...\n",
      "Results for model - Layers: 6, Units: 64, Dropout: 0.5, Activation: softplus\n",
      "Accuracy: 0.8397, AUC-PR: 0.9576, Precision: 0.7829, Recall: 0.9401\n",
      "\n",
      "Training model with 5 layers, 256 units, 0.0 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 256, Dropout: 0.0, Activation: relu\n",
      "Accuracy: 0.8621, AUC-PR: 0.8713, Precision: 0.8653, Recall: 0.8576\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8603, AUC-PR: 0.9346, Precision: 0.8216, Recall: 0.9205\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.5 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.5, Activation: leakyrelu\n",
      "Accuracy: 0.8360, AUC-PR: 0.9139, Precision: 0.7914, Recall: 0.9126\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8643, AUC-PR: 0.8767, Precision: 0.8376, Recall: 0.9038\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.6 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.6, Activation: leakyrelu\n",
      "Accuracy: 0.8547, AUC-PR: 0.9054, Precision: 0.8065, Recall: 0.9332\n",
      "\n",
      "Training model with 5 layers, 128 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 128, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8652, AUC-PR: 0.8862, Precision: 0.8295, Recall: 0.9195\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8493, AUC-PR: 0.9405, Precision: 0.8001, Recall: 0.9313\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 6, Units: 300, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8559, AUC-PR: 0.9306, Precision: 0.8117, Recall: 0.9269\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8461, AUC-PR: 0.8939, Precision: 0.8018, Recall: 0.9195\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8515, AUC-PR: 0.9468, Precision: 0.7966, Recall: 0.9440\n",
      "\n",
      "Training model with 3 layers, 128 units, 0.0 dropout, and softplus activation...\n",
      "Results for model - Layers: 3, Units: 128, Dropout: 0.0, Activation: softplus\n",
      "Accuracy: 0.8589, AUC-PR: 0.8755, Precision: 0.8394, Recall: 0.8876\n",
      "\n",
      "Training model with 4 layers, 300 units, 0.4 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 300, Dropout: 0.4, Activation: softplus\n",
      "Accuracy: 0.8606, AUC-PR: 0.8934, Precision: 0.8159, Recall: 0.9313\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8390, AUC-PR: 0.9150, Precision: 0.7952, Recall: 0.9131\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8662, AUC-PR: 0.9360, Precision: 0.8614, Recall: 0.8729\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.3 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.3, Activation: softplus\n",
      "Accuracy: 0.8503, AUC-PR: 0.8880, Precision: 0.7984, Recall: 0.9372\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8623, AUC-PR: 0.9188, Precision: 0.8159, Recall: 0.9357\n",
      "\n",
      "Training model with 5 layers, 64 units, 0.5 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 64, Dropout: 0.5, Activation: gelu\n",
      "Accuracy: 0.8412, AUC-PR: 0.9557, Precision: 0.7872, Recall: 0.9352\n",
      "\n",
      "Training model with 5 layers, 300 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 300, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8412, AUC-PR: 0.8487, Precision: 0.7980, Recall: 0.9136\n",
      "\n",
      "Training model with 4 layers, 128 units, 0.0 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 4, Units: 128, Dropout: 0.0, Activation: leakyrelu\n",
      "Accuracy: 0.8520, AUC-PR: 0.8808, Precision: 0.8104, Recall: 0.9190\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8417, AUC-PR: 0.9203, Precision: 0.8127, Recall: 0.8881\n",
      "\n",
      "Training model with 6 layers, 32 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 32, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8508, AUC-PR: 0.9458, Precision: 0.8001, Recall: 0.9352\n",
      "\n",
      "Training model with 5 layers, 16 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 5, Units: 16, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8439, AUC-PR: 0.9036, Precision: 0.7886, Recall: 0.9396\n",
      "\n",
      "Training model with 4 layers, 32 units, 0.6 dropout, and softplus activation...\n",
      "Results for model - Layers: 4, Units: 32, Dropout: 0.6, Activation: softplus\n",
      "Accuracy: 0.8417, AUC-PR: 0.9185, Precision: 0.7957, Recall: 0.9195\n",
      "\n",
      "Training model with 4 layers, 64 units, 0.3 dropout, and gelu activation...\n",
      "Results for model - Layers: 4, Units: 64, Dropout: 0.3, Activation: gelu\n",
      "Accuracy: 0.8630, AUC-PR: 0.8927, Precision: 0.8312, Recall: 0.9111\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.4 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.4, Activation: relu\n",
      "Accuracy: 0.8662, AUC-PR: 0.8557, Precision: 0.8373, Recall: 0.9092\n",
      "\n",
      "Training model with 3 layers, 16 units, 0.5 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 16, Dropout: 0.5, Activation: relu\n",
      "Accuracy: 0.8623, AUC-PR: 0.9278, Precision: 0.8452, Recall: 0.8871\n",
      "\n",
      "Training model with 5 layers, 32 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 5, Units: 32, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8584, AUC-PR: 0.9091, Precision: 0.8125, Recall: 0.9318\n",
      "\n",
      "Training model with 6 layers, 256 units, 0.6 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 256, Dropout: 0.6, Activation: gelu\n",
      "Accuracy: 0.8544, AUC-PR: 0.8854, Precision: 0.8096, Recall: 0.9269\n",
      "\n",
      "Training model with 6 layers, 128 units, 0.4 dropout, and gelu activation...\n",
      "Results for model - Layers: 6, Units: 128, Dropout: 0.4, Activation: gelu\n",
      "Accuracy: 0.8424, AUC-PR: 0.8615, Precision: 0.8096, Recall: 0.8954\n",
      "\n",
      "Training model with 3 layers, 64 units, 0.0 dropout, and gelu activation...\n",
      "Results for model - Layers: 3, Units: 64, Dropout: 0.0, Activation: gelu\n",
      "Accuracy: 0.8324, AUC-PR: 0.7705, Precision: 0.8261, Recall: 0.8419\n",
      "\n",
      "Training model with 3 layers, 256 units, 0.3 dropout, and relu activation...\n",
      "Results for model - Layers: 3, Units: 256, Dropout: 0.3, Activation: relu\n",
      "Accuracy: 0.8520, AUC-PR: 0.9274, Precision: 0.8107, Recall: 0.9185\n",
      "\n",
      "Training model with 3 layers, 300 units, 0.3 dropout, and leakyrelu activation...\n",
      "Results for model - Layers: 3, Units: 300, Dropout: 0.3, Activation: leakyrelu\n",
      "Accuracy: 0.8760, AUC-PR: 0.8706, Precision: 0.8583, Recall: 0.9008\n",
      "\n",
      "Training model with 6 layers, 300 units, 0.5 dropout, and leakyrelu activation...\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from itertools import product\n",
    "from random import shuffle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout, Input, Activation, LeakyReLU\n",
    "from tensorflow.keras.metrics import AUC, Precision, Recall\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "import tensorflow as tf\n",
    "\n",
    "# Assuming X_train_scaled, y_train, X_valid_scaled, y_valid are already defined\n",
    "\n",
    "# Define the range of configurations\n",
    "layers_range = range(3, 7)\n",
    "units_range = [16, 32, 64, 128, 256, 300]\n",
    "dropout_range = [0.6, 0.5, 0.4, 0.3, 0.0]\n",
    "activation_functions = ['relu', 'gelu', 'softplus', 'leakyrelu']\n",
    "\n",
    "# Generate all combinations\n",
    "configurations = list(product(layers_range, units_range, dropout_range, activation_functions))\n",
    "\n",
    "# Shuffle the configurations\n",
    "shuffle(configurations)\n",
    "\n",
    "# Initialize a DataFrame to store the results\n",
    "dl_results = pd.DataFrame(columns=['layers', 'units', 'dropout', 'activation_function', 'accuracy', 'auc_pr', 'precision', 'recall'])\n",
    "\n",
    "# Iterate through each configuration and train the model\n",
    "for layers, units, dropout, activation in configurations:\n",
    "    print(f\"Training model with {layers} layers, {units} units, {dropout} dropout, and {activation} activation...\")\n",
    "\n",
    "    # Build the model\n",
    "    model = Sequential()\n",
    "    model.add(Input(shape=(X_train_scaled.shape[1],)))\n",
    "\n",
    "    # Add the first layer separately to include input_shape\n",
    "    model.add(Dense(units))\n",
    "    if activation == 'leakyrelu':\n",
    "        model.add(LeakyReLU(alpha=0.01))\n",
    "    else:\n",
    "        model.add(Activation(activation))\n",
    "    model.add(Dropout(dropout))\n",
    "\n",
    "    # Add additional layers\n",
    "    for _ in range(1, layers):\n",
    "        model.add(Dense(units))\n",
    "        if activation == 'leakyrelu':\n",
    "            model.add(LeakyReLU(alpha=0.01))\n",
    "        else:\n",
    "            model.add(Activation(activation))\n",
    "        model.add(Dropout(dropout))\n",
    "\n",
    "    # Add the output layer\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', AUC(curve='PR', name='auc_pr'), Precision(name='precision'), Recall(name='recall')])\n",
    "\n",
    "    # Define the checkpoint callback to save the best model weights only\n",
    "    temp_checkpoint_path = f\"temp_model_layers{layers}_units{units}_dropout{dropout}_{activation}.h5\"\n",
    "    checkpoint = ModelCheckpoint(temp_checkpoint_path, save_best_only=True, save_weights_only=True, monitor='val_accuracy', mode='max')\n",
    "\n",
    "    # Train the model\n",
    "    history = model.fit(X_train_scaled, y_train, epochs=20, batch_size=512, validation_data=(X_valid_scaled, y_valid), callbacks=[checkpoint], verbose=0)\n",
    "\n",
    "    # Extract the best metrics from the training history\n",
    "    best_index = np.argmax(history.history['val_accuracy'])\n",
    "    best_accuracy = history.history['val_accuracy'][best_index]\n",
    "    best_auc_pr = history.history['val_auc_pr'][best_index]\n",
    "    best_precision = history.history['val_precision'][best_index]\n",
    "    best_recall = history.history['val_recall'][best_index]\n",
    "\n",
    "    # Store the results in the DataFrame\n",
    "    current_result = pd.DataFrame([[layers, units, dropout, activation, best_accuracy, best_auc_pr, best_precision, best_recall]],\n",
    "                                  columns=['layers', 'units', 'dropout', 'activation_function', 'accuracy', 'auc_pr', 'precision', 'recall'])\n",
    "    dl_results = pd.concat([dl_results, current_result], ignore_index=True)\n",
    "\n",
    "    # Print the current model's result\n",
    "    print(f\"Results for model - Layers: {layers}, Units: {units}, Dropout: {dropout}, Activation: {activation}\")\n",
    "    print(f\"Accuracy: {best_accuracy:.4f}, AUC-PR: {best_auc_pr:.4f}, Precision: {best_precision:.4f}, Recall: {best_recall:.4f}\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "714007af",
   "metadata": {},
   "outputs": [],
   "source": [
    "dl_results_sorted = dl_results.sort_values('accuracy', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd4b6f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = dl_results_sorted.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c140d53",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best model configuration:\")\n",
    "print(f\"Layers: {best_model['layers']}\")\n",
    "print(f\"Units: {best_model['units']}\")\n",
    "print(f\"Dropout: {best_model['dropout']}\")\n",
    "print(f\"Activation Function: {best_model['activation_function']}\")\n",
    "print(f\"Accuracy: {best_model['accuracy']:.4f}\")\n",
    "print(f\"AUC-PR: {best_model['auc_pr']:.4f}\")\n",
    "print(f\"Precision: {best_model['precision']:.4f}\")\n",
    "print(f\"Recall: {best_model['recall']:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81541caa",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_path = f\"temp_model_layers{best_model['layers']}_units{best_model['units']}_dropout{best_model['dropout']}_{best_model['activation_function']}.h5\"\n",
    "\n",
    "# Recreate the best model architecture\n",
    "best_model_architecture = Sequential()\n",
    "best_model_architecture.add(Input(shape=(X_train_scaled.shape[1],)))\n",
    "best_model_architecture.add(Dense(best_model['units']))\n",
    "if best_model['activation_function'] == 'leakyrelu':\n",
    "    best_model_architecture.add(LeakyReLU(alpha=0.01))\n",
    "else:\n",
    "    best_model_architecture.add(Activation(best_model['activation_function']))\n",
    "best_model_architecture.add(Dropout(best_model['dropout']))\n",
    "\n",
    "for _ in range(1, best_model['layers']):\n",
    "    best_model_architecture.add(Dense(best_model['units']))\n",
    "    if best_model['activation_function'] == 'leakyrelu':\n",
    "        best_model_architecture.add(LeakyReLU(alpha=0.01))\n",
    "    else:\n",
    "        best_model_architecture.add(Activation(best_model['activation_function']))\n",
    "    best_model_architecture.add(Dropout(best_model['dropout']))\n",
    "\n",
    "best_model_architecture.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "# Compile the model\n",
    "best_model_architecture.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', AUC(curve='PR', name='auc_pr'), Precision(name='precision'), Recall(name='recall')])\n",
    "\n",
    "# Load the best weights\n",
    "best_model_architecture.load_weights(best_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e54a5161",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluation = best_model_architecture.evaluate(X_valid_scaled, y_valid, verbose=0)\n",
    "print(\"\\nBest model performance on validation set:\")\n",
    "print(f\"Loss: {evaluation[0]:.4f}\")\n",
    "print(f\"Accuracy: {evaluation[1]:.4f}\")\n",
    "print(f\"AUC-PR: {evaluation[2]:.4f}\")\n",
    "print(f\"Precision: {evaluation[3]:.4f}\")\n",
    "print(f\"Recall: {evaluation[4]:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e37136d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from sklearn.metrics import accuracy_score, average_precision_score, precision_score, recall_score\n",
    "\n",
    "# Assuming X_train_scaled, y_train, X_valid_scaled, y_valid are numpy arrays\n",
    "# Convert data to PyTorch tensors\n",
    "X_train = torch.FloatTensor(X_train_scaled)\n",
    "y_train = torch.FloatTensor(y_train)\n",
    "X_valid = torch.FloatTensor(X_valid_scaled)\n",
    "y_valid = torch.FloatTensor(y_valid)\n",
    "\n",
    "# Create DataLoader for batching\n",
    "train_dataset = TensorDataset(X_train, y_train)\n",
    "train_loader = DataLoader(train_dataset, batch_size=512, shuffle=True)\n",
    "\n",
    "# Define the model architecture\n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_layers, dropout, activation):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        self.layers = nn.ModuleList()\n",
    "        self.layers.append(nn.Linear(input_size, hidden_size))\n",
    "        \n",
    "        for _ in range(num_layers - 1):\n",
    "            self.layers.append(nn.Linear(hidden_size, hidden_size))\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            self.activation = nn.ReLU()\n",
    "        elif activation == 'leakyrelu':\n",
    "            self.activation = nn.LeakyReLU(0.01)\n",
    "        elif activation == 'gelu':\n",
    "            self.activation = nn.GELU()\n",
    "        else:  # softplus\n",
    "            self.activation = nn.Softplus()\n",
    "        \n",
    "        self.output = nn.Linear(hidden_size, 1)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = self.dropout(self.activation(layer(x)))\n",
    "        return self.sigmoid(self.output(x))\n",
    "\n",
    "# Set the best hyperparameters\n",
    "best_layers = 5\n",
    "best_units = 128\n",
    "best_dropout = 0.3\n",
    "best_activation = 'relu'\n",
    "\n",
    "# Initialize the model\n",
    "input_size = X_train.shape[1]\n",
    "model = NeuralNet(input_size, best_units, best_layers, best_dropout, best_activation)\n",
    "\n",
    "# Define loss function and optimizer\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "# Training loop\n",
    "num_epochs = 20\n",
    "best_val_accuracy = 0\n",
    "best_model_path = f\"best_model_layers{best_layers}_units{best_units}_dropout{best_dropout}_{best_activation}.pth\"\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    for batch_X, batch_y in train_loader:\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(batch_X).squeeze()\n",
    "        loss = criterion(outputs, batch_y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "    # Validation\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        val_outputs = model(X_valid).squeeze()\n",
    "        val_loss = criterion(val_outputs, y_valid)\n",
    "        val_preds = (val_outputs > 0.5).float()\n",
    "        val_accuracy = accuracy_score(y_valid, val_preds)\n",
    "        val_auc_pr = average_precision_score(y_valid, val_outputs)\n",
    "        val_precision = precision_score(y_valid, val_preds)\n",
    "        val_recall = recall_score(y_valid, val_preds)\n",
    "    \n",
    "    print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}, Val Loss: {val_loss.item():.4f}, Val Accuracy: {val_accuracy:.4f}\")\n",
    "    \n",
    "    if val_accuracy > best_val_accuracy:\n",
    "        best_val_accuracy = val_accuracy\n",
    "        torch.save(model.state_dict(), best_model_path)\n",
    "\n",
    "# Load the best model\n",
    "model.load_state_dict(torch.load(best_model_path))\n",
    "\n",
    "# Final evaluation\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    val_outputs = model(X_valid).squeeze()\n",
    "    val_loss = criterion(val_outputs, y_valid)\n",
    "    val_preds = (val_outputs > 0.5).float()\n",
    "    val_accuracy = accuracy_score(y_valid, val_preds)\n",
    "    val_auc_pr = average_precision_score(y_valid, val_outputs)\n",
    "    val_precision = precision_score(y_valid, val_preds)\n",
    "    val_recall = recall_score(y_valid, val_preds)\n",
    "\n",
    "print(f\"\\nFinal model performance on validation set:\")\n",
    "print(f\"Loss: {val_loss.item():.4f}\")\n",
    "print(f\"Accuracy: {val_accuracy:.4f}\")\n",
    "print(f\"AUC-PR: {val_auc_pr:.4f}\")\n",
    "print(f\"Precision: {val_precision:.4f}\")\n",
    "print(f\"Recall: {val_recall:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e28d8601",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}