collect_trial_stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/georg/Projects/mil/mil_env/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "\n",
    "from make_summary import summarize_group\n",
    "from mil.utils import human_format\n",
    "\n",
    "selected_model_type = \"mnist_collage\"\n",
    "# selected_model_type = \"mnist_collage_ablations\"\n",
    "# selected_model_type = \"camelyon16\"\n",
    "\n",
    "if \"mnist_collage\" in selected_model_type:\n",
    "    datasets = [\"mnist_collage\", \"mnist_collage_inverse\"]\n",
    "else:\n",
    "    datasets = [\"camelyon16\"]\n",
    "yaml_folder = Path(\"conf\") / \"selected_model\" / selected_model_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abmil',\n",
       " 'discrete_rel_pos_self_attention',\n",
       " 'distance_aware_self_attention',\n",
       " 'gnn_gat',\n",
       " 'gnn_gcn',\n",
       " 'induced_set_transformer',\n",
       " 'just_pool',\n",
       " 'mil_gnn',\n",
       " 'mil_gnn_ds',\n",
       " 'self_attention',\n",
       " 'self_attention_axial_pe',\n",
       " 'self_attention_fourier_pe',\n",
       " 'set_transformer',\n",
       " 'transmil']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = [file.stem for file in yaml_folder.glob(\"*.yaml\") if file.name[0] != \"_\"]\n",
    "if selected_model_type == \"mnist_collage_ablations\":\n",
    "    models.append(\"distance_aware_self_attention\")\n",
    "models.sort()\n",
    "models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2023-05-19 16:21:45.050\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-abmil\u001b[0m\n",
      "\u001b[32m2023-05-19 16:21:47.893\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-discrete_rel_pos_self_attention\u001b[0m\n",
      "\u001b[32m2023-05-19 16:21:50.683\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-distance_aware_self_attention\u001b[0m\n",
      "\u001b[32m2023-05-19 16:21:53.730\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-gnn_gat\u001b[0m\n",
      "\u001b[32m2023-05-19 16:21:56.903\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-gnn_gcn\u001b[0m\n",
      "\u001b[32m2023-05-19 16:21:59.870\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-induced_set_transformer\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:02.638\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-just_pool\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:12.981\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-mil_gnn\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:15.748\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-mil_gnn_ds\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:19.227\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-self_attention\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:21.890\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-self_attention_axial_pe\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:24.757\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-self_attention_fourier_pe\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:27.724\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-set_transformer\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:30.419\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage-transmil\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:33.437\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-abmil\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:36.216\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-discrete_rel_pos_self_attention\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:38.874\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-distance_aware_self_attention\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:41.836\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-gnn_gat\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:44.604\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-gnn_gcn\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:47.363\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-induced_set_transformer\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:49.932\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-just_pool\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:52.768\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-mil_gnn\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:55.452\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-mil_gnn_ds\u001b[0m\n",
      "\u001b[32m2023-05-19 16:22:57.903\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-self_attention\u001b[0m\n",
      "\u001b[32m2023-05-19 16:23:00.769\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-self_attention_axial_pe\u001b[0m\n",
      "\u001b[32m2023-05-19 16:23:03.841\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-self_attention_fourier_pe\u001b[0m\n",
      "\u001b[32m2023-05-19 16:23:06.400\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-set_transformer\u001b[0m\n",
      "\u001b[32m2023-05-19 16:23:08.960\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mmake_summary\u001b[0m:\u001b[36msummarize_group\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mSummarizing group selected-mnist_collage_inverse-transmil\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean(train/acc)</th>\n",
       "      <th>std(train/acc)</th>\n",
       "      <th>mean(test/acc)</th>\n",
       "      <th>std(test/acc)</th>\n",
       "      <th>mean(train/auc)</th>\n",
       "      <th>std(train/auc)</th>\n",
       "      <th>mean(test/auc)</th>\n",
       "      <th>std(test/auc)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">mnist_collage</th>\n",
       "      <th>abmil</th>\n",
       "      <td>0.799</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.740</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.825</td>\n",
       "      <td>0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discrete_rel_pos_self_attention</th>\n",
       "      <td>0.921</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.926</td>\n",
       "      <td>0.030</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distance_aware_self_attention</th>\n",
       "      <td>0.955</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.958</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.990</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.992</td>\n",
       "      <td>0.009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gnn_gat</th>\n",
       "      <td>0.811</td>\n",
       "      <td>0.089</td>\n",
       "      <td>0.758</td>\n",
       "      <td>0.041</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.086</td>\n",
       "      <td>0.820</td>\n",
       "      <td>0.068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gnn_gcn</th>\n",
       "      <td>0.865</td>\n",
       "      <td>0.031</td>\n",
       "      <td>0.790</td>\n",
       "      <td>0.058</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.851</td>\n",
       "      <td>0.033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induced_set_transformer</th>\n",
       "      <td>0.813</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.734</td>\n",
       "      <td>0.040</td>\n",
       "      <td>0.889</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.829</td>\n",
       "      <td>0.013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>just_pool</th>\n",
       "      <td>0.846</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.828</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.924</td>\n",
       "      <td>0.007</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mil_gnn</th>\n",
       "      <td>0.546</td>\n",
       "      <td>0.061</td>\n",
       "      <td>0.578</td>\n",
       "      <td>0.084</td>\n",
       "      <td>0.648</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mil_gnn_ds</th>\n",
       "      <td>0.686</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.656</td>\n",
       "      <td>0.123</td>\n",
       "      <td>0.771</td>\n",
       "      <td>0.130</td>\n",
       "      <td>0.778</td>\n",
       "      <td>0.093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self_attention</th>\n",
       "      <td>0.881</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.848</td>\n",
       "      <td>0.024</td>\n",
       "      <td>0.932</td>\n",
       "      <td>0.028</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self_attention_axial_pe</th>\n",
       "      <td>0.862</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.786</td>\n",
       "      <td>0.051</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.860</td>\n",
       "      <td>0.063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self_attention_fourier_pe</th>\n",
       "      <td>0.878</td>\n",
       "      <td>0.035</td>\n",
       "      <td>0.820</td>\n",
       "      <td>0.022</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.025</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set_transformer</th>\n",
       "      <td>0.815</td>\n",
       "      <td>0.052</td>\n",
       "      <td>0.832</td>\n",
       "      <td>0.041</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.064</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transmil</th>\n",
       "      <td>0.957</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.728</td>\n",
       "      <td>0.054</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.831</td>\n",
       "      <td>0.041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">mnist_collage_inverse</th>\n",
       "      <th>abmil</th>\n",
       "      <td>0.805</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.692</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.790</td>\n",
       "      <td>0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discrete_rel_pos_self_attention</th>\n",
       "      <td>0.877</td>\n",
       "      <td>0.031</td>\n",
       "      <td>0.846</td>\n",
       "      <td>0.060</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.031</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distance_aware_self_attention</th>\n",
       "      <td>0.953</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.991</td>\n",
       "      <td>0.002</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gnn_gat</th>\n",
       "      <td>0.745</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.716</td>\n",
       "      <td>0.018</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.045</td>\n",
       "      <td>0.739</td>\n",
       "      <td>0.032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gnn_gcn</th>\n",
       "      <td>0.883</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.794</td>\n",
       "      <td>0.036</td>\n",
       "      <td>0.941</td>\n",
       "      <td>0.027</td>\n",
       "      <td>0.886</td>\n",
       "      <td>0.013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>induced_set_transformer</th>\n",
       "      <td>0.813</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.881</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>just_pool</th>\n",
       "      <td>0.840</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.788</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mil_gnn</th>\n",
       "      <td>0.543</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.542</td>\n",
       "      <td>0.094</td>\n",
       "      <td>0.723</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.716</td>\n",
       "      <td>0.071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mil_gnn_ds</th>\n",
       "      <td>0.784</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.756</td>\n",
       "      <td>0.019</td>\n",
       "      <td>0.864</td>\n",
       "      <td>0.027</td>\n",
       "      <td>0.817</td>\n",
       "      <td>0.008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self_attention</th>\n",
       "      <td>0.840</td>\n",
       "      <td>0.019</td>\n",
       "      <td>0.786</td>\n",
       "      <td>0.077</td>\n",
       "      <td>0.905</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.845</td>\n",
       "      <td>0.036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self_attention_axial_pe</th>\n",
       "      <td>0.902</td>\n",
       "      <td>0.018</td>\n",
       "      <td>0.774</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.877</td>\n",
       "      <td>0.015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self_attention_fourier_pe</th>\n",
       "      <td>0.879</td>\n",
       "      <td>0.037</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.863</td>\n",
       "      <td>0.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>set_transformer</th>\n",
       "      <td>0.819</td>\n",
       "      <td>0.018</td>\n",
       "      <td>0.792</td>\n",
       "      <td>0.045</td>\n",
       "      <td>0.872</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.855</td>\n",
       "      <td>0.024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transmil</th>\n",
       "      <td>0.957</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.742</td>\n",
       "      <td>0.053</td>\n",
       "      <td>0.993</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.805</td>\n",
       "      <td>0.046</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       mean(train/acc)  \\\n",
       "dataset               model                                              \n",
       "mnist_collage         abmil                                      0.799   \n",
       "                      discrete_rel_pos_self_attention            0.921   \n",
       "                      distance_aware_self_attention              0.955   \n",
       "                      gnn_gat                                    0.811   \n",
       "                      gnn_gcn                                    0.865   \n",
       "                      induced_set_transformer                    0.813   \n",
       "                      just_pool                                  0.846   \n",
       "                      mil_gnn                                    0.546   \n",
       "                      mil_gnn_ds                                 0.686   \n",
       "                      self_attention                             0.881   \n",
       "                      self_attention_axial_pe                    0.862   \n",
       "                      self_attention_fourier_pe                  0.878   \n",
       "                      set_transformer                            0.815   \n",
       "                      transmil                                   0.957   \n",
       "mnist_collage_inverse abmil                                      0.805   \n",
       "                      discrete_rel_pos_self_attention            0.877   \n",
       "                      distance_aware_self_attention              0.953   \n",
       "                      gnn_gat                                    0.745   \n",
       "                      gnn_gcn                                    0.883   \n",
       "                      induced_set_transformer                    0.813   \n",
       "                      just_pool                                  0.840   \n",
       "                      mil_gnn                                    0.543   \n",
       "                      mil_gnn_ds                                 0.784   \n",
       "                      self_attention                             0.840   \n",
       "                      self_attention_axial_pe                    0.902   \n",
       "                      self_attention_fourier_pe                  0.879   \n",
       "                      set_transformer                            0.819   \n",
       "                      transmil                                   0.957   \n",
       "\n",
       "                                                       std(train/acc)  \\\n",
       "dataset               model                                             \n",
       "mnist_collage         abmil                                     0.014   \n",
       "                      discrete_rel_pos_self_attention           0.014   \n",
       "                      distance_aware_self_attention             0.011   \n",
       "                      gnn_gat                                   0.089   \n",
       "                      gnn_gcn                                   0.031   \n",
       "                      induced_set_transformer                   0.017   \n",
       "                      just_pool                                 0.021   \n",
       "                      mil_gnn                                   0.061   \n",
       "                      mil_gnn_ds                                0.143   \n",
       "                      self_attention                            0.013   \n",
       "                      self_attention_axial_pe                   0.008   \n",
       "                      self_attention_fourier_pe                 0.035   \n",
       "                      set_transformer                           0.052   \n",
       "                      transmil                                  0.029   \n",
       "mnist_collage_inverse abmil                                     0.006   \n",
       "                      discrete_rel_pos_self_attention           0.031   \n",
       "                      distance_aware_self_attention             0.004   \n",
       "                      gnn_gat                                   0.033   \n",
       "                      gnn_gcn                                   0.034   \n",
       "                      induced_set_transformer                   0.023   \n",
       "                      just_pool                                 0.009   \n",
       "                      mil_gnn                                   0.095   \n",
       "                      mil_gnn_ds                                0.023   \n",
       "                      self_attention                            0.019   \n",
       "                      self_attention_axial_pe                   0.018   \n",
       "                      self_attention_fourier_pe                 0.037   \n",
       "                      set_transformer                           0.018   \n",
       "                      transmil                                  0.015   \n",
       "\n",
       "                                                       mean(test/acc)  \\\n",
       "dataset               model                                             \n",
       "mnist_collage         abmil                                     0.740   \n",
       "                      discrete_rel_pos_self_attention           0.926   \n",
       "                      distance_aware_self_attention             0.958   \n",
       "                      gnn_gat                                   0.758   \n",
       "                      gnn_gcn                                   0.790   \n",
       "                      induced_set_transformer                   0.734   \n",
       "                      just_pool                                 0.828   \n",
       "                      mil_gnn                                   0.578   \n",
       "                      mil_gnn_ds                                0.656   \n",
       "                      self_attention                            0.848   \n",
       "                      self_attention_axial_pe                   0.786   \n",
       "                      self_attention_fourier_pe                 0.820   \n",
       "                      set_transformer                           0.832   \n",
       "                      transmil                                  0.728   \n",
       "mnist_collage_inverse abmil                                     0.692   \n",
       "                      discrete_rel_pos_self_attention           0.846   \n",
       "                      distance_aware_self_attention             0.906   \n",
       "                      gnn_gat                                   0.716   \n",
       "                      gnn_gcn                                   0.794   \n",
       "                      induced_set_transformer                   0.720   \n",
       "                      just_pool                                 0.788   \n",
       "                      mil_gnn                                   0.542   \n",
       "                      mil_gnn_ds                                0.756   \n",
       "                      self_attention                            0.786   \n",
       "                      self_attention_axial_pe                   0.774   \n",
       "                      self_attention_fourier_pe                 0.808   \n",
       "                      set_transformer                           0.792   \n",
       "                      transmil                                  0.742   \n",
       "\n",
       "                                                       std(test/acc)  \\\n",
       "dataset               model                                            \n",
       "mnist_collage         abmil                                    0.010   \n",
       "                      discrete_rel_pos_self_attention          0.030   \n",
       "                      distance_aware_self_attention            0.013   \n",
       "                      gnn_gat                                  0.041   \n",
       "                      gnn_gcn                                  0.058   \n",
       "                      induced_set_transformer                  0.040   \n",
       "                      just_pool                                0.033   \n",
       "                      mil_gnn                                  0.084   \n",
       "                      mil_gnn_ds                               0.123   \n",
       "                      self_attention                           0.024   \n",
       "                      self_attention_axial_pe                  0.051   \n",
       "                      self_attention_fourier_pe                0.022   \n",
       "                      set_transformer                          0.041   \n",
       "                      transmil                                 0.054   \n",
       "mnist_collage_inverse abmil                                    0.015   \n",
       "                      discrete_rel_pos_self_attention          0.060   \n",
       "                      distance_aware_self_attention            0.034   \n",
       "                      gnn_gat                                  0.018   \n",
       "                      gnn_gcn                                  0.036   \n",
       "                      induced_set_transformer                  0.016   \n",
       "                      just_pool                                0.029   \n",
       "                      mil_gnn                                  0.094   \n",
       "                      mil_gnn_ds                               0.019   \n",
       "                      self_attention                           0.077   \n",
       "                      self_attention_axial_pe                  0.015   \n",
       "                      self_attention_fourier_pe                0.029   \n",
       "                      set_transformer                          0.045   \n",
       "                      transmil                                 0.053   \n",
       "\n",
       "                                                       mean(train/auc)  \\\n",
       "dataset               model                                              \n",
       "mnist_collage         abmil                                      0.893   \n",
       "                      discrete_rel_pos_self_attention            0.970   \n",
       "                      distance_aware_self_attention              0.990   \n",
       "                      gnn_gat                                    0.873   \n",
       "                      gnn_gcn                                    0.932   \n",
       "                      induced_set_transformer                    0.889   \n",
       "                      just_pool                                  0.924   \n",
       "                      mil_gnn                                    0.648   \n",
       "                      mil_gnn_ds                                 0.771   \n",
       "                      self_attention                             0.932   \n",
       "                      self_attention_axial_pe                    0.941   \n",
       "                      self_attention_fourier_pe                  0.943   \n",
       "                      set_transformer                            0.869   \n",
       "                      transmil                                   0.991   \n",
       "mnist_collage_inverse abmil                                      0.890   \n",
       "                      discrete_rel_pos_self_attention            0.954   \n",
       "                      distance_aware_self_attention              0.991   \n",
       "                      gnn_gat                                    0.795   \n",
       "                      gnn_gcn                                    0.941   \n",
       "                      induced_set_transformer                    0.881   \n",
       "                      just_pool                                  0.912   \n",
       "                      mil_gnn                                    0.723   \n",
       "                      mil_gnn_ds                                 0.864   \n",
       "                      self_attention                             0.905   \n",
       "                      self_attention_axial_pe                    0.963   \n",
       "                      self_attention_fourier_pe                  0.939   \n",
       "                      set_transformer                            0.872   \n",
       "                      transmil                                   0.993   \n",
       "\n",
       "                                                       std(train/auc)  \\\n",
       "dataset               model                                             \n",
       "mnist_collage         abmil                                     0.017   \n",
       "                      discrete_rel_pos_self_attention           0.005   \n",
       "                      distance_aware_self_attention             0.003   \n",
       "                      gnn_gat                                   0.086   \n",
       "                      gnn_gcn                                   0.029   \n",
       "                      induced_set_transformer                   0.008   \n",
       "                      just_pool                                 0.007   \n",
       "                      mil_gnn                                   0.022   \n",
       "                      mil_gnn_ds                                0.130   \n",
       "                      self_attention                            0.028   \n",
       "                      self_attention_axial_pe                   0.011   \n",
       "                      self_attention_fourier_pe                 0.025   \n",
       "                      set_transformer                           0.064   \n",
       "                      transmil                                  0.009   \n",
       "mnist_collage_inverse abmil                                     0.012   \n",
       "                      discrete_rel_pos_self_attention           0.031   \n",
       "                      distance_aware_self_attention             0.002   \n",
       "                      gnn_gat                                   0.045   \n",
       "                      gnn_gcn                                   0.027   \n",
       "                      induced_set_transformer                   0.011   \n",
       "                      just_pool                                 0.008   \n",
       "                      mil_gnn                                   0.056   \n",
       "                      mil_gnn_ds                                0.027   \n",
       "                      self_attention                            0.021   \n",
       "                      self_attention_axial_pe                   0.005   \n",
       "                      self_attention_fourier_pe                 0.034   \n",
       "                      set_transformer                           0.014   \n",
       "                      transmil                                  0.004   \n",
       "\n",
       "                                                       mean(test/auc)  \\\n",
       "dataset               model                                             \n",
       "mnist_collage         abmil                                     0.825   \n",
       "                      discrete_rel_pos_self_attention           0.975   \n",
       "                      distance_aware_self_attention             0.992   \n",
       "                      gnn_gat                                   0.820   \n",
       "                      gnn_gcn                                   0.851   \n",
       "                      induced_set_transformer                   0.829   \n",
       "                      just_pool                                 0.905   \n",
       "                      mil_gnn                                   0.708   \n",
       "                      mil_gnn_ds                                0.778   \n",
       "                      self_attention                            0.888   \n",
       "                      self_attention_axial_pe                   0.860   \n",
       "                      self_attention_fourier_pe                 0.874   \n",
       "                      set_transformer                           0.867   \n",
       "                      transmil                                  0.831   \n",
       "mnist_collage_inverse abmil                                     0.790   \n",
       "                      discrete_rel_pos_self_attention           0.923   \n",
       "                      distance_aware_self_attention             0.970   \n",
       "                      gnn_gat                                   0.739   \n",
       "                      gnn_gcn                                   0.886   \n",
       "                      induced_set_transformer                   0.808   \n",
       "                      just_pool                                 0.830   \n",
       "                      mil_gnn                                   0.716   \n",
       "                      mil_gnn_ds                                0.817   \n",
       "                      self_attention                            0.845   \n",
       "                      self_attention_axial_pe                   0.877   \n",
       "                      self_attention_fourier_pe                 0.863   \n",
       "                      set_transformer                           0.855   \n",
       "                      transmil                                  0.805   \n",
       "\n",
       "                                                       std(test/auc)  \n",
       "dataset               model                                           \n",
       "mnist_collage         abmil                                    0.008  \n",
       "                      discrete_rel_pos_self_attention          0.005  \n",
       "                      distance_aware_self_attention            0.009  \n",
       "                      gnn_gat                                  0.068  \n",
       "                      gnn_gcn                                  0.033  \n",
       "                      induced_set_transformer                  0.013  \n",
       "                      just_pool                                0.008  \n",
       "                      mil_gnn                                  0.059  \n",
       "                      mil_gnn_ds                               0.093  \n",
       "                      self_attention                           0.013  \n",
       "                      self_attention_axial_pe                  0.063  \n",
       "                      self_attention_fourier_pe                0.012  \n",
       "                      set_transformer                          0.036  \n",
       "                      transmil                                 0.041  \n",
       "mnist_collage_inverse abmil                                    0.008  \n",
       "                      discrete_rel_pos_self_attention          0.089  \n",
       "                      distance_aware_self_attention            0.020  \n",
       "                      gnn_gat                                  0.032  \n",
       "                      gnn_gcn                                  0.013  \n",
       "                      induced_set_transformer                  0.004  \n",
       "                      just_pool                                0.010  \n",
       "                      mil_gnn                                  0.071  \n",
       "                      mil_gnn_ds                               0.008  \n",
       "                      self_attention                           0.036  \n",
       "                      self_attention_axial_pe                  0.015  \n",
       "                      self_attention_fourier_pe                0.016  \n",
       "                      set_transformer                          0.024  \n",
       "                      transmil                                 0.046  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_stats = {\n",
    "    dataset: {\n",
    "        model: summarize_group(f\"selected-{dataset}-{model}\", log_to_wandb=False)\n",
    "        for model in models\n",
    "    } for dataset in datasets\n",
    "}\n",
    "dfs = []\n",
    "for dataset, dataset_stats in all_stats.items():\n",
    "    df = pd.DataFrame(dataset_stats).T\n",
    "    df[\"dataset\"] = dataset\n",
    "    dfs.append(df)\n",
    "\n",
    "# Merge\n",
    "df = pd.concat(dfs)\n",
    "df.index.name = \"model\"\n",
    "df.reset_index(inplace=True)\n",
    "df.set_index([\"dataset\", \"model\"], inplace=True)\n",
    "df[[\"mean(train/acc)\", \"std(train/acc)\", \"mean(test/acc)\", \"std(test/acc)\", \"mean(train/auc)\", \"std(train/auc)\", \"mean(test/auc)\", \"std(test/auc)\"]].round(3)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{l|c|r|rr|rr}\n",
      "\\toprule\n",
      " & & & \\multicolumn{2}{c|}{\\smaller{MNIST-COLLAGE}} & \\multicolumn{2}{c}{\\smaller{MNIST-COLLAGE-INV}} \\\\\n",
      "Model & \\multicolumn{1}{c|}{Pos} & \\multicolumn{1}{c|}{Params} & \\multicolumn{1}{c}{Train} & \\multicolumn{1}{c|}{Test} & \\multicolumn{1}{c}{Train} & \\multicolumn{1}{c}{Test} \\\\\n",
      "\\midrule\n",
      "MIL with max pool & \\xmark & 15.6K & 0.846 $\\pm$ 0.021 & 0.828 $\\pm$ 0.033 & 0.840 $\\pm$ 0.009 & 0.788 $\\pm$ 0.029 \\\\\n",
      "AB-MIL~\\cite{ilse2018attention} & \\xmark & 16.1K & 0.799 $\\pm$ 0.014 & 0.740 $\\pm$ 0.010 & 0.805 $\\pm$ 0.006 & 0.692 $\\pm$ 0.015 \\\\\n",
      "MIL with GNN (GAT~\\cite{velickovic2018graph}) &  \\smaller{REL} & 16.7K & 0.811 $\\pm$ 0.089 & 0.758 $\\pm$ 0.041 & 0.745 $\\pm$ 0.033 & 0.716 $\\pm$ 0.018 \\\\\n",
      "MIL with GNN (GCN~\\cite{kipf2017semisupervised}) &  \\smaller{REL} & 16.3K & 0.865 $\\pm$ 0.031 & 0.790 $\\pm$ 0.058 & 0.883 $\\pm$ 0.034 & 0.794 $\\pm$ 0.036 \\\\\n",
      "MIL-GNN~\\cite{tu2019multiple} &  \\smaller{REL} & 19.2K & 0.686 $\\pm$ 0.143 & 0.656 $\\pm$ 0.123 & 0.784 $\\pm$ 0.023 & 0.756 $\\pm$ 0.019 \\\\\n",
      "MIL with iSet Transformer~\\cite{lee2019set} & \\xmark & 22.1K & 0.813 $\\pm$ 0.017 & 0.734 $\\pm$ 0.040 & 0.813 $\\pm$ 0.023 & 0.720 $\\pm$ 0.016 \\\\\n",
      "MIL with Set Transformer~\\cite{lee2019set} & \\xmark & 17.6K & 0.815 $\\pm$ 0.052 & 0.832 $\\pm$ 0.041 & 0.819 $\\pm$ 0.018 & 0.792 $\\pm$ 0.045 \\\\\n",
      "MIL with SA~\\cite{vaswani2017attention} & \\xmark & 17.2K & 0.881 $\\pm$ 0.013 & 0.848 $\\pm$ 0.024 & 0.840 $\\pm$ 0.019 & 0.786 $\\pm$ 0.077 \\\\\n",
      "MIL with SA + axial PE~\\cite{ramachandran2019stand} &  \\smaller{ABS} & 17.6K & 0.862 $\\pm$ 0.008 & 0.786 $\\pm$ 0.051 & 0.902 $\\pm$ 0.018 & 0.774 $\\pm$ 0.015 \\\\\n",
      "MIL with SA + Fourier PE~\\cite{yang2021learnable} &  \\smaller{ABS} & 18.4K & 0.878 $\\pm$ 0.035 & 0.820 $\\pm$ 0.022 & 0.879 $\\pm$ 0.037 & 0.808 $\\pm$ 0.029 \\\\\n",
      "MIL with disc.\\ rel.\\ SA~\\cite{wu2021rethinking} &  \\smaller{REL} & 17.8K & 0.921 $\\pm$ 0.014 & 0.926 $\\pm$ 0.030 & 0.877 $\\pm$ 0.031 & 0.846 $\\pm$ 0.060 \\\\\n",
      "TransMIL~\\cite{shao2021transmil} &  \\smaller{ABS} & 2.18M & \\textbf{0.957 $\\pm$ 0.029} & 0.728 $\\pm$ 0.054 & \\textbf{0.957 $\\pm$ 0.015} & 0.742 $\\pm$ 0.053 \\\\\n",
      "DAS-MIL (ours) &  \\smaller{REL} & 17.4K & 0.955 $\\pm$ 0.011 & \\textbf{0.958 $\\pm$ 0.013} & 0.953 $\\pm$ 0.004 & \\textbf{0.906 $\\pm$ 0.034} \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n"
     ]
    }
   ],
   "source": [
    "if selected_model_type == \"mnist_collage\":\n",
    "    NAMES = {\n",
    "        \"just_pool\": \"MIL with max pool\",\n",
    "        \"abmil\": \"AB-MIL~\\\\cite{ilse2018attention}\",\n",
    "        \"gnn_gat\": \"MIL with GNN (GAT~\\\\cite{velickovic2018graph})\",\n",
    "        \"gnn_gcn\": \"MIL with GNN (GCN~\\\\cite{kipf2017semisupervised})\",\n",
    "        # \"mil_gnn\": \"MIL-GNN~\\\\cite{tu2019multiple}\",\n",
    "        # \"mil_gnn_ds\": \"MIL-GNN-DS~\\\\cite{tu2019multiple}\",\n",
    "        \"mil_gnn_ds\": \"MIL-GNN~\\\\cite{tu2019multiple}\",\n",
    "        \"induced_set_transformer\": \"MIL with iSet Transformer~\\\\cite{lee2019set}\",\n",
    "        \"set_transformer\": \"MIL with Set Transformer~\\\\cite{lee2019set}\",\n",
    "        \"self_attention\": \"MIL with SA~\\\\cite{vaswani2017attention}\",\n",
    "        \"self_attention_axial_pe\": \"MIL with SA + axial PE~\\\\cite{ramachandran2019stand}\",\n",
    "        \"self_attention_fourier_pe\": \"MIL with SA + Fourier PE~\\\\cite{yang2021learnable}\",\n",
    "        \"discrete_rel_pos_self_attention\": \"MIL with disc.\\\\ rel.\\\\ SA~\\\\cite{wu2021rethinking}\",\n",
    "        \"transmil\": \"TransMIL~\\\\cite{shao2021transmil}\",\n",
    "        \"distance_aware_self_attention\": \"DAS-MIL (ours)\",\n",
    "    }\n",
    "elif selected_model_type == \"mnist_collage_ablations\":\n",
    "    NAMES = {\n",
    "        \"distance_aware_self_attention_embedk\": \"DAS-MIL ($\\\\vb^K$)\",\n",
    "        \"distance_aware_self_attention_embedq\": \"DAS-MIL ($\\\\vb^Q$)\",\n",
    "        \"distance_aware_self_attention_embedv\": \"DAS-MIL ($\\\\vb^V$)\",\n",
    "        \"distance_aware_self_attention_embedkq\": \"DAS-MIL ($\\\\vb^K, \\\\vb^Q$) ($+ \\\\vb^Q{\\\\vb^K}^\\\\top$ in \\\\cref{eq:das:compatibility:impl})\",\n",
    "        \"distance_aware_self_attention_embedkq_noterm3\": \"DAS-MIL ($\\\\vb^K, \\\\vb^Q$)\",\n",
    "        \"distance_aware_self_attention_embedkv\": \"DAS-MIL ($\\\\vb^K, \\\\vb^V$)\",\n",
    "        \"distance_aware_self_attention_embedqv\": \"DAS-MIL ($\\\\vb^Q, \\\\vb^V$)\",\n",
    "        \"distance_aware_self_attention_embedkqv\": \"DAS-MIL ($\\\\vb^K, \\\\vb^Q, \\\\vb^V$) ($+ \\\\vb^Q{\\\\vb^K}^\\\\top$ in \\\\cref{eq:das:compatibility:impl})\",\n",
    "        \"line1\": None,\n",
    "        \"distance_aware_self_attention_fixedembed\": \"DAS-MIL (non-trainable $\\\\vb^K, \\\\vb^Q, \\\\vb^V$)\",\n",
    "        \"line2\": None,\n",
    "        # \"distance_aware_self_attention_embedkqv_noterm3\": \"DAS-MIL ($\\\\vb^K, \\\\vb^Q, \\\\vb^V$)\",\n",
    "        \"distance_aware_self_attention\": \"DAS-MIL ($\\\\vb^K, \\\\vb^Q, \\\\vb^V$)\",\n",
    "    }\n",
    "elif selected_model_type == \"camelyon16\":\n",
    "    NAMES = {\n",
    "        \"just_pool\": \"MIL with max pool\",\n",
    "        \"abmil\": \"AB-MIL~\\\\cite{ilse2018attention}\",\n",
    "        \"self_attention\": \"MIL with SA~\\\\cite{vaswani2017attention}\",\n",
    "        \"self_attention_axial_pe\": \"MIL with SA + axial PE~\\\\cite{ramachandran2019stand}\",\n",
    "        \"self_attention_fourier_pe\": \"MIL with SA + Fourier PE~\\\\cite{yang2021learnable}\",\n",
    "        'discrete_rel_pos_self_attention': \"MIL with disc.\\\\ rel.\\\\ SA~\\\\cite{wu2021rethinking}\",\n",
    "        'transmil': \"TransMIL~\\\\cite{shao2021transmil}\",\n",
    "        # 'transmil_ourparams': \"TransMIL~\\\\cite{shao2021transmil}\",\n",
    "        # 'distance_aware_self_attention': \"DAS-MIL (no FC)\",\n",
    "        'distance_aware_self_attention_fc': \"DAS-MIL (ours)\",\n",
    "        # 'distance_aware_self_attention_fc_t3': \"DAS-MIL (FC, T3)\",\n",
    "    }\n",
    "\n",
    "pos_col = selected_model_type == \"mnist_collage\" or selected_model_type == \"camelyon16\"\n",
    "def if_pos(s):\n",
    "    return s if pos_col else \"\"\n",
    "\n",
    "abs_pos = [\"self_attention_axial_pe\", \"self_attention_fourier_pe\", \"transmil\"]\n",
    "rel_pos = [\"discrete_rel_pos_self_attention\", \"distance_aware_self_attention_fc\", \"distance_aware_self_attention\", \"mil_gnn\", \"mil_gnn_ds\", \"gnn_gat\", \"gnn_gcn\"]\n",
    "\n",
    "\n",
    "if \"mnist_collage\" in selected_model_type:\n",
    "    metrics = [\"balanced_acc\"]\n",
    "    # metrics = [\"auc\"]\n",
    "    print(\"\\\\begin{tabular}{l\" + if_pos(\"|c\") + \"|r|rr|rr}\")\n",
    "    print(\"\\\\toprule\")\n",
    "    print(\" & \" + if_pos(\"&\") + \" & \\\\multicolumn{2}{c|}{\\\\smaller{MNIST-COLLAGE}} & \\\\multicolumn{2}{c}{\\\\smaller{MNIST-COLLAGE-INV}} \\\\\\\\\")\n",
    "    print(\"Model \" + if_pos(\"& \\\\multicolumn{1}{c|}{Pos}\") + \" & \\\\multicolumn{1}{c|}{Params} & \\\\multicolumn{1}{c}{Train} & \\\\multicolumn{1}{c|}{Test} & \\\\multicolumn{1}{c}{Train} & \\\\multicolumn{1}{c}{Test} \\\\\\\\\")\n",
    "    print(\"\\\\midrule\")\n",
    "\n",
    "elif selected_model_type == \"camelyon16\":\n",
    "    metrics = [\"auc\", \"balanced_acc\"]\n",
    "    print(\"\\\\begin{tabular}{l|c|c|rr|rr}\")\n",
    "    print(\"\\\\toprule\")\n",
    "    print(\" & & & \\\\multicolumn{2}{c|}{AUROC} & \\\\multicolumn{2}{c}{Balanced accuracy} \\\\\\\\\")\n",
    "    print(\"Model & Pos & \\\\multicolumn{1}{c|}{Params} & \\\\multicolumn{1}{c}{Train} & \\\\multicolumn{1}{c|}{Test} & \\\\multicolumn{1}{c}{Train} & \\\\multicolumn{1}{c}{Test} \\\\\\\\\")\n",
    "    print(\"\\\\midrule\")\n",
    "\n",
    "rows = {name: row for name, row in df.iterrows()}\n",
    "for name, desc in NAMES.items():\n",
    "    if desc is None:\n",
    "        print(\"\\\\hline\")\n",
    "        continue\n",
    "    pos = \"abs\" if name in abs_pos else \"rel\" if name in rel_pos else None\n",
    "    row = f\"{NAMES[name]}\"\n",
    "    pos = f\" \\\\smaller{{{pos.upper()}}}\" if pos in (\"abs\", \"rel\") else \"\\\\xmark\"\n",
    "    row += if_pos(f\" & {pos}\")\n",
    "    row += f\" & {human_format(df.loc[datasets[0], name]['num_parameters'])}\"\n",
    "\n",
    "\n",
    "    for dataset in datasets:\n",
    "        for metric in metrics:\n",
    "            df_ds = df.loc[dataset]\n",
    "            for split in (\"train\", \"test\"):\n",
    "                if df_ds[f\"mean({split}/{metric})\"].loc[list(NAMES)].max().round(3) == df_ds[f\"mean({split}/{metric})\"].loc[name].round(3):\n",
    "                    row += f\" & \\\\textbf{{{df_ds[f'mean({split}/{metric})'].loc[name]:.03f} $\\pm$ {df_ds[f'std({split}/{metric})'].loc[name]:.03f}}}\"\n",
    "                else:\n",
    "                    row += f\" & {df_ds[f'mean({split}/{metric})'].loc[name]:.03f} $\\pm$ {df_ds[f'std({split}/{metric})'].loc[name]:.03f}\"\n",
    "    print(row + \" \\\\\\\\\")\n",
    "print(\"\\\\bottomrule\")\n",
    "print(\"\\\\end{tabular}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table of hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{l|rrrrr}\n",
      "\\toprule\n",
      "Model & optimiser & LR & weight decay & hidden dim & agg \\\\\n",
      "\\midrule\n",
      "MIL with max pool & Adam & 0.0001 & 0.01 & 10 & max \\\\\n",
      "AB-MIL~\\cite{ilse2018attention} & Adam & 0.0001 & 0.001 & 15 & N/A \\\\\n",
      "MIL with GNN (GAT~\\cite{velickovic2018graph}) & Adam & 0.001 & 0.1 & 20 & max \\\\\n",
      "MIL with GNN (GCN~\\cite{kipf2017semisupervised}) & Adam & 0.001 & 0.1 & 15 & max \\\\\n",
      "MIL-GNN~\\cite{tu2019multiple} & Adam & 0.001 & 0.01 & 20 & N/A \\\\\n",
      "MIL with iSet Transformer~\\cite{lee2019set} & Adam & 0.0001 & 0.1 & 15 & N/A \\\\\n",
      "MIL with Set Transformer~\\cite{lee2019set} & Adam & 0.001 & 0.1 & 10 & N/A \\\\\n",
      "MIL with SA~\\cite{vaswani2017attention} & Adam & 0.001 & 0.1 & 10 & max \\\\\n",
      "MIL with SA + axial PE~\\cite{ramachandran2019stand} & Adam & 0.0001 & 0.1 & 15 & max \\\\\n",
      "MIL with SA + Fourier PE~\\cite{yang2021learnable} & Adam & 0.001 & 0.001 & 15 & max \\\\\n",
      "MIL with disc.\\ rel.\\ SA~\\cite{wu2021rethinking} & Adam & 0.0001 & 0.1 & 10 & max \\\\\n",
      "TransMIL~\\cite{shao2021transmil} & Lookahead & 0.0001 & 0.1 & N/A & N/A \\\\\n",
      "DAS-MIL (ours) & Adam & 0.001 & 0.01 & 10 & max \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n"
     ]
    }
   ],
   "source": [
    "from omegaconf import OmegaConf, DictConfig\n",
    "\n",
    "print(\"\\\\begin{tabular}{l|rrrrr}\")\n",
    "print(\"\\\\toprule\")\n",
    "print(\"Model & optimiser & LR & weight decay & hidden dim & agg \\\\\\\\\")\n",
    "print(\"\\\\midrule\")\n",
    "NA = \"N/A\"\n",
    "\n",
    "for name in NAMES:\n",
    "    file = yaml_folder / f\"{name}.yaml\"\n",
    "    cfg = OmegaConf.load(file)\n",
    "    # Model & LR & weight decay & hidden dim & agg \\\\\n",
    "    optim = \"Adam\" if \"base_optimizer\" not in cfg.optimizer else \"Lookahead\"\n",
    "    lr = cfg.optimizer.lr if \"lr\" in cfg.optimizer else cfg.optimizer.base_optimizer.lr\n",
    "    weight_decay = cfg.optimizer.weight_decay if \"weight_decay\" in cfg.optimizer else cfg.optimizer.base_optimizer.weight_decay\n",
    "    if \"settings\" in cfg:\n",
    "        hidden_dim = cfg.settings.get(\"hidden_dim\", NA)\n",
    "        agg = cfg.settings.get(\"agg\", NA)\n",
    "    else:\n",
    "        hidden_dim = NA\n",
    "        agg = NA\n",
    "    if agg == NA:\n",
    "        base_model = [x for x in cfg.defaults if isinstance(x, DictConfig) and \"/model\" in x]\n",
    "        if len(base_model) > 0:\n",
    "            base_model = base_model[0].get(\"/model\")\n",
    "            if \"max\" in base_model:\n",
    "                agg = \"max\"\n",
    "    print(f\"{NAMES[name]} & {optim} & {lr} & {weight_decay} & {hidden_dim} & {agg} \\\\\\\\\")\n",
    "print(\"\\\\bottomrule\")\n",
    "print(\"\\\\end{tabular}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mil_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.14"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}