experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from load_data import load_data\n",
    "import torch\n",
    "from modules import GNN\n",
    "from train_model import train_model\n",
    "from subgraph_relevance import subgraph_original, subgraph_mp_transcription, subgraph_mp_forward_hook, get_H_transform\n",
    "from utils import create_ground_truth, get_feat_order_local_best_guess, get_auac_aupc, get_stats\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "import pandas as pd\n",
    "from io import StringIO\n",
    "import pickle as pkl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Efficiency"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As a function of model layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_graphs: 1000\n"
     ]
    }
   ],
   "source": [
    "graphs, pos_idx, neg_idx = load_data('BA-2motif')\n",
    "\n",
    "model_dirs = ['gin-2-ba2motif.torch',\n",
    "            'gin-3-ba2motif.torch',\n",
    "            'gin-4-ba2motif.torch',\n",
    "            'gin-5-ba2motif.torch',\n",
    "            'gin-6-ba2motif.torch',\n",
    "            'gin-7-ba2motif.torch']\n",
    "\n",
    "g = graphs[44]\n",
    "S = [0,1,2,3]\n",
    "alpha = 0.\n",
    "verbose = False\n",
    "num_samples = 50\n",
    "sample_idx = np.random.choice(len(graphs),num_samples,replace=False)\n",
    "\n",
    "model_times = []\n",
    "\n",
    "nn = torch.load('models/'+model_dirs[1])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmin(b): return -0.5*torch.log(1.0+torch.exp(-2*b))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Runtime experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_graphs: 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/6 [00:00<?, ?it/s]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:00,  6.65it/s]\u001b[A\n",
      "7it [00:00, 31.82it/s]\u001b[A\n",
      "13it [00:00, 41.16it/s]\u001b[A\n",
      "19it [00:00, 45.55it/s]\u001b[A\n",
      "25it [00:00, 48.61it/s]\u001b[A\n",
      "31it [00:00, 50.01it/s]\u001b[A\n",
      "37it [00:00, 49.01it/s]\u001b[A\n",
      "42it [00:00, 48.38it/s]\u001b[A\n",
      "50it [00:01, 45.36it/s]\u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 276.22it/s]\u001b[A\n",
      "\n",
      "100%|██████████| 50/50 [00:00<00:00, 748.77it/s]\n",
      " 17%|█▋        | 1/6 [00:01<00:06,  1.36s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:00,  6.79it/s]\u001b[A\n",
      "2it [00:00,  7.41it/s]\u001b[A\n",
      "3it [00:00,  8.12it/s]\u001b[A\n",
      "4it [00:00,  8.17it/s]\u001b[A\n",
      "5it [00:00,  8.27it/s]\u001b[A\n",
      "6it [00:00,  8.29it/s]\u001b[A\n",
      "7it [00:00,  8.35it/s]\u001b[A\n",
      "8it [00:00,  8.38it/s]\u001b[A\n",
      "9it [00:01,  8.07it/s]\u001b[A\n",
      "10it [00:01,  8.31it/s]\u001b[A\n",
      "11it [00:01,  8.29it/s]\u001b[A\n",
      "12it [00:01,  8.32it/s]\u001b[A\n",
      "13it [00:01,  8.36it/s]\u001b[A\n",
      "14it [00:01,  8.36it/s]\u001b[A\n",
      "15it [00:01,  8.36it/s]\u001b[A\n",
      "16it [00:01,  8.39it/s]\u001b[A\n",
      "17it [00:02,  8.39it/s]\u001b[A\n",
      "18it [00:02,  8.09it/s]\u001b[A\n",
      "19it [00:02,  8.35it/s]\u001b[A\n",
      "20it [00:02,  7.70it/s]\u001b[A\n",
      "21it [00:02,  7.40it/s]\u001b[A\n",
      "22it [00:02,  7.90it/s]\u001b[A\n",
      "23it [00:02,  8.03it/s]\u001b[A\n",
      "24it [00:02,  8.14it/s]\u001b[A\n",
      "25it [00:03,  8.24it/s]\u001b[A\n",
      "26it [00:03,  8.25it/s]\u001b[A\n",
      "27it [00:03,  8.29it/s]\u001b[A\n",
      "28it [00:03,  8.33it/s]\u001b[A\n",
      "29it [00:03,  8.31it/s]\u001b[A\n",
      "30it [00:03,  8.37it/s]\u001b[A\n",
      "31it [00:03,  8.38it/s]\u001b[A\n",
      "32it [00:03,  8.37it/s]\u001b[A\n",
      "33it [00:04,  8.09it/s]\u001b[A\n",
      "34it [00:04,  8.43it/s]\u001b[A\n",
      "35it [00:04,  8.39it/s]\u001b[A\n",
      "36it [00:04,  8.38it/s]\u001b[A\n",
      "37it [00:04,  8.41it/s]\u001b[A\n",
      "38it [00:04,  8.06it/s]\u001b[A\n",
      "39it [00:04,  8.12it/s]\u001b[A\n",
      "40it [00:04,  8.12it/s]\u001b[A\n",
      "41it [00:05,  8.15it/s]\u001b[A\n",
      "42it [00:05,  8.08it/s]\u001b[A\n",
      "43it [00:05,  8.15it/s]\u001b[A\n",
      "44it [00:05,  7.67it/s]\u001b[A\n",
      "45it [00:05,  7.65it/s]\u001b[A\n",
      "46it [00:05,  7.61it/s]\u001b[A\n",
      "47it [00:05,  7.84it/s]\u001b[A\n",
      "48it [00:05,  7.77it/s]\u001b[A\n",
      "49it [00:06,  7.96it/s]\u001b[A\n",
      "50it [00:06,  8.10it/s]\u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      " 18%|█▊        | 9/50 [00:00<00:00, 87.21it/s]\u001b[A\n",
      " 46%|████▌     | 23/50 [00:00<00:00, 110.35it/s]\u001b[A\n",
      " 68%|██████▊   | 34/50 [00:00<00:00, 105.06it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 95.50it/s] \u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 383.87it/s]\u001b[A\n",
      " 33%|███▎      | 2/6 [00:08<00:18,  4.58s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:01,  1.07s/it]\u001b[A\n",
      "2it [00:01,  1.23it/s]\u001b[A\n",
      "3it [00:02,  1.36it/s]\u001b[A\n",
      "4it [00:02,  1.46it/s]\u001b[A\n",
      "5it [00:03,  1.49it/s]\u001b[A\n",
      "6it [00:04,  1.51it/s]\u001b[A\n",
      "7it [00:04,  1.54it/s]\u001b[A\n",
      "8it [00:05,  1.51it/s]\u001b[A\n",
      "9it [00:06,  1.52it/s]\u001b[A\n",
      "10it [00:06,  1.57it/s]\u001b[A\n",
      "11it [00:07,  1.59it/s]\u001b[A\n",
      "12it [00:08,  1.59it/s]\u001b[A\n",
      "13it [00:08,  1.60it/s]\u001b[A\n",
      "14it [00:09,  1.63it/s]\u001b[A\n",
      "15it [00:09,  1.61it/s]\u001b[A\n",
      "16it [00:10,  1.63it/s]\u001b[A\n",
      "17it [00:11,  1.65it/s]\u001b[A\n",
      "18it [00:11,  1.65it/s]\u001b[A\n",
      "19it [00:12,  1.63it/s]\u001b[A\n",
      "20it [00:12,  1.62it/s]\u001b[A\n",
      "21it [00:13,  1.63it/s]\u001b[A\n",
      "22it [00:14,  1.60it/s]\u001b[A\n",
      "23it [00:14,  1.59it/s]\u001b[A\n",
      "24it [00:15,  1.59it/s]\u001b[A\n",
      "25it [00:16,  1.56it/s]\u001b[A\n",
      "26it [00:16,  1.56it/s]\u001b[A\n",
      "27it [00:17,  1.60it/s]\u001b[A\n",
      "28it [00:17,  1.62it/s]\u001b[A\n",
      "29it [00:18,  1.62it/s]\u001b[A\n",
      "30it [00:19,  1.60it/s]\u001b[A\n",
      "31it [00:19,  1.59it/s]\u001b[A\n",
      "32it [00:20,  1.58it/s]\u001b[A\n",
      "33it [00:21,  1.62it/s]\u001b[A\n",
      "34it [00:21,  1.63it/s]\u001b[A\n",
      "35it [00:22,  1.65it/s]\u001b[A\n",
      "36it [00:22,  1.65it/s]\u001b[A\n",
      "37it [00:23,  1.66it/s]\u001b[A\n",
      "38it [00:24,  1.67it/s]\u001b[A\n",
      "39it [00:24,  1.67it/s]\u001b[A\n",
      "40it [00:25,  1.67it/s]\u001b[A\n",
      "41it [00:25,  1.68it/s]\u001b[A\n",
      "42it [00:26,  1.68it/s]\u001b[A\n",
      "43it [00:27,  1.68it/s]\u001b[A\n",
      "44it [00:27,  1.64it/s]\u001b[A\n",
      "45it [00:28,  1.59it/s]\u001b[A\n",
      "46it [00:28,  1.58it/s]\u001b[A\n",
      "47it [00:29,  1.59it/s]\u001b[A\n",
      "48it [00:30,  1.59it/s]\u001b[A\n",
      "49it [00:30,  1.59it/s]\u001b[A\n",
      "50it [00:31,  1.59it/s]\u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 10/50 [00:00<00:00, 97.05it/s]\u001b[A\n",
      " 42%|████▏     | 21/50 [00:00<00:00, 98.80it/s]\u001b[A\n",
      " 62%|██████▏   | 31/50 [00:00<00:00, 95.33it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 91.66it/s]\u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 270.11it/s]\u001b[A\n",
      " 50%|█████     | 3/6 [00:40<00:51, 17.20s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:03,  3.45s/it]\u001b[A\n",
      "2it [00:06,  3.33s/it]\u001b[A\n",
      "3it [00:09,  3.30s/it]\u001b[A\n",
      "4it [00:13,  3.31s/it]\u001b[A\n",
      "5it [00:16,  3.26s/it]\u001b[A\n",
      "6it [00:19,  3.23s/it]\u001b[A\n",
      "7it [00:22,  3.25s/it]\u001b[A\n",
      "8it [00:26,  3.22s/it]\u001b[A\n",
      "9it [00:29,  3.16s/it]\u001b[A\n",
      "10it [00:32,  3.18s/it]\u001b[A\n",
      "11it [00:35,  3.15s/it]\u001b[A\n",
      "12it [00:38,  3.17s/it]\u001b[A\n",
      "13it [00:41,  3.15s/it]\u001b[A\n",
      "14it [00:44,  3.12s/it]\u001b[A\n",
      "15it [00:47,  3.09s/it]\u001b[A\n",
      "16it [00:50,  3.07s/it]\u001b[A\n",
      "17it [00:53,  3.05s/it]\u001b[A\n",
      "18it [00:56,  3.04s/it]\u001b[A\n",
      "19it [00:59,  3.05s/it]\u001b[A\n",
      "20it [01:03,  3.06s/it]\u001b[A\n",
      "21it [01:06,  3.11s/it]\u001b[A\n",
      "22it [01:09,  3.12s/it]\u001b[A\n",
      "23it [01:12,  3.10s/it]\u001b[A\n",
      "24it [01:15,  3.08s/it]\u001b[A\n",
      "25it [01:18,  3.06s/it]\u001b[A\n",
      "26it [01:21,  3.07s/it]\u001b[A\n",
      "27it [01:24,  3.06s/it]\u001b[A\n",
      "28it [01:27,  3.04s/it]\u001b[A\n",
      "29it [01:30,  3.04s/it]\u001b[A\n",
      "30it [01:33,  3.03s/it]\u001b[A\n",
      "31it [01:36,  3.03s/it]\u001b[A\n",
      "32it [01:39,  3.04s/it]\u001b[A\n",
      "33it [01:42,  3.05s/it]\u001b[A\n",
      "34it [01:46,  3.10s/it]\u001b[A\n",
      "35it [01:49,  3.09s/it]\u001b[A\n",
      "36it [01:52,  3.10s/it]\u001b[A\n",
      "37it [01:55,  3.10s/it]\u001b[A\n",
      "38it [01:58,  3.11s/it]\u001b[A\n",
      "39it [02:01,  3.20s/it]\u001b[A\n",
      "40it [02:05,  3.20s/it]\u001b[A\n",
      "41it [02:08,  3.21s/it]\u001b[A\n",
      "42it [02:11,  3.20s/it]\u001b[A\n",
      "43it [02:14,  3.21s/it]\u001b[A\n",
      "44it [02:17,  3.17s/it]\u001b[A\n",
      "45it [02:21,  3.21s/it]\u001b[A\n",
      "46it [02:24,  3.21s/it]\u001b[A\n",
      "47it [02:27,  3.21s/it]\u001b[A\n",
      "48it [02:30,  3.23s/it]\u001b[A\n",
      "49it [02:33,  3.19s/it]\u001b[A\n",
      "50it [02:37,  3.14s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      " 16%|█▌        | 8/50 [00:00<00:00, 73.20it/s]\u001b[A\n",
      " 32%|███▏      | 16/50 [00:00<00:00, 74.80it/s]\u001b[A\n",
      " 48%|████▊     | 24/50 [00:00<00:00, 71.61it/s]\u001b[A\n",
      " 64%|██████▍   | 32/50 [00:00<00:00, 67.37it/s]\u001b[A\n",
      " 78%|███████▊  | 39/50 [00:00<00:00, 66.22it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 67.73it/s]\u001b[A\n",
      "\n",
      "  0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "100%|██████████| 50/50 [00:00<00:00, 251.70it/s]\u001b[A\n",
      " 67%|██████▋   | 4/6 [03:18<02:25, 72.79s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:15, 15.77s/it]\u001b[A"
     ]
    }
   ],
   "source": [
    "graphs, pos_idx, neg_idx = load_data('BA-2motif')\n",
    "\n",
    "model_dirs = ['gin-2-ba2motif.torch',\n",
    "            'gin-3-ba2motif.torch',\n",
    "            'gin-4-ba2motif.torch',\n",
    "            'gin-5-ba2motif.torch',\n",
    "            'gin-6-ba2motif.torch',\n",
    "            'gin-7-ba2motif.torch']\n",
    "\n",
    "# g = graphs[44]\n",
    "S = [0,1,2,3]\n",
    "alpha = 0.\n",
    "verbose = False\n",
    "num_samples = 50\n",
    "sample_idx = np.random.choice(len(graphs),num_samples,replace=False)\n",
    "\n",
    "model_times = []\n",
    "\n",
    "for model_dir in tqdm(model_dirs):\n",
    "    ts = []\n",
    "    nn = torch.load('models/'+model_dir)\n",
    "    \n",
    "    t_temp = 0\n",
    "    for j, i in tqdm(enumerate(sample_idx)):\n",
    "        g = graphs[i]\n",
    "        timea = time.time()\n",
    "        subgraph_original(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "        timeb = time.time()\n",
    "        t_temp += timeb - timea\n",
    "        \n",
    "    ts.append(t_temp / num_samples)\n",
    "\n",
    "    t_temp = 0\n",
    "    for i in tqdm(sample_idx):\n",
    "        g = graphs[i]\n",
    "        timea = time.time()\n",
    "        subgraph_mp_transcription(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "        timeb = time.time()\n",
    "        t_temp += timeb - timea\n",
    "    ts.append(t_temp / num_samples)\n",
    "\n",
    "    t_temp = 0\n",
    "    for i in tqdm(sample_idx):\n",
    "        g = graphs[i]\n",
    "        timea = time.time()\n",
    "        subgraph_mp_forward_hook(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "        timeb = time.time()\n",
    "        t_temp += timeb - timea\n",
    "    ts.append(t_temp / num_samples)\n",
    "\n",
    "    model_times.append(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting\n",
    "num_layers = np.arange(1,len(model_times)+1)\n",
    "model_times = np.array(model_times)\n",
    "fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, gridspec_kw={'height_ratios': [3, 1]}, figsize=(3.5,4))\n",
    "fig.subplots_adjust(hspace=0.05)  # adjust space between axes\n",
    "\n",
    "plt.rc('legend', fontsize=14.5) \n",
    "ax2.spines['top'].set_visible(False)\n",
    "\n",
    "for item in ([ax1.title, ax1.xaxis.label, ax1.yaxis.label] +\n",
    "             ax1.get_xticklabels() + ax1.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "for item in ([ax2.title, ax2.xaxis.label, ax2.yaxis.label] +\n",
    "             ax2.get_xticklabels() + ax2.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "\n",
    "ax1.set_ylabel(\"Time (s)\")\n",
    "ax2.set_xlabel(r'$L$')\n",
    "plt.xticks(num_layers, [str(i) if i % 2 == 1 else '' for i in range(2,len(model_times)+2)])\n",
    "\n",
    "ax1.plot(num_layers, model_times[:,0], 'c--')\n",
    "line2, = ax1.plot(num_layers, [0]*len(num_layers), 'r-')\n",
    "ax1.legend(['GNN-LRP naive', 'sGNN-LRP'])\n",
    "line2.remove()\n",
    "ax2.plot(num_layers, model_times[:,0], 'c--')\n",
    "ax2.plot(num_layers, model_times[:,2], 'r-')\n",
    "ax1.spines.bottom.set_visible(False)\n",
    "ax2.spines.top.set_visible(False)\n",
    "ax1.xaxis.tick_top()\n",
    "ax1.tick_params(labeltop=False)  # don't put tick labels at the top\n",
    "\n",
    "ax1.set_ylim(0.005)  # outliers only\n",
    "ax2.set_ylim(-0,0.013)\n",
    "\n",
    "d = .5  # proportion of vertical to horizontal extent of the slanted line\n",
    "kwargs = dict(marker=[(-1, -d), (1, d)], markersize=12,\n",
    "              linestyle=\"none\", color='k', mec='k', mew=1, clip_on=False)\n",
    "ax1.plot([0, 1], [0, 0], transform=ax1.transAxes, **kwargs)\n",
    "ax2.plot([0, 1], [1, 1], transform=ax2.transAxes, **kwargs)\n",
    "\n",
    "plt.savefig('imgs/time_consumption_L.eps', dpi=600, format='eps', bbox_inches='tight')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### |S| dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graphs, pos_idx, neg_idx = load_data('BA-2motif')\n",
    "\n",
    "model_dir = 'gin-3-ba2motif.torch'\n",
    "\n",
    "alpha = 0.\n",
    "verbose = False\n",
    "num_samples = 50\n",
    "sample_idx = np.random.choice(len(graphs),num_samples,replace=False)\n",
    "nn = torch.load('models/'+model_dir)\n",
    "\n",
    "model_times = []\n",
    "\n",
    "for size_S in tqdm(range(25)):\n",
    "    S = list(range(size_S))\n",
    "\n",
    "    ts = []\n",
    "    \n",
    "    t_temp = 0\n",
    "    for j, i in tqdm(enumerate(sample_idx)):\n",
    "        g = graphs[i]\n",
    "        timea = time.time()\n",
    "        subgraph_original(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "        timeb = time.time()\n",
    "        t_temp += timeb - timea\n",
    "        \n",
    "    ts.append(t_temp / num_samples)\n",
    "\n",
    "    t_temp = 0\n",
    "    for i in tqdm(sample_idx):\n",
    "        g = graphs[i]\n",
    "        timea = time.time()\n",
    "        subgraph_mp_transcription(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "        timeb = time.time()\n",
    "        t_temp += timeb - timea\n",
    "    ts.append(t_temp / num_samples)\n",
    "\n",
    "    t_temp = 0\n",
    "    for i in tqdm(sample_idx):\n",
    "        g = graphs[i]\n",
    "        timea = time.time()\n",
    "        subgraph_mp_forward_hook(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "        timeb = time.time()\n",
    "        t_temp += timeb - timea\n",
    "    ts.append(t_temp / num_samples)\n",
    "\n",
    "    model_times.append(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting\n",
    "num_layers = np.arange(1,len(model_times)+1)\n",
    "model_times = np.array(model_times)\n",
    "fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, gridspec_kw={'height_ratios': [3, 1]}, figsize=(3.5,4))\n",
    "fig.subplots_adjust(hspace=0.05)  # adjust space between axes\n",
    "ax1.yaxis.tick_right()\n",
    "ax1.yaxis.set_label_position(\"right\")\n",
    "ax2.yaxis.tick_right()\n",
    "\n",
    "plt.rc('legend', fontsize=12)\n",
    "ax2.spines['top'].set_visible(False)\n",
    "\n",
    "for item in ([ax1.title, ax1.xaxis.label, ax1.yaxis.label] +\n",
    "             ax1.get_xticklabels() + ax1.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "for item in ([ax2.title, ax2.xaxis.label, ax2.yaxis.label] +\n",
    "             ax2.get_xticklabels() + ax2.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "\n",
    "ax2.set_xlabel(r'$|\\mathcal{S}|$')\n",
    "plt.xticks(num_layers, [str(i) if i % 3 == 1 else '' for i in range(1,len(model_times)+1)])\n",
    "\n",
    "ax1.plot(num_layers, model_times[:,0], 'c--')\n",
    "line2, = ax1.plot(num_layers, [0]*len(num_layers), 'r-')\n",
    "line2.remove()\n",
    "ax2.plot(num_layers, model_times[:,0], 'c--')\n",
    "ax2.plot(num_layers, model_times[:,2], 'r-')\n",
    "\n",
    "ax1.spines.bottom.set_visible(False)\n",
    "ax2.spines.top.set_visible(False)\n",
    "ax1.xaxis.tick_top()\n",
    "ax1.tick_params(labeltop=False)  # don't put tick labels at the top\n",
    "\n",
    "ax1.set_ylim(0.05)  # outliers only\n",
    "ax2.set_ylim(-0,0.01)\n",
    "\n",
    "d = .5  # proportion of vertical to horizontal extent of the slanted line\n",
    "kwargs = dict(marker=[(-1, -d), (1, d)], markersize=12,\n",
    "              linestyle=\"none\", color='k', mec='k', mew=1, clip_on=False)\n",
    "ax1.plot([0, 1], [0, 0], transform=ax1.transAxes, **kwargs)\n",
    "ax2.plot([0, 1], [1, 1], transform=ax2.transAxes, **kwargs)\n",
    "\n",
    "plt.savefig('imgs/time_consumption_S.eps', dpi=600, format='eps', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare time consumptions of the three methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.\n",
    "S = np.arange(5)\n",
    "verbose = True\n",
    "dataset_model_dirs = [['BA-2motif','gin-3-ba2motif.torch'],\n",
    "                      ['BA-2motif','gin-5-ba2motif.torch'],\n",
    "                      ['BA-2motif','gin-7-ba2motif.torch'],\n",
    "                      ['MUTAG', 'gin-3-mutag.torch'],\n",
    "                      ['Mutagenicity', 'gin-3-mutagenicity.torch'],\n",
    "                      ['REDDIT-BINARY', 'gin-5-reddit.torch'],\n",
    "                      ['Graph-SST2', 'gcn-3-sst2graph.torch']]\n",
    "\n",
    "efficiency_result_originals = []\n",
    "efficiency_result_mp_transcs = []\n",
    "efficiency_result_forward_hooks = []\n",
    "\n",
    "for dataset, model_dir in dataset_model_dirs:\n",
    "    print(dataset, model_dir)\n",
    "    graphs, pos_idx, neg_idx = load_data(dataset)\n",
    "\n",
    "    nn = torch.load('models/'+model_dir)\n",
    "    s = StringIO()\n",
    "    old_stdout = sys.stdout\n",
    "    sys.stdout = s\n",
    "    lists = []\n",
    "\n",
    "    for _ in tqdm(range(50)):\n",
    "        i = np.random.randint(len(graphs))\n",
    "        g = graphs[i]\n",
    "\n",
    "        while g.nbnodes < 5:\n",
    "            i = np.random.randint(len(graphs))\n",
    "            g = graphs[i]\n",
    "\n",
    "        for _ in range(3):\n",
    "            subgraph_original(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "            subgraph_mp_transcription(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "            subgraph_mp_forward_hook(nn, g, S, alpha=alpha, gamma=None, verbose=verbose)\n",
    "    lists = s.getvalue().splitlines()\n",
    "\n",
    "    sys.stdout = old_stdout\n",
    "\n",
    "    lists_ = [[float(data.split(': ')[-1].split(',')[0]) for data in l.split('\\t')[1:]] for l in lists]\n",
    "\n",
    "    efficiency_result_original = pd.DataFrame(lists_[::3],columns=['nbnodes','layers','overhead','subrel'])\n",
    "    efficiency_result_mp_transc = pd.DataFrame(lists_[1::3],columns=['nbnodes','layers','overhead','subrel'])\n",
    "    efficiency_result_forward_hook = pd.DataFrame(lists_[2::3],columns=['nbnodes','layers','forward1','backward1'])\n",
    "\n",
    "    efficiency_result_originals.append(efficiency_result_original.mean(axis=0))\n",
    "    efficiency_result_mp_transcs.append(efficiency_result_mp_transc.mean(axis=0))\n",
    "    efficiency_result_forward_hooks.append(efficiency_result_forward_hook.mean(axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test different alphas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ACC AUC AUPC AUFC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "verbose = False\n",
    "test_samples = 200\n",
    "\n",
    "# BA-2motif\n",
    "dataset = 'BA-2motif'\n",
    "graphs, pos_idx, neg_idx = load_data('BA-2motif')\n",
    "model_dir = \"models/gin-3-ba2motif.torch\"; num_layer= 3\n",
    "nn = torch.load(model_dir)\n",
    "\n",
    "# # MUTAG\n",
    "# dataset = 'MUTAG'\n",
    "# graphs, pos_idx, neg_idx = load_data('MUTAG')\n",
    "# model_dir = \"models/gin-3-mutag.torch\"; num_layer= 3\n",
    "# nn = torch.load(model_dir)\n",
    "\n",
    "# # Graph-SST2\n",
    "# dataset = 'Graph-SST2'\n",
    "# graphs, pos_idx, neg_idx = load_data('Graph-SST2')\n",
    "# model_dir = \"models/gcn-3-sst2graph.torch\"; num_layer= 3\n",
    "# nn = torch.load(model_dir)\n",
    "\n",
    "message = '{}\\nModel depth: {}, model: {}, nb of samples: {}\\n'.format(model_dir,num_layer, 'gin', test_samples)\n",
    "print(message)\n",
    "\n",
    "messages = []\n",
    "test_sample_idx = []\n",
    "for alpha in np.arange(0.0,1.01,0.05):\n",
    "    if dataset == 'BA-2motif':\n",
    "        stats = {'acc': [], 'auc': [], 'auac': [], 'aupc': [], 'acs': [], 'pcs': [], 'label': []}\n",
    "    else:\n",
    "        stats = {'auac': [], 'aupc': [], 'acs': [], 'pcs': [], 'label': []}\n",
    "    cnt_pos = test_samples / 2\n",
    "    cnt_neg = test_samples / 2\n",
    "    start = time.time()\n",
    "    random_sample = True if test_sample_idx == [] else False\n",
    "    i = 0\n",
    "    while cnt_pos > 0 or cnt_neg > 0:\n",
    "        if random_sample:\n",
    "            idx = np.random.randint(len(graphs))\n",
    "            g = graphs[idx]\n",
    "            if g.nbnodes < 3: continue\n",
    "            if g.label == 0:\n",
    "                if cnt_pos == 0: continue\n",
    "                else: cnt_pos -= 1\n",
    "            else:\n",
    "                if cnt_neg == 0: continue\n",
    "                else: cnt_neg -= 1\n",
    "            test_sample_idx.append(idx)\n",
    "        else:\n",
    "            if i >= len(test_sample_idx): break\n",
    "            g = graphs[test_sample_idx[i]]\n",
    "            i += 1\n",
    "\n",
    "        gr_tr, all_feats = create_ground_truth(g)\n",
    "\n",
    "        # mode = 'prun'\n",
    "        mode = 'extr'\n",
    "\n",
    "        H, transforms = get_H_transform(g.get_adj(),nn,gammas=None)\n",
    "        fo = get_feat_order_local_best_guess(nn, g, alpha, H, transforms, mode='extr')\n",
    "        \n",
    "        if mode == 'extr':\n",
    "            acc, auc = get_stats(gr_tr, fo, all_feats)\n",
    "            auac, acs = get_auac_aupc(nn, g, fo, task=mode, use_softmax=True)\n",
    "            aupc, pcs = [], []\n",
    "        else:\n",
    "            acc, auc = [], []\n",
    "            aupc, pcs = get_auac_aupc(nn, g, fo, task=mode, use_softmax=False)\n",
    "            auac, acs = [], []\n",
    "\n",
    "        if dataset == 'BA-2motif':\n",
    "            stats['acc'].append(acc)\n",
    "            stats['auc'].append(auc)\n",
    "        stats['auac'].append(auac)\n",
    "        stats['aupc'].append(aupc)\n",
    "        stats['acs'].append(acs)\n",
    "        stats['pcs'].append(pcs)\n",
    "        stats['label'].append(g.label)\n",
    "\n",
    "    message = 'alpha {}, took {} s\\n'.format(alpha, round(time.time() - start, 4))\n",
    "    for key, lst in  stats.items():\n",
    "        if key in ['acs', 'pcs', 'label']: continue\n",
    "        elif key in ['acc', 'auc']:\n",
    "            mstat = round(sum(lst)/len(lst), 4)\n",
    "            message += '\\t {} : {}'.format(key, mstat)\n",
    "        elif key in ['auac', 'aupc']:\n",
    "            lst = np.array(lst)\n",
    "            pos = list(np.argwhere(np.array(stats['label']) == 0).flatten())\n",
    "            mstat = round(np.sum(lst[pos])/len(pos), 4)\n",
    "            message += '\\t {}_pos : {}'.format(key, mstat)\n",
    "            neg = list(np.argwhere(np.array(stats['label']) == 1).flatten())\n",
    "            mstat = round(np.sum(lst[neg])/len(neg), 4)\n",
    "            message += '\\t {}_neg : {}'.format(key, mstat)\n",
    "            mstat = round(np.sum(lst)/len(lst), 4)\n",
    "            message += '\\t {} : {}'.format(key, mstat)\n",
    "    message += '\\n'\n",
    "\n",
    "    print(message)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot acc auc\n",
    "\n",
    "with open('evaluation_results/result_local_best_guess.txt','r') as f:\n",
    "    s = f.readlines()[2:]\n",
    "stats = {}\n",
    "for i in range(len(s)//2):\n",
    "    alpha = float(s[i * 2].split(',')[0].split(' ')[-1])\n",
    "    for sss in s[i * 2 + 1].split('\\n')[0].split('\\t')[1:]:\n",
    "        sss = sss.split(':')\n",
    "        if sss[0].strip() not in stats:\n",
    "            stats[sss[0].strip()] = []\n",
    "        stats[sss[0].strip()].append(float(sss[1]))\n",
    "fig, ax = plt.subplots(figsize=(5,1.5))\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['top'].set_visible(False)\n",
    "\n",
    "for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] +\n",
    "             ax.get_xticklabels() + ax.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "plt.rc('legend', fontsize=15)  \n",
    "\n",
    "plt.plot(np.arange(0,1.01,0.05),stats['acc'], 'r-')\n",
    "plt.plot(np.arange(0,1.01,0.05),stats['auc'], 'b--')\n",
    "plt.legend([\"Accuracy\", \"AUROC\"])\n",
    "plt.xlim(0,1)\n",
    "plt.ylim(top=1)\n",
    "plt.xlabel(r'$\\alpha$')\n",
    "# plt.xticks(np.arange(0,1.01,0.05))\n",
    "plt.savefig('imgs/ba2motif_acc_auc.eps', dpi=600, format='eps', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot aupc aufc\n",
    "\n",
    "# data_dir = 'evaluation_results/mutag_prun_result.txt'; dataset = 'mutag'\n",
    "# data_dir = 'evaluation_results/mutag_acti_result.txt'; dataset = 'mutag'\n",
    "# data_dir = 'evaluation_results/graphsst2_prun_result.txt'; dataset = 'graphsst2'\n",
    "data_dir = 'evaluation_results/graphsst2_acti_result.txt'; dataset = 'graphsst2'\n",
    "\n",
    "with open(data_dir,'r') as f:\n",
    "    s = f.readlines()[2:]\n",
    "stats = {}\n",
    "for i in range(len(s)//2):\n",
    "    alpha = float(s[i * 2].split(',')[0].split(' ')[-1])\n",
    "    for sss in s[i * 2 + 1].split('\\n')[0].split('\\t')[1:]:\n",
    "        sss = sss.split(':')\n",
    "        if sss[0].strip() not in stats:\n",
    "            stats[sss[0].strip()] = []\n",
    "        stats[sss[0].strip()].append(float(sss[1]))\n",
    "\n",
    "fig = plt.figure(figsize=(3,3))\n",
    "ax1 = fig.add_subplot(111)\n",
    "\n",
    "for item in ([ax1.title, ax1.xaxis.label, ax1.yaxis.label] +\n",
    "             ax1.get_xticklabels() + ax1.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "\n",
    "if 'aupc_pos' in stats.keys():\n",
    "    key = 'aupc_pos'\n",
    "    ax1.annotate(xy=(np.argmin(stats[key])*0.05,np.min(stats[key])), text='', \n",
    "                 xytext=(np.argmin(stats[key])*0.05,np.min(stats[key])-0.15), arrowprops={'arrowstyle':'->','color':'red'})\n",
    "else:\n",
    "    key = 'auac_pos'\n",
    "    ax1.annotate(xy=(np.argmax(stats[key])*0.05,np.max(stats[key])), text='', \n",
    "                 xytext=(np.argmax(stats[key])*0.05,np.max(stats[key])-0.25), arrowprops={'arrowstyle':'->','color':'red'})\n",
    "ax1.plot(np.arange(0,1.01,0.05),stats[key], 'r-')\n",
    "\n",
    "ax1.set_ylabel('positive')\n",
    "ax1.yaxis.label.set_color('red')\n",
    "ax1.tick_params(axis='y', colors='red')\n",
    "ax1.set_xlabel(r'$\\alpha$')\n",
    "\n",
    "ax2 = ax1.twinx()\n",
    "for item in ([ax2.title, ax2.xaxis.label, ax2.yaxis.label] +\n",
    "             ax2.get_xticklabels() + ax2.get_yticklabels()):\n",
    "    item.set_fontsize(15)\n",
    "if 'aupc_neg' in stats.keys():\n",
    "    key = 'aupc_neg'\n",
    "    ax2.annotate(xy=(np.argmin(stats[key])*0.05,np.min(stats[key])), text='', \n",
    "                 xytext=(np.argmin(stats[key])*0.05,np.min(stats[key])-0.15), arrowprops={'arrowstyle':'->','color':'blue'})\n",
    "else:\n",
    "    key = 'auac_neg'\n",
    "    ax2.annotate(xy=(np.argmax(stats[key])*0.05,np.max(stats[key])), text='', \n",
    "                 xytext=(np.argmax(stats[key])*0.05,np.max(stats[key])-0.25), \n",
    "                 arrowprops={'arrowstyle':'->','color':'blue'})\n",
    "\n",
    "ax2.set_ylabel('negative')\n",
    "ax2.yaxis.label.set_color('blue')\n",
    "ax2.tick_params(axis='y', colors='blue')\n",
    "\n",
    "ax2.plot(np.arange(0,1.01,0.05),stats[key], 'b--')\n",
    "ax2.set_xlim(0,1)\n",
    "ax2.set_xlabel(r'$\\alpha$')\n",
    "\n",
    "if 'aupc_neg' in stats.keys():\n",
    "    plt.savefig('imgs/'+dataset+'_aupc.eps', dpi=600, format='eps',bbox_inches='tight')\n",
    "else:\n",
    "    plt.savefig('imgs/'+dataset+'_auac.eps', dpi=600, format='eps',bbox_inches='tight')\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "8806367dad1e0a2cfd7da8c1ed95d51b1c4f8061d16620a9f0a678416f1237e6"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}