GATv2_p2p_normalization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "collapsed_sections": [
        "4KD-KUsKDiLE",
        "igYJ4y99D-bU",
        "z_dcpDhREijU",
        "fTEnWpOOEZW9",
        "8rh6Ca1XE6ar",
        "HWrzKUrFFcNV",
        "RRCYp7yuLxMe"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TruongDuyLongPTIT/Experiment2/blob/main/GATv2_p2p_normalization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# I. Pip & Import."
      ],
      "metadata": {
        "id": "4KD-KUsKDiLE"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H7D6x31EU0fi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e307e88c-b170-4884-d6f9-7820ed1d0aea"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 7.9 MB 4.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.5 MB 4.3 MB/s \n",
            "\u001b[?25hLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting torch-geometric\n",
            "  Downloading torch_geometric-2.1.0.post1.tar.gz (467 kB)\n",
            "\u001b[K     |████████████████████████████████| 467 kB 4.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (4.64.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (1.21.6)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (1.7.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (2.11.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (2.23.0)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (3.0.9)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from torch-geometric) (1.0.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->torch-geometric) (2.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torch-geometric) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torch-geometric) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torch-geometric) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torch-geometric) (2022.9.24)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->torch-geometric) (3.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->torch-geometric) (1.2.0)\n",
            "Building wheels for collected packages: torch-geometric\n",
            "  Building wheel for torch-geometric (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for torch-geometric: filename=torch_geometric-2.1.0.post1-py3-none-any.whl size=689859 sha256=301d6b1724293daea7bca07e6d5021685762df0860870ca6b9eaf80cee506be9\n",
            "  Stored in directory: /root/.cache/pip/wheels/d1/cb/43/f7f2e472de4d7cff31bceddadc36d634e1e545fbc17961c282\n",
            "Successfully built torch-geometric\n",
            "Installing collected packages: torch-geometric\n",
            "Successfully installed torch-geometric-2.1.0.post1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Looking in links: https://data.pyg.org/whl/torch-1.12.1+cu113.html\n",
            "Collecting torch-cluster\n",
            "  Downloading https://data.pyg.org/whl/torch-1.12.0%2Bcu113/torch_cluster-1.6.0-cp37-cp37m-linux_x86_64.whl (2.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.4 MB 1.7 MB/s \n",
            "\u001b[?25hInstalling collected packages: torch-cluster\n",
            "Successfully installed torch-cluster-1.6.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting deepdish==0.3.5\n",
            "  Downloading deepdish-0.3.5-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from deepdish==0.3.5) (1.7.3)\n",
            "Requirement already satisfied: tables in /usr/local/lib/python3.7/dist-packages (from deepdish==0.3.5) (3.7.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from deepdish==0.3.5) (1.21.6)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from tables->deepdish==0.3.5) (21.3)\n",
            "Requirement already satisfied: numexpr>=2.6.2 in /usr/local/lib/python3.7/dist-packages (from tables->deepdish==0.3.5) (2.8.4)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->tables->deepdish==0.3.5) (3.0.9)\n",
            "Installing collected packages: deepdish\n",
            "Successfully installed deepdish-0.3.5\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting torch-optimizer\n",
            "  Downloading torch_optimizer-0.3.0-py3-none-any.whl (61 kB)\n",
            "\u001b[K     |████████████████████████████████| 61 kB 417 kB/s \n",
            "\u001b[?25hCollecting pytorch-ranger>=0.1.1\n",
            "  Downloading pytorch_ranger-0.1.1-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: torch>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from torch-optimizer) (1.12.1+cu113)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.5.0->torch-optimizer) (4.1.1)\n",
            "Installing collected packages: pytorch-ranger, torch-optimizer\n",
            "Successfully installed pytorch-ranger-0.1.1 torch-optimizer-0.3.0\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import os.path as osp\n",
        "import torch\n",
        "import sys\n",
        "\n",
        "os.environ['TORCH'] = torch.__version__\n",
        "\n",
        "!pip install -q torch-scatter -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install -q torch-sparse -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install torch-geometric\n",
        "!pip install torch-cluster -f https://data.pyg.org/whl/torch-${TORCH}.html\n",
        "!pip install deepdish==0.3.5\n",
        "!pip install torch-optimizer\n",
        "\n",
        "import torch.nn as nn\n",
        "from torch.nn import Parameter\n",
        "from torch_geometric.utils import scatter\n",
        "from torch_geometric.nn.inits import glorot, zeros\n",
        "from torch_scatter import scatter_add\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "import torch_optimizer as optimization\n",
        "\n",
        "\n",
        "from itertools import chain\n",
        "import pickle\n",
        "import h5py\n",
        "import deepdish as dd\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "import inspect\n",
        "import time\n",
        "import math\n",
        "import random"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def time_since(start):\n",
        "    now = time.time()\n",
        "    s = now - start\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    h = math.floor(m / 60)\n",
        "    m -= h * 60\n",
        "    if h == 0:\n",
        "        if m == 0:\n",
        "            return '%ds' % s\n",
        "        else:\n",
        "            return '%dm %ds' % (m, s)\n",
        "    else:\n",
        "        return '%dh %dm %ds' % (h, m, s)\n"
      ],
      "metadata": {
        "id": "X5j9SrCo4uM5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# II. Data"
      ],
      "metadata": {
        "id": "r0VJxZnIDpJT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Data utils\n",
        "def h5group_to_dict(h5group):\n",
        "    group_dict = {k: v[()] for k, v in chain(h5group.items(), h5group.attrs.items())}\n",
        "    return group_dict\n",
        "\n",
        "def sub_dict(full_dict, *keys, to_tensor):\n",
        "    return {k: torch.tensor(full_dict[k]) if to_tensor else full_dict[k] for k in keys if k in full_dict}\n",
        "\n",
        "def build_graph_from_dict_pyg(graph_dict, to_tensor=True):\n",
        "    from torch_geometric.data import Data\n",
        "\n",
        "    g = Data(**sub_dict(graph_dict, 'edge_index', 'x', 'y', 'edge_attr', 'edge_y', to_tensor=to_tensor))\n",
        "    return g\n",
        "\n",
        "# Data loader\n",
        "class GraphDataLoader(DataLoader):\n",
        "    def __init__(self, dataset, batch_size=128, shuffle=False, num_workers=0):\n",
        "\n",
        "        def collate_graph(graph_obj_list):\n",
        "            from torch_geometric.data import Batch\n",
        "            batch = Batch.from_data_list(graph_obj_list)\n",
        "            return batch\n",
        "\n",
        "        super().__init__(\n",
        "            dataset,\n",
        "            batch_size=batch_size,\n",
        "            shuffle=shuffle,\n",
        "            collate_fn=collate_graph,\n",
        "            num_workers=num_workers)\n",
        "\n",
        "# BotnetDataset\n",
        "class BotnetDataset(Dataset):\n",
        "\n",
        "    def __init__(self, name='chord', root='data/botnet', split='train', graph_format='pyg', split_idx=None, add_nfeat_ones=True,\n",
        "                 in_memory=True):\n",
        "        super().__init__()\n",
        "        assert name in ['chord', 'debru', 'kadem', 'leet', 'c2', 'p2p']\n",
        "        assert split in ['train', 'val', 'test']\n",
        "\n",
        "        if isinstance(root, str):\n",
        "            root = osp.expanduser(osp.normpath(root))\n",
        "\n",
        "        self.name = name\n",
        "        self.root = root\n",
        "        self.split = split\n",
        "        self.split_idx = split_idx\n",
        "        self.add_nfeat_ones = add_nfeat_ones\n",
        "\n",
        "        self._graph_format = graph_format\n",
        "        if split == 'train':\n",
        "            self.path = self.processed_paths[0]\n",
        "            self.num_graphs = 768\n",
        "        elif split == 'val':\n",
        "            self.path = self.processed_paths[1]\n",
        "            self.num_graphs = 96\n",
        "        elif split == 'test':\n",
        "            self.path = self.processed_paths[2]\n",
        "            self.num_graphs = 96\n",
        "\n",
        "        if in_memory:\n",
        "            self.data = dd.io.load(self.path)  # dictionary\n",
        "            self.data_type = 'dict'\n",
        "        else:\n",
        "            # self.data = h5py.File(self.path, 'r')\n",
        "            self.data = None    # defer opening file in each process to make multiprocessing work\n",
        "            self.data_type = 'file'\n",
        "            \n",
        "    @property\n",
        "    def processed_dir(self):\n",
        "        return osp.join(self.root, 'processed')\n",
        "\n",
        "    @property\n",
        "    def processed_file_names(self):\n",
        "        return [self.name + '_' + s + '.hdf5' for s in ('train', 'val', 'test')]\n",
        "\n",
        "    @property\n",
        "    def processed_paths(self):\n",
        "        return [osp.join(self.processed_dir, f) for f in self.processed_file_names]\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.num_graphs\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        if self.data_type == 'dict':\n",
        "            graph_dict = self.data[str(index)]\n",
        "        elif self.data_type == 'file':\n",
        "            if self.data is None:\n",
        "                # only open once in each process\n",
        "                self.data = h5py.File(self.path, 'r')\n",
        "            graph_dict = h5group_to_dict(self.data[str(index)])\n",
        "        else:\n",
        "            raise ValueError\n",
        "\n",
        "        # graph_format == 'pyg':\n",
        "        return build_graph_from_dict_pyg(graph_dict)\n",
        "\n",
        "\n",
        "    def __iter__(self):\n",
        "        for i in range(self.num_graphs):\n",
        "            yield self[i]\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}(topology: {self.name} | split: {self.split} | ' \\\n",
        "               f'#graphs: {len(self)} | graph format: {self.graph_format})'\n"
      ],
      "metadata": {
        "id": "JHHcJ7B-4BmU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# III. Measure Performancce"
      ],
      "metadata": {
        "id": "igYJ4y99D-bU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 3.1. Metrics"
      ],
      "metadata": {
        "id": "z_dcpDhREijU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def f1(target, pred, label):\n",
        "    # F1 = 2 * (precision * recall) / (precision + recall)\n",
        "    tp = np.sum((target==label) & (pred==label))\n",
        "    fp = np.sum((target!=label) & (pred==label))\n",
        "    fn = np.sum((pred!=label) & (target==label))\n",
        "    \n",
        "    if tp+fp==0 or tp+fn==0:\n",
        "      return np.nan\n",
        "\n",
        "    precision = tp/(tp+fp)\n",
        "    recall = tp/(tp+fn)\n",
        "    \n",
        "    if precision+recall==0:\n",
        "      return np.nan\n",
        "      \n",
        "    f1 = 2 * (precision * recall) / (precision + recall)\n",
        "    return f1\n",
        "\n",
        "def f1_macro(pred, target):\n",
        "    return np.mean([f1(target, pred, label) for label in range(0, 2)])\n",
        "\n",
        "\n",
        "def accuracy(pred, target):\n",
        "    return (pred == target).sum().item() / len(target)\n",
        "\n",
        "\n",
        "def true_positive(pred, target):\n",
        "    return (target[pred == 1] == 1).sum().item()\n",
        "\n",
        "\n",
        "def false_positive(pred, target):\n",
        "    return (target[pred == 1] == 0).sum().item()\n",
        "\n",
        "\n",
        "def true_negative(pred, target):\n",
        "    return (target[pred == 0] == 0).sum().item()\n",
        "\n",
        "\n",
        "def false_negative(pred, target):\n",
        "    return (target[pred == 0] == 1).sum().item()\n",
        "\n",
        "\n",
        "def recall(pred, target):\n",
        "    try:\n",
        "        return true_positive(pred, target) / (target == 1).sum().item()\n",
        "    except:  # divide by zero\n",
        "        return -1\n",
        "\n",
        "\n",
        "def precision(pred, target):\n",
        "    try:\n",
        "        prec = true_positive(pred, target) / (pred == 1).sum().item()\n",
        "        return prec\n",
        "    except:  # divide by zero\n",
        "        return -1\n",
        "\n",
        "\n",
        "def f1_score(pred, target):\n",
        "    prec = precision(pred, target)\n",
        "    rec = recall(pred, target)\n",
        "    try:\n",
        "        return 2 * (prec * rec) / (prec + rec)\n",
        "    except:\n",
        "        return 0\n",
        "\n",
        "\n",
        "def false_positive_rate(pred, target):\n",
        "    try:\n",
        "        return false_positive(pred, target) / (target == 0).sum().item()\n",
        "    except:  # divide by zero\n",
        "        return -1\n",
        "\n",
        "\n",
        "def false_negative_rate(pred, target):\n",
        "    try:\n",
        "        return false_negative(pred, target) / (target == 1).sum().item()\n",
        "    except:  # divide by zero\n",
        "        return -1\n"
      ],
      "metadata": {
        "id": "BN-QIjMREw-7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 3.2. Evaluation"
      ],
      "metadata": {
        "id": "fTEnWpOOEZW9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def eval_metrics(target, pred_prob, threshold=0.5):\n",
        "    if isinstance(target, torch.Tensor):\n",
        "        target = target.cpu().numpy()\n",
        "    if isinstance(pred_prob, torch.Tensor):\n",
        "        pred_prob = pred_prob.cpu().numpy()\n",
        "\n",
        "    pred = (pred_prob >= threshold).astype(int)\n",
        "\n",
        "    acc = accuracy(pred, target)\n",
        "    fpr = false_positive_rate(pred, target)\n",
        "    fnr = false_negative_rate(pred, target)\n",
        "    rec = recall(pred, target)\n",
        "    prc = precision(pred, target)\n",
        "    f1 = f1_score(pred, target)\n",
        "    f1macro = f1_macro(pred, target)\n",
        "    result_dict = {'acc': acc, 'fpr': fpr, 'fnr': fnr, 'rec': rec, 'prc': prc, 'f1': f1, 'f1_macro': f1macro}\n",
        "\n",
        "    return result_dict\n",
        "\n",
        "\n",
        "def dict_value_add(dict1, dict2):\n",
        "    result = {key: dict1.get(key, 0) + dict2.get(key, 0)\n",
        "              for key in set(dict1) | set(dict2)}\n",
        "    return result\n",
        "\n",
        "\n",
        "def dict_value_div(dict, n):\n",
        "    result = {key: value / n for key, value in dict.items()}\n",
        "    return result\n",
        "\n",
        "\n",
        "def eval_predictor(dataset, predictor):\n",
        "    result_dict_avg = {}\n",
        "    loss_avg = 0\n",
        "\n",
        "    for data in dataset:\n",
        "        # prediction\n",
        "        try:\n",
        "            pred_prob, loss = predictor(data)\n",
        "            loss_avg += loss\n",
        "        except ValueError:  # if \"too many values to unpack\"\n",
        "            pred_prob = predictor(data)\n",
        "\n",
        "        # get the ground truth target\n",
        "        # graph_format == 'pyg':\n",
        "        target = data.y\n",
        "\n",
        "        # compute the evaluation metrics\n",
        "        result_dict = eval_metrics(target, pred_prob)\n",
        "\n",
        "        result_dict_avg = dict_value_add(result_dict_avg, result_dict)\n",
        "\n",
        "    # average the metrics across all graphs in the dataset as final results\n",
        "    result_dict_avg = dict_value_div(result_dict_avg, len(dataset))\n",
        "    loss_avg = loss_avg / len(dataset)\n",
        "\n",
        "    return result_dict_avg, loss_avg\n",
        "\n",
        "\n",
        "# =================================================================================================================\n",
        "# some examples of the 'predictor' model wrapper to be fed into the above evaluation function (for PyG Data format)\n",
        "# =================================================================================================================\n",
        "class PygRandomPredictor:\n",
        "    def __init__(self):\n",
        "        # torch.manual_seed(0)\n",
        "        pass\n",
        "\n",
        "    def __call__(self, data):\n",
        "        pred_prob = torch.rand(len(data.y))\n",
        "        return pred_prob\n",
        "\n",
        "\n",
        "class PygModelPredictor:\n",
        "    def __init__(self, model, loss_fcn=torch.nn.CrossEntropyLoss()):\n",
        "        self.model = model\n",
        "        self.loss_fcn = loss_fcn\n",
        "        self.device = next(model.parameters()).device\n",
        "\n",
        "    def __call__(self, data):\n",
        "        self.model.eval()\n",
        "        data = data.to(self.device)\n",
        "        with torch.no_grad():\n",
        "            # custom the below line to adjust to your model's input format for forward pass\n",
        "            out = self.model(data.x, data.edge_index)\n",
        "            loss = self.loss_fcn(out, data.y.long())\n",
        "            pred_prob = torch.softmax(out, dim=1)[:, 1]\n",
        "        return pred_prob, loss.float()\n"
      ],
      "metadata": {
        "id": "pqjalkx1EmHc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# IV. Model"
      ],
      "metadata": {
        "id": "5fvRugR_E1QL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 5.1. Activation "
      ],
      "metadata": {
        "id": "8rh6Ca1XE6ar"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def activation(act, negative_slope=0.2):\n",
        "    activations = nn.ModuleDict([\n",
        "        ['lrelu', nn.LeakyReLU(negative_slope)],\n",
        "        ['relu', nn.ReLU()],\n",
        "        ['elu', nn.ELU()],\n",
        "        ['none', nn.Identity()],\n",
        "    ])\n",
        "    return activations[act]"
      ],
      "metadata": {
        "id": "bzEVz1VwFjKK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 5.2. GIN model"
      ],
      "metadata": {
        "id": "HWrzKUrFFcNV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.nn.modules.activation import LeakyReLU\n",
        "import torch.nn as nn\n",
        "\n",
        "class GINModel(nn.Module):\n",
        "    def __init__(self, dim_input_feature, dim_hidden_feature, num_layers, num_classes, non_linear='relu', dropout=0.2):\n",
        "        super().__init__()\n",
        "\n",
        "        self.dim_input_feature = dim_input_feature\n",
        "        self.dim_hidden_feature = dim_hidden_feature\n",
        "\n",
        "        self.num_layers = num_layers\n",
        "        self.num_classes = num_classes\n",
        "        \n",
        "        from torch.nn import Linear, Sequential, BatchNorm1d, ReLU, Dropout\n",
        "        from torch_geometric.nn import GATv2Conv, GINConv, TransformerConv\n",
        "        \n",
        "        self.gin_net = nn.ModuleList()\n",
        "        for i in range(self.num_layers):\n",
        "          if i == 0:\n",
        "            self.gin_net.append(GATv2Conv(self.dim_input_feature, self.dim_hidden_feature, heads=1, concat=False))\n",
        "          else:\n",
        "            self.gin_net.append(GATv2Conv(self.dim_hidden_feature, self.dim_hidden_feature, heads=1, concat=False))\n",
        "\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "        self.residuals = nn.ModuleList()\n",
        "        for i in range(self.num_layers):\n",
        "          if i == 0:\n",
        "            self.residuals.append(nn.Linear(self.dim_input_feature, self.dim_hidden_feature, bias=True))\n",
        "          else:\n",
        "            self.residuals.append(nn.Identity())\n",
        "\n",
        "        self.num_residuals = len(self.residuals)\n",
        "\n",
        "        self.non_linear = activation(non_linear)\n",
        "\n",
        "        # self.final_type == 'proj':\n",
        "        self.final = nn.Linear(self.dim_hidden_feature, num_classes)\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        for net in self.gin_net:\n",
        "            net.reset_parameters()\n",
        "        # self.residual_hop = 1\n",
        "        for net in self.residuals:\n",
        "            net.reset_parameters()\n",
        "        # self.final_type != 'none':\n",
        "        self.final.reset_parameters()\n",
        "\n",
        "    def forward(self, x, edge_index):\n",
        "        xr = None\n",
        "        add_xr_at = -1\n",
        "\n",
        "        for n, net in enumerate(self.gin_net):\n",
        "            # pass to a GIN layer with non-linear activation\n",
        "            xo = net(x, edge_index)\n",
        "            xo = self.dropout(xo)\n",
        "            # deal with residual connections\n",
        "            # self.residual_hop = 1\n",
        "            if n < self.num_residuals:\n",
        "                xr = self.residuals[n](x)\n",
        "                xo = self.non_linear(xo + xr)\n",
        "\n",
        "            x = xo\n",
        "        # size of x: (B * N, dim_hidden_feature) -> (B * N, num_classes)\n",
        "        x = self.final(x)\n",
        "\n",
        "        return x\n"
      ],
      "metadata": {
        "id": "vLv8xNFcFqcT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# V. Load data"
      ],
      "metadata": {
        "id": "RRCYp7yuLxMe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data_dir = '/content/drive/Shareddrives/botnetdata/p2p_normalization'\n",
        "data_name = 'p2p' # 'chord', 'debru', 'kadem', 'leet', 'c2', 'p2p'\n",
        "shuffle = False\n",
        "\n",
        "# ========== load the dataset\n",
        "print('loading dataset...')\n",
        "\n",
        "train_ds = BotnetDataset(name=data_name, root=data_dir, split='train',\n",
        "                         in_memory=True, graph_format='pyg')\n",
        "val_ds = BotnetDataset(name=data_name, root=data_dir, split='val',\n",
        "                       in_memory=False, graph_format='pyg')\n",
        "test_ds = BotnetDataset(name=data_name, root=data_dir, split='test',\n",
        "                        in_memory=False, graph_format='pyg')\n"
      ],
      "metadata": {
        "id": "msBAy6CKLwl8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "730e45e5-9c12-45bc-c756-caeb2b176d55"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "loading dataset...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 1\n",
        "train_loader = GraphDataLoader(train_ds, batch_size=batch_size, shuffle=bool(shuffle), num_workers=0)\n"
      ],
      "metadata": {
        "id": "q23YTuSlmot5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# VI. Train"
      ],
      "metadata": {
        "id": "PSY11P9WEX5a"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ============== some default parameters =============\n",
        "devid = 0\n",
        "seed = 0\n",
        "logmode = 'w'\n",
        "log_interval = 96\n",
        "\n",
        "dim_input_feature = 1\n",
        "dim_hidden_feature = 32\n",
        "act = 'relu' # 'none', 'lrelu', 'relu', 'elu'\n",
        "\n",
        "num_layers = 12\n",
        "num_classes = 2 \n",
        "\n",
        "dropout = 0.0\n",
        "bias = True\n",
        "\n",
        "lr = 0.005 # learning rate\n",
        "weight_decay = 5e-4\n",
        "epochs = 50\n",
        "save_dir = './saved_models'\n",
        "save_name = \"GIN_model.pt\"\n",
        "# ====================================================\n",
        "\n",
        "def train(model, train_loader, val_dataset, test_dataset, optimizer, scheduler=None):\n",
        "    device = next(model.parameters()).device\n",
        "    predictor = PygModelPredictor(model)\n",
        "\n",
        "    best_epoch = 0\n",
        "    max_f1_score = 0\n",
        "    start = time.time()\n",
        "    for ep in range(epochs):\n",
        "        loss_avg_train = 0\n",
        "        num_train_graph = 0\n",
        "        model.train()\n",
        "        for n, batch in enumerate(train_loader):\n",
        "            batch.to(device)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            x = model(batch.x, batch.edge_index)\n",
        "            loss = criterion(x, batch.y.long())\n",
        "\n",
        "            loss_avg_train += float(loss)\n",
        "            num_train_graph += batch.num_graphs\n",
        "\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            if num_train_graph % log_interval == 0 or n == len(train_loader) - 1:\n",
        "                with torch.no_grad():\n",
        "                    # pred = x.argmax(dim=1)\n",
        "                    pred_prob = torch.softmax(x, dim=1)[:, 1]\n",
        "                    y = batch.y.long()\n",
        "                    result_dict = eval_metrics(y, pred_prob)\n",
        "                print(f'epoch: {ep + 1}, passed number of graphs: {num_train_graph}, '\n",
        "                        f'train running loss: {loss_avg_train / num_train_graph:.5f} (passed time: {time_since(start)})')\n",
        "                print(' ' * 10 + ', '.join(['{}: {:.5f}'.format(k, v) for k, v in result_dict.items()]))\n",
        "\n",
        "        result_dict_avg, loss_avg = eval_predictor(val_dataset, predictor)\n",
        "        print(f'Validation --- epoch: {ep + 1}, loss: {loss_avg:.5f}')\n",
        "        print(' ' * 10 + ', '.join(['{}: {:.5f}'.format(k, v) for k, v in result_dict_avg.items()]))\n",
        "\n",
        "        if scheduler is not None:\n",
        "            scheduler.step(loss_avg)\n",
        "\n",
        "        if result_dict_avg['f1'] > max_f1_score:\n",
        "            save_name = f\"GIN_{ep}: {result_dict_avg['f1']}.pt\"\n",
        "            torch.save(model, os.path.join(save_dir, save_name))\n",
        "            print(f'Better model saved at {os.path.join(save_dir, save_name)}.')\n",
        "            best_epoch = ep\n",
        "            max_f1_score = result_dict_avg['f1']\n",
        "\n",
        "    best_model = torch.load(os.path.join(save_dir, save_name))\n",
        "    print('*' * 12 + f' best model obtained after epoch {best_epoch + 1}, '\n",
        "                       f'saved at {os.path.join(save_dir, save_name)} ' + '*' * 12)\n",
        "    \n",
        "    predictor = PygModelPredictor(best_model)\n",
        "\n",
        "    result_dict_avg, loss_avg = eval_predictor(test_dataset, predictor)\n",
        "    print(f'Testing --- loss: {loss_avg:.5f}')\n",
        "    print(' ' * 10 + ', '.join(['{}: {:.5f}'.format(k, v) for k, v in result_dict_avg.items()]))\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    os.makedirs(save_dir, exist_ok=True)\n",
        "\n",
        "    # ========== random seeds and device\n",
        "    random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "\n",
        "    device = torch.device(f'cuda:{devid}') if devid > -1 else torch.device('cpu')\n",
        "\n",
        "    # ========== logging setup\n",
        "    log_name = os.path.splitext(save_name)[0]\n",
        "    # logger = logging_config(__name__, folder=save_dir, name=log_name, filemode=logmode)\n",
        "    # logger = logging_config(os.path.basename(__file__), folder=save_dir, name=log_name, filemode=logmode)\n",
        "\n",
        "    print('python ' + ' '.join(sys.argv))\n",
        "    print('-' * 30)\n",
        "    #logger.info(args)\n",
        "    print('-' * 30)\n",
        "    print(time.ctime())\n",
        "    print('-' * 30)\n",
        "\n",
        "    # ========== define the model, optimizer, and loss\n",
        "\n",
        "    model = GINModel(dim_input_feature,\n",
        "                     dim_hidden_feature,\n",
        "                     num_layers,\n",
        "                     num_classes,\n",
        "                     non_linear=act,\n",
        "                     dropout=dropout)\n",
        "\n",
        "    print('model ' + '-' * 10)\n",
        "    print(repr(model))\n",
        "    model.to(device)\n",
        "\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
        "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.25, patience=1)\n",
        "\n",
        "    # ========== train the model\n",
        "    print(f\"Batch size: {batch_size}\")\n",
        "    train(model, train_loader, val_ds, test_ds, optimizer, scheduler)\n"
      ],
      "metadata": {
        "id": "3iFDupzCdBl0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc48346c-d8fe-436d-d05a-d1f25d001ae5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-e485dbaa-acda-4196-a428-9ce33a025e35.json\n",
            "------------------------------\n",
            "------------------------------\n",
            "Fri Nov  4 01:23:44 2022\n",
            "------------------------------\n",
            "model ----------\n",
            "GINModel(\n",
            "  (gin_net): ModuleList(\n",
            "    (0): GATv2Conv(1, 32, heads=1)\n",
            "    (1): GATv2Conv(32, 32, heads=1)\n",
            "    (2): GATv2Conv(32, 32, heads=1)\n",
            "    (3): GATv2Conv(32, 32, heads=1)\n",
            "    (4): GATv2Conv(32, 32, heads=1)\n",
            "    (5): GATv2Conv(32, 32, heads=1)\n",
            "    (6): GATv2Conv(32, 32, heads=1)\n",
            "    (7): GATv2Conv(32, 32, heads=1)\n",
            "    (8): GATv2Conv(32, 32, heads=1)\n",
            "    (9): GATv2Conv(32, 32, heads=1)\n",
            "    (10): GATv2Conv(32, 32, heads=1)\n",
            "    (11): GATv2Conv(32, 32, heads=1)\n",
            "  )\n",
            "  (dropout): Dropout(p=0.0, inplace=False)\n",
            "  (residuals): ModuleList(\n",
            "    (0): Linear(in_features=1, out_features=32, bias=True)\n",
            "    (1): Identity()\n",
            "    (2): Identity()\n",
            "    (3): Identity()\n",
            "    (4): Identity()\n",
            "    (5): Identity()\n",
            "    (6): Identity()\n",
            "    (7): Identity()\n",
            "    (8): Identity()\n",
            "    (9): Identity()\n",
            "    (10): Identity()\n",
            "    (11): Identity()\n",
            "  )\n",
            "  (non_linear): ReLU()\n",
            "  (final): Linear(in_features=32, out_features=2, bias=True)\n",
            ")\n",
            "Batch size: 1\n",
            "epoch: 1, passed number of graphs: 96, train running loss: 0.25658 (passed time: 50s)\n",
            "          acc: 0.97869, fpr: 0.00000, fnr: 1.00000, rec: 0.00000, prc: -1.00000, f1: 0.00000, f1_macro: nan\n",
            "epoch: 1, passed number of graphs: 192, train running loss: 0.18001 (passed time: 1m 39s)\n",
            "          acc: 0.97884, fpr: 0.00000, fnr: 1.00000, rec: 0.00000, prc: -1.00000, f1: 0.00000, f1_macro: nan\n",
            "epoch: 1, passed number of graphs: 288, train running loss: 0.15443 (passed time: 2m 27s)\n",
            "          acc: 0.97924, fpr: 0.00000, fnr: 1.00000, rec: 0.00000, prc: -1.00000, f1: 0.00000, f1_macro: nan\n",
            "epoch: 1, passed number of graphs: 384, train running loss: 0.14138 (passed time: 3m 16s)\n",
            "          acc: 0.97787, fpr: 0.00000, fnr: 1.00000, rec: 0.00000, prc: -1.00000, f1: 0.00000, f1_macro: nan\n",
            "epoch: 1, passed number of graphs: 480, train running loss: 0.12745 (passed time: 4m 4s)\n",
            "          acc: 0.99972, fpr: 0.00006, fnr: 0.01044, rec: 0.98956, prc: 0.99737, f1: 0.99345, f1_macro: 0.99665\n",
            "epoch: 1, passed number of graphs: 576, train running loss: 0.12076 (passed time: 4m 53s)\n",
            "          acc: 0.97854, fpr: 0.00000, fnr: 1.00000, rec: 0.00000, prc: -1.00000, f1: 0.00000, f1_macro: nan\n",
            "epoch: 1, passed number of graphs: 672, train running loss: 0.11643 (passed time: 5m 41s)\n",
            "          acc: 0.97883, fpr: 0.00000, fnr: 1.00000, rec: 0.00000, prc: -1.00000, f1: 0.00000, f1_macro: nan\n",
            "epoch: 1, passed number of graphs: 768, train running loss: 0.10858 (passed time: 6m 29s)\n",
            "          acc: 0.99349, fpr: 0.00019, fnr: 0.29481, rec: 0.70519, prc: 0.98765, f1: 0.82286, f1_macro: 0.90977\n",
            "Validation --- epoch: 1, loss: 0.02425\n",
            "          prc: 0.91068, fpr: 0.00156, rec: 0.64697, fnr: 0.35303, f1: 0.72389, acc: 0.99088, f1_macro: nan\n",
            "Better model saved at ./saved_models/GIN_0: 0.7238947869714817.pt.\n",
            "epoch: 2, passed number of graphs: 96, train running loss: 0.01513 (passed time: 8m 10s)\n",
            "          acc: 0.99963, fpr: 0.00022, fnr: 0.00767, rec: 0.99233, prc: 0.99011, f1: 0.99122, f1_macro: 0.99551\n",
            "epoch: 2, passed number of graphs: 192, train running loss: 0.01359 (passed time: 8m 59s)\n",
            "          acc: 0.99958, fpr: 0.00013, fnr: 0.01414, rec: 0.98586, prc: 0.99403, f1: 0.98993, f1_macro: 0.99486\n",
            "epoch: 2, passed number of graphs: 288, train running loss: 0.01115 (passed time: 9m 47s)\n",
            "          acc: 0.99921, fpr: 0.00023, fnr: 0.02723, rec: 0.97277, prc: 0.98899, f1: 0.98081, f1_macro: 0.99021\n",
            "epoch: 2, passed number of graphs: 384, train running loss: 0.00972 (passed time: 10m 35s)\n",
            "          acc: 0.99965, fpr: 0.00011, fnr: 0.01098, rec: 0.98902, prc: 0.99513, f1: 0.99206, f1_macro: 0.99594\n",
            "epoch: 2, passed number of graphs: 480, train running loss: 0.00839 (passed time: 11m 23s)\n",
            "          acc: 0.99985, fpr: 0.00006, fnr: 0.00424, rec: 0.99576, prc: 0.99739, f1: 0.99657, f1_macro: 0.99825\n",
            "epoch: 2, passed number of graphs: 576, train running loss: 0.00805 (passed time: 12m 12s)\n",
            "          acc: 0.99971, fpr: 0.00020, fnr: 0.00454, rec: 0.99546, prc: 0.99096, f1: 0.99321, f1_macro: 0.99653\n",
            "epoch: 2, passed number of graphs: 672, train running loss: 0.00797 (passed time: 13m 0s)\n",
            "          acc: 0.99445, fpr: 0.00011, fnr: 0.25700, rec: 0.74300, prc: 0.99312, f1: 0.85005, f1_macro: 0.92361\n",
            "epoch: 2, passed number of graphs: 768, train running loss: 0.00801 (passed time: 13m 49s)\n",
            "          acc: 0.99974, fpr: 0.00007, fnr: 0.00881, rec: 0.99119, prc: 0.99672, f1: 0.99394, f1_macro: 0.99691\n",
            "Validation --- epoch: 2, loss: 0.00414\n",
            "          prc: 0.99507, fpr: 0.00011, rec: 0.97398, fnr: 0.02602, f1: 0.98301, acc: 0.99934, f1_macro: 0.99133\n",
            "Better model saved at ./saved_models/GIN_1: 0.9830058595248558.pt.\n",
            "epoch: 3, passed number of graphs: 96, train running loss: 0.00404 (passed time: 15m 10s)\n",
            "          acc: 0.99987, fpr: 0.00005, fnr: 0.00384, rec: 0.99616, prc: 0.99776, f1: 0.99696, f1_macro: 0.99845\n",
            "epoch: 3, passed number of graphs: 192, train running loss: 0.00509 (passed time: 15m 58s)\n",
            "          acc: 0.99970, fpr: 0.00000, fnr: 0.01414, rec: 0.98586, prc: 1.00000, f1: 0.99288, f1_macro: 0.99636\n",
            "epoch: 3, passed number of graphs: 288, train running loss: 0.00505 (passed time: 16m 46s)\n",
            "          acc: 0.99967, fpr: 0.00016, fnr: 0.00820, rec: 0.99180, prc: 0.99245, f1: 0.99212, f1_macro: 0.99598\n",
            "epoch: 3, passed number of graphs: 384, train running loss: 0.00466 (passed time: 17m 35s)\n",
            "          acc: 0.99978, fpr: 0.00005, fnr: 0.00775, rec: 0.99225, prc: 0.99773, f1: 0.99498, f1_macro: 0.99743\n",
            "epoch: 3, passed number of graphs: 480, train running loss: 0.00417 (passed time: 18m 23s)\n",
            "          acc: 0.99984, fpr: 0.00004, fnr: 0.00555, rec: 0.99445, prc: 0.99804, f1: 0.99624, f1_macro: 0.99808\n",
            "epoch: 3, passed number of graphs: 576, train running loss: 0.00438 (passed time: 19m 11s)\n",
            "          acc: 0.99962, fpr: 0.00031, fnr: 0.00357, rec: 0.99643, prc: 0.98620, f1: 0.99129, f1_macro: 0.99555\n",
            "epoch: 3, passed number of graphs: 672, train running loss: 0.00449 (passed time: 20m 0s)\n",
            "          acc: 0.99444, fpr: 0.00014, fnr: 0.25603, rec: 0.74397, prc: 0.99143, f1: 0.85006, f1_macro: 0.92361\n",
            "epoch: 3, passed number of graphs: 768, train running loss: 0.00509 (passed time: 20m 48s)\n",
            "          acc: 0.99980, fpr: 0.00008, fnr: 0.00588, rec: 0.99412, prc: 0.99640, f1: 0.99526, f1_macro: 0.99758\n",
            "Validation --- epoch: 3, loss: 0.00355\n",
            "          prc: 0.99695, fpr: 0.00007, rec: 0.97407, fnr: 0.02593, f1: 0.98397, acc: 0.99938, f1_macro: 0.99183\n",
            "Better model saved at ./saved_models/GIN_2: 0.9839715862606427.pt.\n",
            "epoch: 4, passed number of graphs: 96, train running loss: 0.00351 (passed time: 22m 9s)\n",
            "          acc: 0.99985, fpr: 0.00008, fnr: 0.00352, rec: 0.99648, prc: 0.99648, f1: 0.99648, f1_macro: 0.99820\n",
            "epoch: 4, passed number of graphs: 192, train running loss: 0.00449 (passed time: 22m 58s)\n",
            "          acc: 0.99798, fpr: 0.00200, fnr: 0.00263, rec: 0.99737, prc: 0.91496, f1: 0.95439, f1_macro: 0.97668\n",
            "epoch: 4, passed number of graphs: 288, train running loss: 0.00431 (passed time: 23m 46s)\n",
            "          acc: 0.99977, fpr: 0.00013, fnr: 0.00525, rec: 0.99475, prc: 0.99410, f1: 0.99442, f1_macro: 0.99715\n",
            "epoch: 4, passed number of graphs: 384, train running loss: 0.00386 (passed time: 24m 34s)\n",
            "          acc: 0.99981, fpr: 0.00004, fnr: 0.00678, rec: 0.99322, prc: 0.99805, f1: 0.99563, f1_macro: 0.99777\n",
            "epoch: 4, passed number of graphs: 480, train running loss: 0.00348 (passed time: 25m 23s)\n",
            "          acc: 0.99984, fpr: 0.00014, fnr: 0.00131, rec: 0.99869, prc: 0.99383, f1: 0.99626, f1_macro: 0.99809\n",
            "epoch: 4, passed number of graphs: 576, train running loss: 0.00351 (passed time: 26m 11s)\n",
            "          acc: 0.99962, fpr: 0.00030, fnr: 0.00421, rec: 0.99579, prc: 0.98651, f1: 0.99113, f1_macro: 0.99547\n",
            "epoch: 4, passed number of graphs: 672, train running loss: 0.00381 (passed time: 27m 0s)\n",
            "          acc: 0.99444, fpr: 0.00010, fnr: 0.25796, rec: 0.74204, prc: 0.99354, f1: 0.84957, f1_macro: 0.92337\n",
            "epoch: 4, passed number of graphs: 768, train running loss: 0.00389 (passed time: 27m 48s)\n",
            "          acc: 0.99978, fpr: 0.00004, fnr: 0.00881, rec: 0.99119, prc: 0.99836, f1: 0.99476, f1_macro: 0.99732\n",
            "Validation --- epoch: 4, loss: 0.00373\n",
            "          prc: 0.99935, fpr: 0.00001, rec: 0.97226, fnr: 0.02774, f1: 0.98420, acc: 0.99939, f1_macro: 0.99194\n",
            "Better model saved at ./saved_models/GIN_3: 0.9841963287227858.pt.\n",
            "epoch: 5, passed number of graphs: 96, train running loss: 0.00299 (passed time: 29m 10s)\n",
            "          acc: 0.99989, fpr: 0.00007, fnr: 0.00192, rec: 0.99808, prc: 0.99681, f1: 0.99744, f1_macro: 0.99869\n",
            "epoch: 5, passed number of graphs: 192, train running loss: 0.00369 (passed time: 29m 58s)\n",
            "          acc: 0.99932, fpr: 0.00065, fnr: 0.00230, rec: 0.99770, prc: 0.97089, f1: 0.98411, f1_macro: 0.99188\n",
            "epoch: 5, passed number of graphs: 288, train running loss: 0.00382 (passed time: 30m 46s)\n",
            "          acc: 0.99980, fpr: 0.00018, fnr: 0.00131, rec: 0.99869, prc: 0.99153, f1: 0.99510, f1_macro: 0.99750\n",
            "epoch: 5, passed number of graphs: 384, train running loss: 0.00332 (passed time: 31m 35s)\n",
            "          acc: 0.99984, fpr: 0.00004, fnr: 0.00581, rec: 0.99419, prc: 0.99838, f1: 0.99628, f1_macro: 0.99810\n",
            "epoch: 5, passed number of graphs: 480, train running loss: 0.00297 (passed time: 32m 23s)\n",
            "          acc: 0.99977, fpr: 0.00022, fnr: 0.00033, rec: 0.99967, prc: 0.98998, f1: 0.99481, f1_macro: 0.99734\n",
            "epoch: 5, passed number of graphs: 576, train running loss: 0.00318 (passed time: 33m 11s)\n",
            "          acc: 0.99935, fpr: 0.00062, fnr: 0.00194, rec: 0.99806, prc: 0.97252, f1: 0.98512, f1_macro: 0.99240\n",
            "epoch: 5, passed number of graphs: 672, train running loss: 0.00328 (passed time: 34m 0s)\n",
            "          acc: 0.99444, fpr: 0.00008, fnr: 0.25860, rec: 0.74140, prc: 0.99482, f1: 0.84961, f1_macro: 0.92339\n",
            "epoch: 5, passed number of graphs: 768, train running loss: 0.00321 (passed time: 34m 48s)\n",
            "          acc: 0.99989, fpr: 0.00009, fnr: 0.00098, rec: 0.99902, prc: 0.99609, f1: 0.99756, f1_macro: 0.99875\n",
            "Validation --- epoch: 5, loss: 0.00262\n",
            "          prc: 0.99582, fpr: 0.00009, rec: 0.98251, fnr: 0.01749, f1: 0.98788, acc: 0.99954, f1_macro: 0.99382\n",
            "Better model saved at ./saved_models/GIN_4: 0.9878751650413671.pt.\n",
            "epoch: 6, passed number of graphs: 96, train running loss: 0.00222 (passed time: 36m 9s)\n",
            "          acc: 0.99992, fpr: 0.00007, fnr: 0.00064, rec: 0.99936, prc: 0.99681, f1: 0.99808, f1_macro: 0.99902\n",
            "epoch: 6, passed number of graphs: 192, train running loss: 0.00319 (passed time: 36m 58s)\n",
            "          acc: 0.99647, fpr: 0.00360, fnr: 0.00033, rec: 0.99967, prc: 0.85710, f1: 0.92291, f1_macro: 0.96055\n",
            "epoch: 6, passed number of graphs: 288, train running loss: 0.00294 (passed time: 37m 46s)\n",
            "          acc: 0.99977, fpr: 0.00023, fnr: 0.00033, rec: 0.99967, prc: 0.98929, f1: 0.99445, f1_macro: 0.99717\n",
            "epoch: 6, passed number of graphs: 384, train running loss: 0.00267 (passed time: 38m 34s)\n",
            "          acc: 0.99989, fpr: 0.00004, fnr: 0.00355, rec: 0.99645, prc: 0.99838, f1: 0.99741, f1_macro: 0.99868\n",
            "epoch: 6, passed number of graphs: 480, train running loss: 0.00244 (passed time: 39m 23s)\n",
            "          acc: 0.99978, fpr: 0.00022, fnr: 0.00033, rec: 0.99967, prc: 0.99030, f1: 0.99497, f1_macro: 0.99743\n",
            "epoch: 6, passed number of graphs: 576, train running loss: 0.00240 (passed time: 40m 11s)\n",
            "          acc: 0.99964, fpr: 0.00035, fnr: 0.00097, rec: 0.99903, prc: 0.98435, f1: 0.99163, f1_macro: 0.99572\n",
            "epoch: 6, passed number of graphs: 672, train running loss: 0.00309 (passed time: 40m 59s)\n",
            "          acc: 0.99451, fpr: 0.00004, fnr: 0.25732, rec: 0.74268, prc: 0.99741, f1: 0.85140, f1_macro: 0.92430\n",
            "epoch: 6, passed number of graphs: 768, train running loss: 0.00309 (passed time: 41m 48s)\n",
            "          acc: 0.99992, fpr: 0.00006, fnr: 0.00098, rec: 0.99902, prc: 0.99739, f1: 0.99821, f1_macro: 0.99908\n",
            "Validation --- epoch: 6, loss: 0.00209\n",
            "          prc: 0.99722, fpr: 0.00006, rec: 0.98448, fnr: 0.01552, f1: 0.98954, acc: 0.99961, f1_macro: 0.99467\n",
            "Better model saved at ./saved_models/GIN_5: 0.9895426580892068.pt.\n",
            "epoch: 7, passed number of graphs: 96, train running loss: 0.00201 (passed time: 43m 9s)\n",
            "          acc: 0.99994, fpr: 0.00006, fnr: 0.00000, rec: 1.00000, prc: 0.99713, f1: 0.99856, f1_macro: 0.99927\n",
            "epoch: 7, passed number of graphs: 192, train running loss: 0.00263 (passed time: 43m 57s)\n",
            "          acc: 0.99972, fpr: 0.00028, fnr: 0.00033, rec: 0.99967, prc: 0.98734, f1: 0.99347, f1_macro: 0.99666\n",
            "epoch: 7, passed number of graphs: 288, train running loss: 0.00252 (passed time: 44m 46s)\n",
            "          acc: 0.99983, fpr: 0.00015, fnr: 0.00098, rec: 0.99902, prc: 0.99283, f1: 0.99591, f1_macro: 0.99791\n",
            "epoch: 7, passed number of graphs: 384, train running loss: 0.00224 (passed time: 45m 34s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00161, rec: 0.99839, prc: 0.99903, f1: 0.99871, f1_macro: 0.99934\n",
            "epoch: 7, passed number of graphs: 480, train running loss: 0.00208 (passed time: 46m 22s)\n",
            "          acc: 0.99975, fpr: 0.00024, fnr: 0.00033, rec: 0.99967, prc: 0.98903, f1: 0.99432, f1_macro: 0.99710\n",
            "epoch: 7, passed number of graphs: 576, train running loss: 0.00218 (passed time: 47m 11s)\n",
            "          acc: 0.99939, fpr: 0.00062, fnr: 0.00032, rec: 0.99968, prc: 0.97256, f1: 0.98593, f1_macro: 0.99281\n",
            "epoch: 7, passed number of graphs: 672, train running loss: 0.00254 (passed time: 47m 59s)\n",
            "          acc: 0.99459, fpr: 0.00004, fnr: 0.25346, rec: 0.74654, prc: 0.99742, f1: 0.85394, f1_macro: 0.92559\n",
            "epoch: 7, passed number of graphs: 768, train running loss: 0.00254 (passed time: 48m 47s)\n",
            "          acc: 0.99995, fpr: 0.00003, fnr: 0.00098, rec: 0.99902, prc: 0.99869, f1: 0.99886, f1_macro: 0.99942\n",
            "Validation --- epoch: 7, loss: 0.00191\n",
            "          prc: 0.99487, fpr: 0.00011, rec: 0.98552, fnr: 0.01448, f1: 0.98902, acc: 0.99958, f1_macro: 0.99440\n",
            "epoch: 8, passed number of graphs: 96, train running loss: 0.00161 (passed time: 50m 9s)\n",
            "          acc: 0.99994, fpr: 0.00006, fnr: 0.00000, rec: 1.00000, prc: 0.99713, f1: 0.99856, f1_macro: 0.99927\n",
            "epoch: 8, passed number of graphs: 192, train running loss: 0.00239 (passed time: 50m 57s)\n",
            "          acc: 0.99989, fpr: 0.00011, fnr: 0.00033, rec: 0.99967, prc: 0.99509, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 8, passed number of graphs: 288, train running loss: 0.00238 (passed time: 51m 45s)\n",
            "          acc: 0.99986, fpr: 0.00011, fnr: 0.00131, rec: 0.99869, prc: 0.99477, f1: 0.99673, f1_macro: 0.99833\n",
            "epoch: 8, passed number of graphs: 384, train running loss: 0.00210 (passed time: 52m 34s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 8, passed number of graphs: 480, train running loss: 0.00197 (passed time: 53m 22s)\n",
            "          acc: 0.99981, fpr: 0.00019, fnr: 0.00033, rec: 0.99967, prc: 0.99159, f1: 0.99561, f1_macro: 0.99776\n",
            "epoch: 8, passed number of graphs: 576, train running loss: 0.00193 (passed time: 54m 10s)\n",
            "          acc: 0.99960, fpr: 0.00040, fnr: 0.00032, rec: 0.99968, prc: 0.98217, f1: 0.99084, f1_macro: 0.99532\n",
            "epoch: 8, passed number of graphs: 672, train running loss: 0.00243 (passed time: 54m 59s)\n",
            "          acc: 0.99452, fpr: 0.00000, fnr: 0.25860, rec: 0.74140, prc: 1.00000, f1: 0.85150, f1_macro: 0.92435\n",
            "epoch: 8, passed number of graphs: 768, train running loss: 0.00245 (passed time: 55m 47s)\n",
            "          acc: 0.99996, fpr: 0.00003, fnr: 0.00065, rec: 0.99935, prc: 0.99869, f1: 0.99902, f1_macro: 0.99950\n",
            "Validation --- epoch: 8, loss: 0.00163\n",
            "          prc: 0.99402, fpr: 0.00013, rec: 0.98961, fnr: 0.01039, f1: 0.99110, acc: 0.99965, f1_macro: 0.99546\n",
            "Better model saved at ./saved_models/GIN_7: 0.9910956310491369.pt.\n",
            "epoch: 9, passed number of graphs: 96, train running loss: 0.00142 (passed time: 57m 9s)\n",
            "          acc: 0.99995, fpr: 0.00005, fnr: 0.00000, rec: 1.00000, prc: 0.99777, f1: 0.99888, f1_macro: 0.99943\n",
            "epoch: 9, passed number of graphs: 192, train running loss: 0.00184 (passed time: 57m 57s)\n",
            "          acc: 0.99981, fpr: 0.00018, fnr: 0.00033, rec: 0.99967, prc: 0.99152, f1: 0.99558, f1_macro: 0.99774\n",
            "epoch: 9, passed number of graphs: 288, train running loss: 0.00180 (passed time: 58m 45s)\n",
            "          acc: 0.99988, fpr: 0.00008, fnr: 0.00197, rec: 0.99803, prc: 0.99640, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 9, passed number of graphs: 384, train running loss: 0.00166 (passed time: 59m 34s)\n",
            "          acc: 0.99996, fpr: 0.00002, fnr: 0.00065, rec: 0.99935, prc: 0.99903, f1: 0.99919, f1_macro: 0.99959\n",
            "epoch: 9, passed number of graphs: 480, train running loss: 0.00159 (passed time: 1h 0m 22s)\n",
            "          acc: 0.99988, fpr: 0.00005, fnr: 0.00326, rec: 0.99674, prc: 0.99771, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 9, passed number of graphs: 576, train running loss: 0.00155 (passed time: 1h 1m 10s)\n",
            "          acc: 0.99957, fpr: 0.00043, fnr: 0.00065, rec: 0.99935, prc: 0.98091, f1: 0.99004, f1_macro: 0.99491\n",
            "epoch: 9, passed number of graphs: 672, train running loss: 0.00185 (passed time: 1h 1m 59s)\n",
            "          acc: 0.99457, fpr: 0.00000, fnr: 0.25667, rec: 0.74333, prc: 1.00000, f1: 0.85277, f1_macro: 0.92500\n",
            "epoch: 9, passed number of graphs: 768, train running loss: 0.00187 (passed time: 1h 2m 47s)\n",
            "          acc: 0.99989, fpr: 0.00001, fnr: 0.00490, rec: 0.99510, prc: 0.99967, f1: 0.99738, f1_macro: 0.99866\n",
            "Validation --- epoch: 9, loss: 0.00221\n",
            "          prc: 0.99539, fpr: 0.00010, rec: 0.98153, fnr: 0.01847, f1: 0.98718, acc: 0.99950, f1_macro: 0.99346\n",
            "epoch: 10, passed number of graphs: 96, train running loss: 0.00092 (passed time: 1h 4m 9s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 10, passed number of graphs: 192, train running loss: 0.00175 (passed time: 1h 4m 57s)\n",
            "          acc: 0.99993, fpr: 0.00004, fnr: 0.00131, rec: 0.99869, prc: 0.99803, f1: 0.99836, f1_macro: 0.99916\n",
            "epoch: 10, passed number of graphs: 288, train running loss: 0.00161 (passed time: 1h 5m 45s)\n",
            "          acc: 0.99990, fpr: 0.00008, fnr: 0.00098, rec: 0.99902, prc: 0.99607, f1: 0.99754, f1_macro: 0.99875\n",
            "epoch: 10, passed number of graphs: 384, train running loss: 0.00155 (passed time: 1h 6m 34s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 10, passed number of graphs: 480, train running loss: 0.00152 (passed time: 1h 7m 22s)\n",
            "          acc: 0.99742, fpr: 0.00000, fnr: 0.11941, rec: 0.88059, prc: 1.00000, f1: 0.93650, f1_macro: 0.96759\n",
            "epoch: 10, passed number of graphs: 576, train running loss: 0.00166 (passed time: 1h 8m 10s)\n",
            "          acc: 0.99967, fpr: 0.00031, fnr: 0.00130, rec: 0.99870, prc: 0.98592, f1: 0.99227, f1_macro: 0.99605\n",
            "epoch: 10, passed number of graphs: 672, train running loss: 0.00188 (passed time: 1h 8m 58s)\n",
            "          acc: 0.99457, fpr: 0.00000, fnr: 0.25635, rec: 0.74365, prc: 1.00000, f1: 0.85298, f1_macro: 0.92511\n",
            "epoch: 10, passed number of graphs: 768, train running loss: 0.00189 (passed time: 1h 9m 47s)\n",
            "          acc: 0.99992, fpr: 0.00001, fnr: 0.00326, rec: 0.99674, prc: 0.99935, f1: 0.99804, f1_macro: 0.99900\n",
            "Validation --- epoch: 10, loss: 0.00178\n",
            "          prc: 0.99870, fpr: 0.00003, rec: 0.98620, fnr: 0.01380, f1: 0.99142, acc: 0.99968, f1_macro: 0.99563\n",
            "Better model saved at ./saved_models/GIN_9: 0.9914196685021919.pt.\n",
            "epoch: 11, passed number of graphs: 96, train running loss: 0.00081 (passed time: 1h 11m 8s)\n",
            "          acc: 0.99997, fpr: 0.00002, fnr: 0.00032, rec: 0.99968, prc: 0.99904, f1: 0.99936, f1_macro: 0.99967\n",
            "epoch: 11, passed number of graphs: 192, train running loss: 0.00095 (passed time: 1h 11m 57s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00296, rec: 0.99704, prc: 1.00000, f1: 0.99852, f1_macro: 0.99924\n",
            "epoch: 11, passed number of graphs: 288, train running loss: 0.00086 (passed time: 1h 12m 45s)\n",
            "          acc: 0.99990, fpr: 0.00008, fnr: 0.00098, rec: 0.99902, prc: 0.99640, f1: 0.99771, f1_macro: 0.99883\n",
            "epoch: 11, passed number of graphs: 384, train running loss: 0.00086 (passed time: 1h 13m 33s)\n",
            "          acc: 0.99993, fpr: 0.00002, fnr: 0.00226, rec: 0.99774, prc: 0.99903, f1: 0.99838, f1_macro: 0.99917\n",
            "epoch: 11, passed number of graphs: 480, train running loss: 0.00084 (passed time: 1h 14m 21s)\n",
            "          acc: 0.99985, fpr: 0.00014, fnr: 0.00065, rec: 0.99935, prc: 0.99384, f1: 0.99658, f1_macro: 0.99825\n",
            "epoch: 11, passed number of graphs: 576, train running loss: 0.00080 (passed time: 1h 15m 10s)\n",
            "          acc: 0.99970, fpr: 0.00028, fnr: 0.00130, rec: 0.99870, prc: 0.98750, f1: 0.99307, f1_macro: 0.99646\n",
            "epoch: 11, passed number of graphs: 672, train running loss: 0.00096 (passed time: 1h 15m 58s)\n",
            "          acc: 0.99939, fpr: 0.00049, fnr: 0.00611, rec: 0.99389, prc: 0.97785, f1: 0.98580, f1_macro: 0.99275\n",
            "epoch: 11, passed number of graphs: 768, train running loss: 0.00102 (passed time: 1h 16m 46s)\n",
            "          acc: 0.99988, fpr: 0.00001, fnr: 0.00490, rec: 0.99510, prc: 0.99934, f1: 0.99722, f1_macro: 0.99858\n",
            "Validation --- epoch: 11, loss: 0.00187\n",
            "          prc: 0.99884, fpr: 0.00003, rec: 0.98584, fnr: 0.01416, f1: 0.99146, acc: 0.99967, f1_macro: 0.99564\n",
            "Better model saved at ./saved_models/GIN_10: 0.9914569743807733.pt.\n",
            "epoch: 12, passed number of graphs: 96, train running loss: 0.00065 (passed time: 1h 18m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 12, passed number of graphs: 192, train running loss: 0.00093 (passed time: 1h 18m 56s)\n",
            "          acc: 0.99993, fpr: 0.00000, fnr: 0.00329, rec: 0.99671, prc: 1.00000, f1: 0.99835, f1_macro: 0.99916\n",
            "epoch: 12, passed number of graphs: 288, train running loss: 0.00086 (passed time: 1h 19m 45s)\n",
            "          acc: 0.99990, fpr: 0.00008, fnr: 0.00098, rec: 0.99902, prc: 0.99640, f1: 0.99771, f1_macro: 0.99883\n",
            "epoch: 12, passed number of graphs: 384, train running loss: 0.00087 (passed time: 1h 20m 33s)\n",
            "          acc: 0.99994, fpr: 0.00003, fnr: 0.00161, rec: 0.99839, prc: 0.99871, f1: 0.99855, f1_macro: 0.99926\n",
            "epoch: 12, passed number of graphs: 480, train running loss: 0.00084 (passed time: 1h 21m 21s)\n",
            "          acc: 0.99984, fpr: 0.00015, fnr: 0.00065, rec: 0.99935, prc: 0.99319, f1: 0.99626, f1_macro: 0.99809\n",
            "epoch: 12, passed number of graphs: 576, train running loss: 0.00080 (passed time: 1h 22m 10s)\n",
            "          acc: 0.99969, fpr: 0.00027, fnr: 0.00194, rec: 0.99806, prc: 0.98781, f1: 0.99291, f1_macro: 0.99637\n",
            "epoch: 12, passed number of graphs: 672, train running loss: 0.00100 (passed time: 1h 22m 58s)\n",
            "          acc: 0.99941, fpr: 0.00051, fnr: 0.00386, rec: 0.99614, prc: 0.97666, f1: 0.98631, f1_macro: 0.99300\n",
            "epoch: 12, passed number of graphs: 768, train running loss: 0.00105 (passed time: 1h 23m 46s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00196, rec: 0.99804, prc: 0.99902, f1: 0.99853, f1_macro: 0.99925\n",
            "Validation --- epoch: 12, loss: 0.00132\n",
            "          prc: 0.99780, fpr: 0.00005, rec: 0.99074, fnr: 0.00926, f1: 0.99352, acc: 0.99975, f1_macro: 0.99670\n",
            "Better model saved at ./saved_models/GIN_11: 0.9935161056967313.pt.\n",
            "epoch: 13, passed number of graphs: 96, train running loss: 0.00056 (passed time: 1h 25m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 13, passed number of graphs: 192, train running loss: 0.00088 (passed time: 1h 25m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 13, passed number of graphs: 288, train running loss: 0.00082 (passed time: 1h 26m 45s)\n",
            "          acc: 0.99990, fpr: 0.00008, fnr: 0.00098, rec: 0.99902, prc: 0.99607, f1: 0.99754, f1_macro: 0.99875\n",
            "epoch: 13, passed number of graphs: 384, train running loss: 0.00082 (passed time: 1h 27m 33s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00161, rec: 0.99839, prc: 0.99903, f1: 0.99871, f1_macro: 0.99934\n",
            "epoch: 13, passed number of graphs: 480, train running loss: 0.00078 (passed time: 1h 28m 21s)\n",
            "          acc: 0.99983, fpr: 0.00016, fnr: 0.00065, rec: 0.99935, prc: 0.99287, f1: 0.99610, f1_macro: 0.99801\n",
            "epoch: 13, passed number of graphs: 576, train running loss: 0.00074 (passed time: 1h 29m 10s)\n",
            "          acc: 0.99969, fpr: 0.00028, fnr: 0.00194, rec: 0.99806, prc: 0.98749, f1: 0.99275, f1_macro: 0.99629\n",
            "epoch: 13, passed number of graphs: 672, train running loss: 0.00093 (passed time: 1h 29m 58s)\n",
            "          acc: 0.99787, fpr: 0.00022, fnr: 0.09070, rec: 0.90930, prc: 0.98915, f1: 0.94754, f1_macro: 0.97323\n",
            "epoch: 13, passed number of graphs: 768, train running loss: 0.00098 (passed time: 1h 30m 46s)\n",
            "          acc: 0.99993, fpr: 0.00002, fnr: 0.00229, rec: 0.99771, prc: 0.99902, f1: 0.99837, f1_macro: 0.99917\n",
            "Validation --- epoch: 13, loss: 0.00144\n",
            "          prc: 0.99765, fpr: 0.00005, rec: 0.99068, fnr: 0.00932, f1: 0.99341, acc: 0.99975, f1_macro: 0.99664\n",
            "epoch: 14, passed number of graphs: 96, train running loss: 0.00049 (passed time: 1h 32m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 14, passed number of graphs: 192, train running loss: 0.00076 (passed time: 1h 32m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 14, passed number of graphs: 288, train running loss: 0.00071 (passed time: 1h 33m 45s)\n",
            "          acc: 0.99990, fpr: 0.00008, fnr: 0.00098, rec: 0.99902, prc: 0.99640, f1: 0.99771, f1_macro: 0.99883\n",
            "epoch: 14, passed number of graphs: 384, train running loss: 0.00072 (passed time: 1h 34m 33s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00194, rec: 0.99806, prc: 0.99903, f1: 0.99855, f1_macro: 0.99926\n",
            "epoch: 14, passed number of graphs: 480, train running loss: 0.00069 (passed time: 1h 35m 21s)\n",
            "          acc: 0.99980, fpr: 0.00019, fnr: 0.00065, rec: 0.99935, prc: 0.99158, f1: 0.99545, f1_macro: 0.99767\n",
            "epoch: 14, passed number of graphs: 576, train running loss: 0.00066 (passed time: 1h 36m 10s)\n",
            "          acc: 0.99967, fpr: 0.00031, fnr: 0.00130, rec: 0.99870, prc: 0.98592, f1: 0.99227, f1_macro: 0.99605\n",
            "epoch: 14, passed number of graphs: 672, train running loss: 0.00083 (passed time: 1h 36m 58s)\n",
            "          acc: 0.99894, fpr: 0.00028, fnr: 0.03699, rec: 0.96301, prc: 0.98682, f1: 0.97477, f1_macro: 0.98711\n",
            "epoch: 14, passed number of graphs: 768, train running loss: 0.00089 (passed time: 1h 37m 46s)\n",
            "          acc: 0.99992, fpr: 0.00001, fnr: 0.00326, rec: 0.99674, prc: 0.99935, f1: 0.99804, f1_macro: 0.99900\n",
            "Validation --- epoch: 14, loss: 0.00159\n",
            "          prc: 0.99784, fpr: 0.00005, rec: 0.98982, fnr: 0.01018, f1: 0.99307, acc: 0.99973, f1_macro: 0.99647\n",
            "epoch: 15, passed number of graphs: 96, train running loss: 0.00050 (passed time: 1h 39m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 15, passed number of graphs: 192, train running loss: 0.00060 (passed time: 1h 39m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 15, passed number of graphs: 288, train running loss: 0.00055 (passed time: 1h 40m 44s)\n",
            "          acc: 0.99988, fpr: 0.00007, fnr: 0.00230, rec: 0.99770, prc: 0.99672, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 15, passed number of graphs: 384, train running loss: 0.00057 (passed time: 1h 41m 33s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00161, rec: 0.99839, prc: 0.99903, f1: 0.99871, f1_macro: 0.99934\n",
            "epoch: 15, passed number of graphs: 480, train running loss: 0.00055 (passed time: 1h 42m 21s)\n",
            "          acc: 0.99987, fpr: 0.00011, fnr: 0.00098, rec: 0.99902, prc: 0.99513, f1: 0.99707, f1_macro: 0.99850\n",
            "epoch: 15, passed number of graphs: 576, train running loss: 0.00053 (passed time: 1h 43m 9s)\n",
            "          acc: 0.99967, fpr: 0.00031, fnr: 0.00130, rec: 0.99870, prc: 0.98592, f1: 0.99227, f1_macro: 0.99605\n",
            "epoch: 15, passed number of graphs: 672, train running loss: 0.00059 (passed time: 1h 43m 58s)\n",
            "          acc: 0.99941, fpr: 0.00056, fnr: 0.00225, rec: 0.99775, prc: 0.97486, f1: 0.98617, f1_macro: 0.99293\n",
            "epoch: 15, passed number of graphs: 768, train running loss: 0.00065 (passed time: 1h 44m 46s)\n",
            "          acc: 0.99994, fpr: 0.00003, fnr: 0.00131, rec: 0.99869, prc: 0.99869, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 15, loss: 0.00135\n",
            "          prc: 0.99696, fpr: 0.00007, rec: 0.99204, fnr: 0.00796, f1: 0.99378, acc: 0.99976, f1_macro: 0.99683\n",
            "Better model saved at ./saved_models/GIN_14: 0.993779208589142.pt.\n",
            "epoch: 16, passed number of graphs: 96, train running loss: 0.00044 (passed time: 1h 46m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 16, passed number of graphs: 192, train running loss: 0.00061 (passed time: 1h 46m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 16, passed number of graphs: 288, train running loss: 0.00056 (passed time: 1h 47m 44s)\n",
            "          acc: 0.99989, fpr: 0.00007, fnr: 0.00197, rec: 0.99803, prc: 0.99672, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 16, passed number of graphs: 384, train running loss: 0.00058 (passed time: 1h 48m 33s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00194, rec: 0.99806, prc: 0.99903, f1: 0.99855, f1_macro: 0.99926\n",
            "epoch: 16, passed number of graphs: 480, train running loss: 0.00057 (passed time: 1h 49m 21s)\n",
            "          acc: 0.99988, fpr: 0.00010, fnr: 0.00098, rec: 0.99902, prc: 0.99545, f1: 0.99723, f1_macro: 0.99859\n",
            "epoch: 16, passed number of graphs: 576, train running loss: 0.00054 (passed time: 1h 50m 9s)\n",
            "          acc: 0.99967, fpr: 0.00031, fnr: 0.00097, rec: 0.99903, prc: 0.98592, f1: 0.99243, f1_macro: 0.99613\n",
            "epoch: 16, passed number of graphs: 672, train running loss: 0.00062 (passed time: 1h 50m 57s)\n",
            "          acc: 0.99941, fpr: 0.00054, fnr: 0.00289, rec: 0.99711, prc: 0.97546, f1: 0.98616, f1_macro: 0.99293\n",
            "epoch: 16, passed number of graphs: 768, train running loss: 0.00067 (passed time: 1h 51m 46s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 16, loss: 0.00132\n",
            "          prc: 0.99715, fpr: 0.00006, rec: 0.99158, fnr: 0.00842, f1: 0.99361, acc: 0.99976, f1_macro: 0.99674\n",
            "epoch: 17, passed number of graphs: 96, train running loss: 0.00043 (passed time: 1h 53m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 17, passed number of graphs: 192, train running loss: 0.00059 (passed time: 1h 53m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 17, passed number of graphs: 288, train running loss: 0.00054 (passed time: 1h 54m 44s)\n",
            "          acc: 0.99989, fpr: 0.00007, fnr: 0.00197, rec: 0.99803, prc: 0.99672, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 17, passed number of graphs: 384, train running loss: 0.00057 (passed time: 1h 55m 32s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00194, rec: 0.99806, prc: 0.99903, f1: 0.99855, f1_macro: 0.99926\n",
            "epoch: 17, passed number of graphs: 480, train running loss: 0.00056 (passed time: 1h 56m 21s)\n",
            "          acc: 0.99988, fpr: 0.00010, fnr: 0.00098, rec: 0.99902, prc: 0.99545, f1: 0.99723, f1_macro: 0.99859\n",
            "epoch: 17, passed number of graphs: 576, train running loss: 0.00054 (passed time: 1h 57m 9s)\n",
            "          acc: 0.99967, fpr: 0.00031, fnr: 0.00097, rec: 0.99903, prc: 0.98592, f1: 0.99243, f1_macro: 0.99613\n",
            "epoch: 17, passed number of graphs: 672, train running loss: 0.00061 (passed time: 1h 57m 57s)\n",
            "          acc: 0.99941, fpr: 0.00053, fnr: 0.00354, rec: 0.99646, prc: 0.97606, f1: 0.98615, f1_macro: 0.99293\n",
            "epoch: 17, passed number of graphs: 768, train running loss: 0.00066 (passed time: 1h 58m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 17, loss: 0.00131\n",
            "          prc: 0.99697, fpr: 0.00007, rec: 0.99149, fnr: 0.00851, f1: 0.99347, acc: 0.99975, f1_macro: 0.99667\n",
            "epoch: 18, passed number of graphs: 96, train running loss: 0.00042 (passed time: 2h 0m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 18, passed number of graphs: 192, train running loss: 0.00058 (passed time: 2h 0m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 18, passed number of graphs: 288, train running loss: 0.00053 (passed time: 2h 1m 43s)\n",
            "          acc: 0.99989, fpr: 0.00007, fnr: 0.00197, rec: 0.99803, prc: 0.99672, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 18, passed number of graphs: 384, train running loss: 0.00056 (passed time: 2h 2m 32s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00194, rec: 0.99806, prc: 0.99903, f1: 0.99855, f1_macro: 0.99926\n",
            "epoch: 18, passed number of graphs: 480, train running loss: 0.00055 (passed time: 2h 3m 20s)\n",
            "          acc: 0.99988, fpr: 0.00010, fnr: 0.00098, rec: 0.99902, prc: 0.99545, f1: 0.99723, f1_macro: 0.99859\n",
            "epoch: 18, passed number of graphs: 576, train running loss: 0.00053 (passed time: 2h 4m 8s)\n",
            "          acc: 0.99967, fpr: 0.00031, fnr: 0.00097, rec: 0.99903, prc: 0.98592, f1: 0.99243, f1_macro: 0.99613\n",
            "epoch: 18, passed number of graphs: 672, train running loss: 0.00060 (passed time: 2h 4m 57s)\n",
            "          acc: 0.99940, fpr: 0.00052, fnr: 0.00418, rec: 0.99582, prc: 0.97635, f1: 0.98599, f1_macro: 0.99284\n",
            "epoch: 18, passed number of graphs: 768, train running loss: 0.00066 (passed time: 2h 5m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00196, rec: 0.99804, prc: 0.99902, f1: 0.99853, f1_macro: 0.99925\n",
            "Validation --- epoch: 18, loss: 0.00137\n",
            "          prc: 0.99717, fpr: 0.00006, rec: 0.99103, fnr: 0.00897, f1: 0.99334, acc: 0.99975, f1_macro: 0.99660\n",
            "epoch: 19, passed number of graphs: 96, train running loss: 0.00042 (passed time: 2h 7m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 19, passed number of graphs: 192, train running loss: 0.00056 (passed time: 2h 7m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 19, passed number of graphs: 288, train running loss: 0.00051 (passed time: 2h 8m 43s)\n",
            "          acc: 0.99989, fpr: 0.00007, fnr: 0.00197, rec: 0.99803, prc: 0.99672, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 19, passed number of graphs: 384, train running loss: 0.00055 (passed time: 2h 9m 31s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00194, rec: 0.99806, prc: 0.99903, f1: 0.99855, f1_macro: 0.99926\n",
            "epoch: 19, passed number of graphs: 480, train running loss: 0.00054 (passed time: 2h 10m 20s)\n",
            "          acc: 0.99986, fpr: 0.00012, fnr: 0.00098, rec: 0.99902, prc: 0.99448, f1: 0.99674, f1_macro: 0.99834\n",
            "epoch: 19, passed number of graphs: 576, train running loss: 0.00052 (passed time: 2h 11m 8s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 19, passed number of graphs: 672, train running loss: 0.00059 (passed time: 2h 11m 56s)\n",
            "          acc: 0.99940, fpr: 0.00052, fnr: 0.00418, rec: 0.99582, prc: 0.97635, f1: 0.98599, f1_macro: 0.99284\n",
            "epoch: 19, passed number of graphs: 768, train running loss: 0.00065 (passed time: 2h 12m 45s)\n",
            "          acc: 0.99993, fpr: 0.00002, fnr: 0.00229, rec: 0.99771, prc: 0.99902, f1: 0.99837, f1_macro: 0.99917\n",
            "Validation --- epoch: 19, loss: 0.00138\n",
            "          prc: 0.99735, fpr: 0.00006, rec: 0.99083, fnr: 0.00917, f1: 0.99333, acc: 0.99975, f1_macro: 0.99660\n",
            "epoch: 20, passed number of graphs: 96, train running loss: 0.00044 (passed time: 2h 14m 6s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 20, passed number of graphs: 192, train running loss: 0.00054 (passed time: 2h 14m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 20, passed number of graphs: 288, train running loss: 0.00050 (passed time: 2h 15m 43s)\n",
            "          acc: 0.99988, fpr: 0.00007, fnr: 0.00262, rec: 0.99738, prc: 0.99672, f1: 0.99705, f1_macro: 0.99849\n",
            "epoch: 20, passed number of graphs: 384, train running loss: 0.00051 (passed time: 2h 16m 31s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 20, passed number of graphs: 480, train running loss: 0.00049 (passed time: 2h 17m 19s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 20, passed number of graphs: 576, train running loss: 0.00047 (passed time: 2h 18m 8s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00097, rec: 0.99903, prc: 0.98624, f1: 0.99259, f1_macro: 0.99621\n",
            "epoch: 20, passed number of graphs: 672, train running loss: 0.00053 (passed time: 2h 18m 56s)\n",
            "          acc: 0.99940, fpr: 0.00055, fnr: 0.00289, rec: 0.99711, prc: 0.97515, f1: 0.98601, f1_macro: 0.99285\n",
            "epoch: 20, passed number of graphs: 768, train running loss: 0.00058 (passed time: 2h 19m 44s)\n",
            "          acc: 0.99995, fpr: 0.00003, fnr: 0.00098, rec: 0.99902, prc: 0.99869, f1: 0.99886, f1_macro: 0.99942\n",
            "Validation --- epoch: 20, loss: 0.00127\n",
            "          prc: 0.99700, fpr: 0.00007, rec: 0.99205, fnr: 0.00795, f1: 0.99380, acc: 0.99976, f1_macro: 0.99684\n",
            "Better model saved at ./saved_models/GIN_19: 0.993800137885532.pt.\n",
            "epoch: 21, passed number of graphs: 96, train running loss: 0.00046 (passed time: 2h 21m 6s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 21, passed number of graphs: 192, train running loss: 0.00058 (passed time: 2h 21m 54s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 21, passed number of graphs: 288, train running loss: 0.00053 (passed time: 2h 22m 42s)\n",
            "          acc: 0.99988, fpr: 0.00006, fnr: 0.00295, rec: 0.99705, prc: 0.99705, f1: 0.99705, f1_macro: 0.99849\n",
            "epoch: 21, passed number of graphs: 384, train running loss: 0.00054 (passed time: 2h 23m 31s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 21, passed number of graphs: 480, train running loss: 0.00052 (passed time: 2h 24m 19s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 21, passed number of graphs: 576, train running loss: 0.00049 (passed time: 2h 25m 7s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00097, rec: 0.99903, prc: 0.98624, f1: 0.99259, f1_macro: 0.99621\n",
            "epoch: 21, passed number of graphs: 672, train running loss: 0.00055 (passed time: 2h 25m 55s)\n",
            "          acc: 0.99941, fpr: 0.00054, fnr: 0.00289, rec: 0.99711, prc: 0.97546, f1: 0.98616, f1_macro: 0.99293\n",
            "epoch: 21, passed number of graphs: 768, train running loss: 0.00060 (passed time: 2h 26m 44s)\n",
            "          acc: 0.99995, fpr: 0.00003, fnr: 0.00098, rec: 0.99902, prc: 0.99869, f1: 0.99886, f1_macro: 0.99942\n",
            "Validation --- epoch: 21, loss: 0.00128\n",
            "          prc: 0.99697, fpr: 0.00007, rec: 0.99191, fnr: 0.00809, f1: 0.99369, acc: 0.99976, f1_macro: 0.99678\n",
            "epoch: 22, passed number of graphs: 96, train running loss: 0.00046 (passed time: 2h 28m 5s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 22, passed number of graphs: 192, train running loss: 0.00058 (passed time: 2h 28m 54s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 22, passed number of graphs: 288, train running loss: 0.00053 (passed time: 2h 29m 42s)\n",
            "          acc: 0.99988, fpr: 0.00006, fnr: 0.00295, rec: 0.99705, prc: 0.99737, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 22, passed number of graphs: 384, train running loss: 0.00054 (passed time: 2h 30m 30s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 22, passed number of graphs: 480, train running loss: 0.00051 (passed time: 2h 31m 18s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 22, passed number of graphs: 576, train running loss: 0.00049 (passed time: 2h 32m 7s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00097, rec: 0.99903, prc: 0.98624, f1: 0.99259, f1_macro: 0.99621\n",
            "epoch: 22, passed number of graphs: 672, train running loss: 0.00055 (passed time: 2h 32m 55s)\n",
            "          acc: 0.99941, fpr: 0.00054, fnr: 0.00289, rec: 0.99711, prc: 0.97546, f1: 0.98616, f1_macro: 0.99293\n",
            "epoch: 22, passed number of graphs: 768, train running loss: 0.00060 (passed time: 2h 33m 44s)\n",
            "          acc: 0.99996, fpr: 0.00003, fnr: 0.00065, rec: 0.99935, prc: 0.99869, f1: 0.99902, f1_macro: 0.99950\n",
            "Validation --- epoch: 22, loss: 0.00127\n",
            "          prc: 0.99692, fpr: 0.00007, rec: 0.99197, fnr: 0.00803, f1: 0.99369, acc: 0.99976, f1_macro: 0.99678\n",
            "epoch: 23, passed number of graphs: 96, train running loss: 0.00046 (passed time: 2h 35m 5s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 23, passed number of graphs: 192, train running loss: 0.00058 (passed time: 2h 35m 53s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 23, passed number of graphs: 288, train running loss: 0.00053 (passed time: 2h 36m 42s)\n",
            "          acc: 0.99988, fpr: 0.00006, fnr: 0.00295, rec: 0.99705, prc: 0.99737, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 23, passed number of graphs: 384, train running loss: 0.00053 (passed time: 2h 37m 30s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 23, passed number of graphs: 480, train running loss: 0.00051 (passed time: 2h 38m 18s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 23, passed number of graphs: 576, train running loss: 0.00049 (passed time: 2h 39m 7s)\n",
            "          acc: 0.99969, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98624, f1: 0.99275, f1_macro: 0.99630\n",
            "epoch: 23, passed number of graphs: 672, train running loss: 0.00054 (passed time: 2h 39m 55s)\n",
            "          acc: 0.99941, fpr: 0.00054, fnr: 0.00289, rec: 0.99711, prc: 0.97546, f1: 0.98616, f1_macro: 0.99293\n",
            "epoch: 23, passed number of graphs: 768, train running loss: 0.00059 (passed time: 2h 40m 43s)\n",
            "          acc: 0.99996, fpr: 0.00003, fnr: 0.00065, rec: 0.99935, prc: 0.99869, f1: 0.99902, f1_macro: 0.99950\n",
            "Validation --- epoch: 23, loss: 0.00126\n",
            "          prc: 0.99691, fpr: 0.00007, rec: 0.99198, fnr: 0.00802, f1: 0.99369, acc: 0.99976, f1_macro: 0.99678\n",
            "epoch: 24, passed number of graphs: 96, train running loss: 0.00045 (passed time: 2h 42m 5s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 24, passed number of graphs: 192, train running loss: 0.00057 (passed time: 2h 42m 53s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 24, passed number of graphs: 288, train running loss: 0.00052 (passed time: 2h 43m 41s)\n",
            "          acc: 0.99988, fpr: 0.00006, fnr: 0.00295, rec: 0.99705, prc: 0.99737, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 24, passed number of graphs: 384, train running loss: 0.00053 (passed time: 2h 44m 30s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 24, passed number of graphs: 480, train running loss: 0.00051 (passed time: 2h 45m 18s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 24, passed number of graphs: 576, train running loss: 0.00049 (passed time: 2h 46m 6s)\n",
            "          acc: 0.99970, fpr: 0.00029, fnr: 0.00065, rec: 0.99935, prc: 0.98688, f1: 0.99307, f1_macro: 0.99646\n",
            "epoch: 24, passed number of graphs: 672, train running loss: 0.00054 (passed time: 2h 46m 55s)\n",
            "          acc: 0.99941, fpr: 0.00054, fnr: 0.00289, rec: 0.99711, prc: 0.97546, f1: 0.98616, f1_macro: 0.99293\n",
            "epoch: 24, passed number of graphs: 768, train running loss: 0.00059 (passed time: 2h 47m 43s)\n",
            "          acc: 0.99996, fpr: 0.00003, fnr: 0.00033, rec: 0.99967, prc: 0.99870, f1: 0.99918, f1_macro: 0.99958\n",
            "Validation --- epoch: 24, loss: 0.00128\n",
            "          prc: 0.99659, fpr: 0.00008, rec: 0.99222, fnr: 0.00778, f1: 0.99368, acc: 0.99976, f1_macro: 0.99678\n",
            "epoch: 25, passed number of graphs: 96, train running loss: 0.00045 (passed time: 2h 49m 5s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 25, passed number of graphs: 192, train running loss: 0.00057 (passed time: 2h 49m 53s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 25, passed number of graphs: 288, train running loss: 0.00052 (passed time: 2h 50m 42s)\n",
            "          acc: 0.99988, fpr: 0.00006, fnr: 0.00295, rec: 0.99705, prc: 0.99737, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 25, passed number of graphs: 384, train running loss: 0.00053 (passed time: 2h 51m 30s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 25, passed number of graphs: 480, train running loss: 0.00051 (passed time: 2h 52m 18s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 25, passed number of graphs: 576, train running loss: 0.00048 (passed time: 2h 53m 7s)\n",
            "          acc: 0.99970, fpr: 0.00029, fnr: 0.00065, rec: 0.99935, prc: 0.98688, f1: 0.99307, f1_macro: 0.99646\n",
            "epoch: 25, passed number of graphs: 672, train running loss: 0.00054 (passed time: 2h 53m 55s)\n",
            "          acc: 0.99941, fpr: 0.00054, fnr: 0.00289, rec: 0.99711, prc: 0.97576, f1: 0.98632, f1_macro: 0.99301\n",
            "epoch: 25, passed number of graphs: 768, train running loss: 0.00059 (passed time: 2h 54m 44s)\n",
            "          acc: 0.99996, fpr: 0.00003, fnr: 0.00033, rec: 0.99967, prc: 0.99870, f1: 0.99918, f1_macro: 0.99958\n",
            "Validation --- epoch: 25, loss: 0.00127\n",
            "          prc: 0.99670, fpr: 0.00007, rec: 0.99203, fnr: 0.00797, f1: 0.99361, acc: 0.99976, f1_macro: 0.99674\n",
            "epoch: 26, passed number of graphs: 96, train running loss: 0.00055 (passed time: 2h 56m 5s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 26, passed number of graphs: 192, train running loss: 0.00060 (passed time: 2h 56m 54s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 26, passed number of graphs: 288, train running loss: 0.00054 (passed time: 2h 57m 42s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00230, rec: 0.99770, prc: 0.99705, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 26, passed number of graphs: 384, train running loss: 0.00053 (passed time: 2h 58m 30s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 26, passed number of graphs: 480, train running loss: 0.00051 (passed time: 2h 59m 19s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 26, passed number of graphs: 576, train running loss: 0.00048 (passed time: 3h 0m 7s)\n",
            "          acc: 0.99969, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98624, f1: 0.99275, f1_macro: 0.99630\n",
            "epoch: 26, passed number of graphs: 672, train running loss: 0.00053 (passed time: 3h 0m 55s)\n",
            "          acc: 0.99941, fpr: 0.00049, fnr: 0.00482, rec: 0.99518, prc: 0.97757, f1: 0.98629, f1_macro: 0.99300\n",
            "epoch: 26, passed number of graphs: 768, train running loss: 0.00058 (passed time: 3h 1m 44s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 26, loss: 0.00119\n",
            "          prc: 0.99763, fpr: 0.00005, rec: 0.99148, fnr: 0.00852, f1: 0.99380, acc: 0.99977, f1_macro: 0.99684\n",
            "Better model saved at ./saved_models/GIN_25: 0.9938025280955806.pt.\n",
            "epoch: 27, passed number of graphs: 96, train running loss: 0.00046 (passed time: 3h 3m 5s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 27, passed number of graphs: 192, train running loss: 0.00055 (passed time: 3h 3m 54s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 27, passed number of graphs: 288, train running loss: 0.00051 (passed time: 3h 4m 42s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 27, passed number of graphs: 384, train running loss: 0.00051 (passed time: 3h 5m 31s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 27, passed number of graphs: 480, train running loss: 0.00049 (passed time: 3h 6m 19s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 27, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 7m 7s)\n",
            "          acc: 0.99969, fpr: 0.00030, fnr: 0.00065, rec: 0.99935, prc: 0.98656, f1: 0.99291, f1_macro: 0.99638\n",
            "epoch: 27, passed number of graphs: 672, train running loss: 0.00052 (passed time: 3h 7m 56s)\n",
            "          acc: 0.99942, fpr: 0.00051, fnr: 0.00386, rec: 0.99614, prc: 0.97697, f1: 0.98646, f1_macro: 0.99308\n",
            "epoch: 27, passed number of graphs: 768, train running loss: 0.00057 (passed time: 3h 8m 44s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 27, loss: 0.00119\n",
            "          prc: 0.99761, fpr: 0.00005, rec: 0.99148, fnr: 0.00852, f1: 0.99379, acc: 0.99976, f1_macro: 0.99683\n",
            "epoch: 28, passed number of graphs: 96, train running loss: 0.00046 (passed time: 3h 10m 6s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 28, passed number of graphs: 192, train running loss: 0.00056 (passed time: 3h 10m 54s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 28, passed number of graphs: 288, train running loss: 0.00051 (passed time: 3h 11m 43s)\n",
            "          acc: 0.99988, fpr: 0.00006, fnr: 0.00295, rec: 0.99705, prc: 0.99737, f1: 0.99721, f1_macro: 0.99858\n",
            "epoch: 28, passed number of graphs: 384, train running loss: 0.00052 (passed time: 3h 12m 31s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 28, passed number of graphs: 480, train running loss: 0.00049 (passed time: 3h 13m 19s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 28, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 14m 8s)\n",
            "          acc: 0.99969, fpr: 0.00030, fnr: 0.00065, rec: 0.99935, prc: 0.98656, f1: 0.99291, f1_macro: 0.99638\n",
            "epoch: 28, passed number of graphs: 672, train running loss: 0.00052 (passed time: 3h 14m 56s)\n",
            "          acc: 0.99941, fpr: 0.00049, fnr: 0.00482, rec: 0.99518, prc: 0.97757, f1: 0.98629, f1_macro: 0.99300\n",
            "epoch: 28, passed number of graphs: 768, train running loss: 0.00057 (passed time: 3h 15m 45s)\n",
            "          acc: 0.99994, fpr: 0.00003, fnr: 0.00163, rec: 0.99837, prc: 0.99869, f1: 0.99853, f1_macro: 0.99925\n",
            "Validation --- epoch: 28, loss: 0.00119\n",
            "          prc: 0.99761, fpr: 0.00005, rec: 0.99148, fnr: 0.00852, f1: 0.99379, acc: 0.99976, f1_macro: 0.99683\n",
            "epoch: 29, passed number of graphs: 96, train running loss: 0.00049 (passed time: 3h 17m 6s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 29, passed number of graphs: 192, train running loss: 0.00057 (passed time: 3h 17m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 29, passed number of graphs: 288, train running loss: 0.00052 (passed time: 3h 18m 43s)\n",
            "          acc: 0.99989, fpr: 0.00007, fnr: 0.00197, rec: 0.99803, prc: 0.99672, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 29, passed number of graphs: 384, train running loss: 0.00052 (passed time: 3h 19m 31s)\n",
            "          acc: 0.99994, fpr: 0.00003, fnr: 0.00129, rec: 0.99871, prc: 0.99871, f1: 0.99871, f1_macro: 0.99934\n",
            "epoch: 29, passed number of graphs: 480, train running loss: 0.00050 (passed time: 3h 20m 20s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 29, passed number of graphs: 576, train running loss: 0.00048 (passed time: 3h 21m 8s)\n",
            "          acc: 0.99969, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98624, f1: 0.99275, f1_macro: 0.99630\n",
            "epoch: 29, passed number of graphs: 672, train running loss: 0.00053 (passed time: 3h 21m 56s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 29, passed number of graphs: 768, train running loss: 0.00057 (passed time: 3h 22m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 29, loss: 0.00119\n",
            "          prc: 0.99793, fpr: 0.00005, rec: 0.99115, fnr: 0.00885, f1: 0.99378, acc: 0.99976, f1_macro: 0.99683\n",
            "epoch: 30, passed number of graphs: 96, train running loss: 0.00044 (passed time: 3h 24m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 30, passed number of graphs: 192, train running loss: 0.00054 (passed time: 3h 24m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 30, passed number of graphs: 288, train running loss: 0.00050 (passed time: 3h 25m 43s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 30, passed number of graphs: 384, train running loss: 0.00051 (passed time: 3h 26m 32s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 30, passed number of graphs: 480, train running loss: 0.00049 (passed time: 3h 27m 20s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 30, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 28m 9s)\n",
            "          acc: 0.99969, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98624, f1: 0.99275, f1_macro: 0.99630\n",
            "epoch: 30, passed number of graphs: 672, train running loss: 0.00052 (passed time: 3h 28m 57s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 30, passed number of graphs: 768, train running loss: 0.00056 (passed time: 3h 29m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 30, loss: 0.00120\n",
            "          prc: 0.99795, fpr: 0.00005, rec: 0.99110, fnr: 0.00890, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 31, passed number of graphs: 96, train running loss: 0.00043 (passed time: 3h 31m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 31, passed number of graphs: 192, train running loss: 0.00054 (passed time: 3h 31m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 31, passed number of graphs: 288, train running loss: 0.00050 (passed time: 3h 32m 43s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00230, rec: 0.99770, prc: 0.99705, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 31, passed number of graphs: 384, train running loss: 0.00051 (passed time: 3h 33m 32s)\n",
            "          acc: 0.99994, fpr: 0.00003, fnr: 0.00129, rec: 0.99871, prc: 0.99871, f1: 0.99871, f1_macro: 0.99934\n",
            "epoch: 31, passed number of graphs: 480, train running loss: 0.00049 (passed time: 3h 34m 20s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 31, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 35m 9s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 31, passed number of graphs: 672, train running loss: 0.00052 (passed time: 3h 35m 57s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 31, passed number of graphs: 768, train running loss: 0.00056 (passed time: 3h 36m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 31, loss: 0.00120\n",
            "          prc: 0.99800, fpr: 0.00004, rec: 0.99104, fnr: 0.00896, f1: 0.99376, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 32, passed number of graphs: 96, train running loss: 0.00043 (passed time: 3h 38m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 32, passed number of graphs: 192, train running loss: 0.00054 (passed time: 3h 38m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 32, passed number of graphs: 288, train running loss: 0.00050 (passed time: 3h 39m 43s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 32, passed number of graphs: 384, train running loss: 0.00050 (passed time: 3h 40m 32s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 32, passed number of graphs: 480, train running loss: 0.00048 (passed time: 3h 41m 20s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 32, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 42m 9s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 32, passed number of graphs: 672, train running loss: 0.00051 (passed time: 3h 42m 57s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 32, passed number of graphs: 768, train running loss: 0.00056 (passed time: 3h 43m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 32, loss: 0.00120\n",
            "          prc: 0.99802, fpr: 0.00004, rec: 0.99101, fnr: 0.00899, f1: 0.99376, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 33, passed number of graphs: 96, train running loss: 0.00043 (passed time: 3h 45m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 33, passed number of graphs: 192, train running loss: 0.00054 (passed time: 3h 45m 55s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 33, passed number of graphs: 288, train running loss: 0.00050 (passed time: 3h 46m 44s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 33, passed number of graphs: 384, train running loss: 0.00050 (passed time: 3h 47m 32s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 33, passed number of graphs: 480, train running loss: 0.00048 (passed time: 3h 48m 20s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 33, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 49m 9s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 33, passed number of graphs: 672, train running loss: 0.00051 (passed time: 3h 49m 57s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 33, passed number of graphs: 768, train running loss: 0.00056 (passed time: 3h 50m 45s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 33, loss: 0.00120\n",
            "          prc: 0.99803, fpr: 0.00004, rec: 0.99101, fnr: 0.00899, f1: 0.99376, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 34, passed number of graphs: 96, train running loss: 0.00043 (passed time: 3h 52m 7s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 34, passed number of graphs: 192, train running loss: 0.00054 (passed time: 3h 52m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 34, passed number of graphs: 288, train running loss: 0.00050 (passed time: 3h 53m 44s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 34, passed number of graphs: 384, train running loss: 0.00050 (passed time: 3h 54m 33s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 34, passed number of graphs: 480, train running loss: 0.00048 (passed time: 3h 55m 21s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 34, passed number of graphs: 576, train running loss: 0.00047 (passed time: 3h 56m 9s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 34, passed number of graphs: 672, train running loss: 0.00051 (passed time: 3h 56m 58s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 34, passed number of graphs: 768, train running loss: 0.00056 (passed time: 3h 57m 46s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 34, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99101, fnr: 0.00899, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 35, passed number of graphs: 96, train running loss: 0.00043 (passed time: 3h 59m 8s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 35, passed number of graphs: 192, train running loss: 0.00054 (passed time: 3h 59m 56s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 35, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 0m 45s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 35, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 1m 33s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 35, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 2m 21s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 35, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 3m 10s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 35, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 3m 58s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 35, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 4m 47s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 35, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 36, passed number of graphs: 96, train running loss: 0.00043 (passed time: 4h 6m 9s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 36, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 6m 57s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 36, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 7m 45s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 36, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 8m 34s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 36, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 9m 22s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 36, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 10m 11s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 36, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 10m 59s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 36, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 11m 47s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 36, loss: 0.00120\n",
            "          prc: 0.99805, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 37, passed number of graphs: 96, train running loss: 0.00042 (passed time: 4h 13m 9s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 37, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 13m 58s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 37, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 14m 46s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 37, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 15m 34s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 37, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 16m 23s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 37, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 17m 11s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 37, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 17m 59s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 37, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 18m 48s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 37, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99376, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 38, passed number of graphs: 96, train running loss: 0.00042 (passed time: 4h 20m 10s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 38, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 20m 58s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 38, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 21m 46s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 38, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 22m 35s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 38, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 23m 23s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 38, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 24m 11s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 38, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 25m 0s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 38, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 25m 48s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 38, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99376, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 39, passed number of graphs: 96, train running loss: 0.00042 (passed time: 4h 27m 10s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 39, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 27m 58s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 39, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 28m 47s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 39, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 29m 35s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 39, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 30m 23s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 39, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 31m 12s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 39, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 32m 0s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 39, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 32m 49s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 39, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99376, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 40, passed number of graphs: 96, train running loss: 0.00042 (passed time: 4h 34m 10s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 40, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 34m 59s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 40, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 35m 47s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 40, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 36m 35s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 40, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 37m 24s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 40, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 38m 12s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 40, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 39m 0s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 40, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 39m 49s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 40, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 41, passed number of graphs: 96, train running loss: 0.00042 (passed time: 4h 41m 11s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 41, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 41m 59s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 41, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 42m 48s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 41, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 43m 36s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 41, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 44m 24s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 41, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 45m 13s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 41, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 46m 1s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 41, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 46m 49s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 41, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 42, passed number of graphs: 96, train running loss: 0.00043 (passed time: 4h 48m 11s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 42, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 49m 0s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 42, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 49m 48s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 42, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 50m 36s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 42, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 51m 25s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 42, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 52m 13s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 42, passed number of graphs: 672, train running loss: 0.00051 (passed time: 4h 53m 1s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 42, passed number of graphs: 768, train running loss: 0.00056 (passed time: 4h 53m 50s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 42, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 43, passed number of graphs: 96, train running loss: 0.00043 (passed time: 4h 55m 12s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 43, passed number of graphs: 192, train running loss: 0.00054 (passed time: 4h 56m 0s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 43, passed number of graphs: 288, train running loss: 0.00050 (passed time: 4h 56m 49s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 43, passed number of graphs: 384, train running loss: 0.00050 (passed time: 4h 57m 37s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 43, passed number of graphs: 480, train running loss: 0.00048 (passed time: 4h 58m 25s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 43, passed number of graphs: 576, train running loss: 0.00047 (passed time: 4h 59m 14s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 43, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 0m 2s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 43, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 0m 51s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 43, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 44, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 2m 13s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 44, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 3m 1s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 44, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 3m 49s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 44, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 4m 38s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 44, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 5m 26s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 44, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 6m 15s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 44, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 7m 3s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 44, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 7m 51s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 44, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 45, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 9m 13s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 45, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 10m 2s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 45, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 10m 50s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 45, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 11m 38s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 45, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 12m 27s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 45, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 13m 15s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 45, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 14m 3s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 45, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 14m 52s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 45, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 46, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 16m 14s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 46, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 17m 2s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 46, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 17m 51s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 46, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 18m 39s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 46, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 19m 27s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 46, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 20m 16s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 46, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 21m 4s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 46, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 21m 53s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 46, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 47, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 23m 15s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 47, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 24m 3s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 47, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 24m 51s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 47, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 25m 40s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 47, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 26m 28s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 47, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 27m 16s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 47, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 28m 5s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 47, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 28m 53s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 47, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 48, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 30m 15s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 48, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 31m 3s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 48, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 31m 52s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 48, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 32m 40s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 48, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 33m 28s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 48, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 34m 17s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 48, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 35m 5s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 48, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 35m 54s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 48, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 49, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 37m 16s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 49, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 38m 4s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 49, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 38m 52s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 49, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 39m 41s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 49, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 40m 29s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 49, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 41m 17s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 49, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 42m 6s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 49, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 42m 54s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 49, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "epoch: 50, passed number of graphs: 96, train running loss: 0.00043 (passed time: 5h 44m 16s)\n",
            "          acc: 0.99999, fpr: 0.00001, fnr: 0.00032, rec: 0.99968, prc: 0.99968, f1: 0.99968, f1_macro: 0.99984\n",
            "epoch: 50, passed number of graphs: 192, train running loss: 0.00054 (passed time: 5h 45m 5s)\n",
            "          acc: 0.99994, fpr: 0.00000, fnr: 0.00263, rec: 0.99737, prc: 1.00000, f1: 0.99868, f1_macro: 0.99933\n",
            "epoch: 50, passed number of graphs: 288, train running loss: 0.00050 (passed time: 5h 45m 53s)\n",
            "          acc: 0.99989, fpr: 0.00006, fnr: 0.00262, rec: 0.99738, prc: 0.99738, f1: 0.99738, f1_macro: 0.99866\n",
            "epoch: 50, passed number of graphs: 384, train running loss: 0.00050 (passed time: 5h 46m 41s)\n",
            "          acc: 0.99995, fpr: 0.00002, fnr: 0.00129, rec: 0.99871, prc: 0.99903, f1: 0.99887, f1_macro: 0.99942\n",
            "epoch: 50, passed number of graphs: 480, train running loss: 0.00048 (passed time: 5h 47m 29s)\n",
            "          acc: 0.99988, fpr: 0.00009, fnr: 0.00131, rec: 0.99869, prc: 0.99577, f1: 0.99723, f1_macro: 0.99858\n",
            "epoch: 50, passed number of graphs: 576, train running loss: 0.00047 (passed time: 5h 48m 18s)\n",
            "          acc: 0.99968, fpr: 0.00031, fnr: 0.00065, rec: 0.99935, prc: 0.98593, f1: 0.99259, f1_macro: 0.99622\n",
            "epoch: 50, passed number of graphs: 672, train running loss: 0.00051 (passed time: 5h 49m 6s)\n",
            "          acc: 0.99940, fpr: 0.00048, fnr: 0.00611, rec: 0.99389, prc: 0.97816, f1: 0.98596, f1_macro: 0.99283\n",
            "epoch: 50, passed number of graphs: 768, train running loss: 0.00056 (passed time: 5h 49m 55s)\n",
            "          acc: 0.99994, fpr: 0.00002, fnr: 0.00163, rec: 0.99837, prc: 0.99902, f1: 0.99869, f1_macro: 0.99933\n",
            "Validation --- epoch: 50, loss: 0.00120\n",
            "          prc: 0.99804, fpr: 0.00004, rec: 0.99100, fnr: 0.00900, f1: 0.99377, acc: 0.99976, f1_macro: 0.99682\n",
            "************ best model obtained after epoch 26, saved at ./saved_models/GIN_25: 0.9938025280955806.pt ************\n",
            "Testing --- loss: 0.00080\n",
            "          prc: 0.99617, fpr: 0.00009, rec: 0.99528, fnr: 0.00472, f1: 0.99550, acc: 0.99981, f1_macro: 0.99770\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r /content/GATv2Conv_p2p_normalization.zip /content/saved_models"
      ],
      "metadata": {
        "id": "hQyvN2Q28XuX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download(\"/content/GATv2Conv_p2p_normalization.zip\")"
      ],
      "metadata": {
        "id": "_t9BObFt8eEl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "b21a3798-541b-477f-8f25-ccb46951615f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_5bd31719-fe32-4f3a-af8a-132c6f84d7d7\", \"GATv2Conv_p2p_normalization.zip\", 1389849)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# from google.colab import runtime\n",
        "# runtime.unassign()"
      ],
      "metadata": {
        "id": "VJTZlDLleS13"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}