02_residual_gated_graph_convnets_semisupervised_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Residual Gated Graph ConvNets\n",
    "### Xavier Bresson, Jan. 15 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda not available\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import pdb \n",
    "import time\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('cuda available')\n",
    "    dtypeFloat = torch.cuda.FloatTensor\n",
    "    dtypeLong = torch.cuda.LongTensor\n",
    "    #torch.cuda.manual_seed(1)\n",
    "else:\n",
    "    print('cuda not available')\n",
    "    dtypeFloat = torch.FloatTensor\n",
    "    dtypeLong = torch.LongTensor\n",
    "    #torch.manual_seed(1)\n",
    "    \n",
    "# import files in folder util\n",
    "import sys\n",
    "sys.path.insert(0, 'util/')\n",
    "import block \n",
    "import graph_generator as g\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#################\n",
    "# select task and task parameters\n",
    "#################\n",
    "\n",
    "\n",
    "# subgraph matching\n",
    "if 2==1:\n",
    "    task_parameters = {}\n",
    "    task_parameters['flag_task'] = 'matching'\n",
    "    task_parameters['nb_communities'] = 10\n",
    "    task_parameters['nb_clusters_target'] = 2\n",
    "    task_parameters['Voc'] = 3\n",
    "    task_parameters['size_min'] = 15\n",
    "    task_parameters['size_max'] = 25\n",
    "    task_parameters['size_subgraph'] = 20\n",
    "    task_parameters['p'] = 0.5\n",
    "    task_parameters['q'] = 0.1\n",
    "    task_parameters['W0'] = block.random_graph(task_parameters['size_subgraph'],task_parameters['p'])\n",
    "    task_parameters['u0'] = np.random.randint(task_parameters['Voc'],size=task_parameters['size_subgraph'])\n",
    "    file_name = 'data/set_100_subgraphs_p05_size20_Voc3_2017-10-31_10-23-00_.txt'\n",
    "    with open(file_name, 'rb') as fp:\n",
    "        all_trainx = pickle.load(fp)\n",
    "    task_parameters['all_trainx'] = all_trainx[:100]\n",
    "    \n",
    "    \n",
    "# semi-supervised clustering\n",
    "if 1==1:\n",
    "    task_parameters = {}\n",
    "    task_parameters['flag_task'] = 'clustering'\n",
    "    task_parameters['nb_communities'] = 10\n",
    "    task_parameters['nb_clusters_target'] = task_parameters['nb_communities']\n",
    "    task_parameters['Voc'] = task_parameters['nb_communities'] + 1\n",
    "    task_parameters['size_min'] = 5\n",
    "    task_parameters['size_max'] = 25\n",
    "    task_parameters['p'] = 0.5\n",
    "    task_parameters['q'] = 0.1  \n",
    "    file_name = 'data/set_100_clustering_maps_p05_q01_size5_25_2017-10-31_10-25-00_.txt'\n",
    "    with open(file_name, 'rb') as fp:\n",
    "        all_trainx = pickle.load(fp)\n",
    "    task_parameters['all_trainx'] = all_trainx[:100]\n",
    "    \n",
    "    \n",
    "#print(task_parameters)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "##############################\n",
    "# Class cell definition\n",
    "##############################\n",
    "class OurConvNetcell(nn.Module):\n",
    "    \n",
    "    def __init__(self, dim_in, dim_out):\n",
    "        super(OurConvNetcell, self).__init__()\n",
    "\n",
    "        # conv1\n",
    "        self.Ui1 = nn.Linear(dim_in, dim_out, bias=False) \n",
    "        self.Uj1 = nn.Linear(dim_in, dim_out, bias=False)\n",
    "        self.Vi1 = nn.Linear(dim_in, dim_out, bias=False) \n",
    "        self.Vj1 = nn.Linear(dim_in, dim_out, bias=False)  \n",
    "        self.bu1 = torch.nn.Parameter( torch.FloatTensor(dim_out), requires_grad=True )\n",
    "        self.bv1 = torch.nn.Parameter( torch.FloatTensor(dim_out), requires_grad=True )\n",
    "        \n",
    "        # conv2\n",
    "        self.Ui2 = nn.Linear(dim_out, dim_out, bias=False) \n",
    "        self.Uj2 = nn.Linear(dim_out, dim_out, bias=False)\n",
    "        self.Vi2 = nn.Linear(dim_out, dim_out, bias=False) \n",
    "        self.Vj2 = nn.Linear(dim_out, dim_out, bias=False)  \n",
    "        self.bu2 = torch.nn.Parameter( torch.FloatTensor(dim_out), requires_grad=True )\n",
    "        self.bv2 = torch.nn.Parameter( torch.FloatTensor(dim_out), requires_grad=True )\n",
    "        \n",
    "        # bn1, bn2\n",
    "        self.bn1 = torch.nn.BatchNorm1d(dim_out)\n",
    "        self.bn2 = torch.nn.BatchNorm1d(dim_out)\n",
    "        \n",
    "        # resnet\n",
    "        self.R = nn.Linear(dim_in, dim_out, bias=False) \n",
    "            \n",
    "        # init\n",
    "        self.init_weights_OurConvNetcell(dim_in, dim_out, 1)\n",
    "        \n",
    "         \n",
    "    def init_weights_OurConvNetcell(self, dim_in, dim_out, gain):\n",
    "        \n",
    "        # conv1\n",
    "        scale = gain* np.sqrt( 2.0/ dim_in )\n",
    "        self.Ui1.weight.data.uniform_(-scale, scale) \n",
    "        self.Uj1.weight.data.uniform_(-scale, scale) \n",
    "        self.Vi1.weight.data.uniform_(-scale, scale) \n",
    "        self.Vj1.weight.data.uniform_(-scale, scale) \n",
    "        scale = gain* np.sqrt( 2.0/ dim_out )\n",
    "        self.bu1.data.fill_(0)\n",
    "        self.bv1.data.fill_(0)\n",
    "        \n",
    "        # conv2\n",
    "        scale = gain* np.sqrt( 2.0/ dim_out )\n",
    "        self.Ui2.weight.data.uniform_(-scale, scale) \n",
    "        self.Uj2.weight.data.uniform_(-scale, scale) \n",
    "        self.Vi2.weight.data.uniform_(-scale, scale) \n",
    "        self.Vj2.weight.data.uniform_(-scale, scale) \n",
    "        scale = gain* np.sqrt( 2.0/ dim_out )\n",
    "        self.bu2.data.fill_(0)\n",
    "        self.bv2.data.fill_(0)\n",
    "        \n",
    "        # RN\n",
    "        scale = gain* np.sqrt( 2.0/ dim_in )\n",
    "        self.R.weight.data.uniform_(-scale, scale)  \n",
    "            \n",
    "            \n",
    "    def forward(self, x, E_start, E_end):\n",
    "        \n",
    "        # E_start, E_end : E x V\n",
    "\n",
    "        xin = x\n",
    "        # conv1\n",
    "        Vix = self.Vi1(x)  #  V x H_out\n",
    "        Vjx = self.Vj1(x)  #  V x H_out\n",
    "        x1 = torch.mm(E_end,Vix) + torch.mm(E_start,Vjx) + self.bv1  # E x H_out\n",
    "        x1 = F.sigmoid(x1)\n",
    "        Ujx = self.Uj1(x)  #  V x H_out\n",
    "        x2 = torch.mm(E_start, Ujx)  #  V x H_out\n",
    "        Uix = self.Ui1(x)  #  V x H_out\n",
    "        x = Uix + torch.mm(E_end.t(), x1*x2) + self.bu1 #  V x H_out\n",
    "        # bn1\n",
    "        x = self.bn1(x)\n",
    "        # relu1\n",
    "        x = F.relu(x)\n",
    "        # conv2\n",
    "        Vix = self.Vi2(x)  #  V x H_out\n",
    "        Vjx = self.Vj2(x)  #  V x H_out\n",
    "        x1 = torch.mm(E_end,Vix) + torch.mm(E_start,Vjx) + self.bv2  # E x H_out\n",
    "        x1 = F.sigmoid(x1)\n",
    "        Ujx = self.Uj2(x)  #  V x H_out\n",
    "        x2 = torch.mm(E_start, Ujx)  #  V x H_out\n",
    "        Uix = self.Ui2(x)  #  V x H_out\n",
    "        x = Uix + torch.mm(E_end.t(), x1*x2) + self.bu2 #  V x H_out\n",
    "        # bn2\n",
    "        x = self.bn2(x)\n",
    "        # addition\n",
    "        x = x + self.R(xin)\n",
    "        # relu2\n",
    "        x = F.relu(x)\n",
    "            \n",
    "        return x\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "##############################\n",
    "# Class NN definition\n",
    "##############################  \n",
    "class Graph_OurConvNet(nn.Module):\n",
    "    \n",
    "    def __init__(self, net_parameters):\n",
    "        \n",
    "        super(Graph_OurConvNet, self).__init__()\n",
    "        \n",
    "        # parameters\n",
    "        flag_task = task_parameters['flag_task']\n",
    "        Voc = net_parameters['Voc']\n",
    "        D = net_parameters['D']\n",
    "        nb_clusters_target = net_parameters['nb_clusters_target']\n",
    "        H = net_parameters['H']\n",
    "        L = net_parameters['L']\n",
    "        \n",
    "        # vector of hidden dimensions\n",
    "        net_layers = []\n",
    "        for layer in range(L):\n",
    "            net_layers.append(H)\n",
    "\n",
    "        # embedding\n",
    "        self.encoder = nn.Embedding(Voc, D)      \n",
    "        \n",
    "        # CL cells\n",
    "        # NOTE: Each graph convnet cell uses *TWO* convolutional operations\n",
    "        net_layers_extended = [D] + net_layers # include embedding dim\n",
    "        L = len(net_layers)\n",
    "        list_of_gnn_cells = [] # list of NN cells\n",
    "        for layer in range(L//2):\n",
    "            Hin, Hout = net_layers_extended[2*layer], net_layers_extended[2*layer+2]\n",
    "            list_of_gnn_cells.append(OurConvNetcell(Hin,Hout))\n",
    "            \n",
    "        # register the cells for pytorch\n",
    "        self.gnn_cells = nn.ModuleList(list_of_gnn_cells)\n",
    "              \n",
    "        # fc\n",
    "        Hfinal = net_layers_extended[-1]\n",
    "        self.fc = nn.Linear(Hfinal,nb_clusters_target) \n",
    "        \n",
    "        # init\n",
    "        self.init_weights_Graph_OurConvNet(Voc,D,Hfinal,nb_clusters_target,1)\n",
    "        \n",
    "        # print\n",
    "        print('\\nnb of hidden layers=',L)\n",
    "        print('dim of layers (w/ embed dim)=',net_layers_extended)      \n",
    "        print('\\n')\n",
    "        \n",
    "        # class variables\n",
    "        self.L = L\n",
    "        self.net_layers_extended = net_layers_extended      \n",
    "        self.flag_task = flag_task\n",
    "        \n",
    "        \n",
    "    def init_weights_Graph_OurConvNet(self, Fin_enc, Fout_enc, Fin_fc, Fout_fc, gain):\n",
    "\n",
    "        scale = gain* np.sqrt( 2.0/ Fin_enc )\n",
    "        self.encoder.weight.data.uniform_(-scale, scale)  \n",
    "        scale = gain* np.sqrt( 2.0/ Fin_fc )\n",
    "        self.fc.weight.data.uniform_(-scale, scale)  \n",
    "        self.fc.bias.data.fill_(0)  \n",
    "    \n",
    "            \n",
    "    def forward(self, G):\n",
    "        \n",
    "        # signal\n",
    "        x = G.signal  # V-dim\n",
    "        x = Variable( torch.LongTensor(x).type(dtypeLong) , requires_grad=False)\n",
    "           \n",
    "        # encoder\n",
    "        x_emb = self.encoder(x) # V x D\n",
    "        \n",
    "        # graph operators\n",
    "        # Edge = start vertex to end vertex\n",
    "        # E_start = E x V mapping matrix from edge index to corresponding start vertex\n",
    "        # E_end = E x V mapping matrix from edge index to corresponding end vertex\n",
    "        E_start = G.edge_to_starting_vertex\n",
    "        E_end   = G.edge_to_ending_vertex \n",
    "        E_start = torch.from_numpy(E_start.toarray()).type(dtypeFloat)\n",
    "        E_end = torch.from_numpy(E_end.toarray()).type(dtypeFloat) \n",
    "        E_start = Variable( E_start , requires_grad=False) \n",
    "        E_end = Variable( E_end , requires_grad=False) \n",
    "        \n",
    "        # convnet cells  \n",
    "        x = x_emb\n",
    "        for layer in range(self.L//2):\n",
    "            gnn_layer = self.gnn_cells[layer]            \n",
    "            x = gnn_layer(x,E_start,E_end) # V x Hfinal\n",
    "            \n",
    "        # FC\n",
    "        x = self.fc(x)   \n",
    "        \n",
    "        return x\n",
    "        \n",
    "        \n",
    "    def loss(self, y, y_target, weight):\n",
    "        \n",
    "        loss = nn.CrossEntropyLoss(weight=weight.type(dtypeFloat))(y,y_target)\n",
    "        \n",
    "        return loss\n",
    "       \n",
    "        \n",
    "    def update(self, lr):\n",
    "                \n",
    "        update = torch.optim.Adam( self.parameters(), lr=lr )\n",
    "        \n",
    "        return update\n",
    "    \n",
    "    \n",
    "    def update_learning_rate(self, optimizer, lr):\n",
    "   \n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n",
    "\n",
    "        return optimizer\n",
    "    \n",
    "    \n",
    "    def nb_param(self):\n",
    "\n",
    "        return self.nb_param\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#################\n",
    "# network and optimization parameters\n",
    "#################\n",
    "\n",
    "# network parameters\n",
    "net_parameters = {}\n",
    "net_parameters['Voc'] = task_parameters['Voc']\n",
    "net_parameters['D'] = 50\n",
    "net_parameters['nb_clusters_target'] = task_parameters['nb_clusters_target']\n",
    "net_parameters['H'] = 50\n",
    "net_parameters['L'] = 10\n",
    "#print(net_parameters)\n",
    "\n",
    "\n",
    "# optimization parameters\n",
    "opt_parameters = {}\n",
    "opt_parameters['learning_rate'] = 0.00075   # ADAM\n",
    "opt_parameters['max_iters'] = 5000   \n",
    "opt_parameters['batch_iters'] = 100\n",
    "if 2==1: # fast debugging\n",
    "    opt_parameters['max_iters'] = 101 \n",
    "    opt_parameters['batch_iters'] = 10\n",
    "opt_parameters['decay_rate'] = 1.25   \n",
    "#print(opt_parameters)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No existing network to delete\n",
      "\n",
      "\n",
      "nb of hidden layers= 10\n",
      "dim of layers (w/ embed dim)= [50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50]\n",
      "\n",
      "\n",
      "Graph_OurConvNet(\n",
      "  (encoder): Embedding(11, 50)\n",
      "  (gnn_cells): ModuleList(\n",
      "    (0): OurConvNetcell(\n",
      "      (Ui1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Ui2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (bn1): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (bn2): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (R): Linear(in_features=50, out_features=50, bias=False)\n",
      "    )\n",
      "    (1): OurConvNetcell(\n",
      "      (Ui1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Ui2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (bn1): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (bn2): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (R): Linear(in_features=50, out_features=50, bias=False)\n",
      "    )\n",
      "    (2): OurConvNetcell(\n",
      "      (Ui1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Ui2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (bn1): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (bn2): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (R): Linear(in_features=50, out_features=50, bias=False)\n",
      "    )\n",
      "    (3): OurConvNetcell(\n",
      "      (Ui1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Ui2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (bn1): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (bn2): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (R): Linear(in_features=50, out_features=50, bias=False)\n",
      "    )\n",
      "    (4): OurConvNetcell(\n",
      "      (Ui1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj1): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Ui2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Uj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vi2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (Vj2): Linear(in_features=50, out_features=50, bias=False)\n",
      "      (bn1): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (bn2): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True)\n",
      "      (R): Linear(in_features=50, out_features=50, bias=False)\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear(in_features=50, out_features=10, bias=True)\n",
      ")\n",
      "nb_param= 115560  L= 10\n",
      "\n",
      "iteration= 0, loss(100iter)= 2.370, lr= 0.00075000, time(100iter)= 0.24\n",
      "accuracy= 13.301\n",
      "\n",
      "iteration= 100, loss(100iter)= 1.863, lr= 0.00075000, time(100iter)= 12.55\n",
      "accuracy= 35.473\n",
      "\n",
      "iteration= 200, loss(100iter)= 1.166, lr= 0.00075000, time(100iter)= 12.12\n",
      "accuracy= 55.437\n",
      "\n",
      "iteration= 300, loss(100iter)= 0.871, lr= 0.00075000, time(100iter)= 12.28\n",
      "accuracy= 63.005\n",
      "\n",
      "iteration= 400, loss(100iter)= 0.754, lr= 0.00075000, time(100iter)= 10.56\n",
      "accuracy= 65.931\n",
      "\n",
      "iteration= 500, loss(100iter)= 0.656, lr= 0.00075000, time(100iter)= 11.90\n",
      "accuracy= 69.349\n",
      "\n",
      "iteration= 600, loss(100iter)= 0.636, lr= 0.00075000, time(100iter)= 12.31\n",
      "accuracy= 70.493\n",
      "\n",
      "iteration= 700, loss(100iter)= 0.597, lr= 0.00075000, time(100iter)= 10.71\n",
      "accuracy= 73.124\n",
      "\n",
      "iteration= 800, loss(100iter)= 0.603, lr= 0.00060000, time(100iter)= 11.28\n",
      "accuracy= 72.737\n",
      "\n",
      "iteration= 900, loss(100iter)= 0.572, lr= 0.00060000, time(100iter)= 10.65\n",
      "accuracy= 74.056\n",
      "\n",
      "iteration= 1000, loss(100iter)= 0.560, lr= 0.00060000, time(100iter)= 10.58\n",
      "accuracy= 74.709\n",
      "\n",
      "iteration= 1100, loss(100iter)= 0.630, lr= 0.00048000, time(100iter)= 11.38\n",
      "accuracy= 71.877\n",
      "\n",
      "iteration= 1200, loss(100iter)= 0.576, lr= 0.00048000, time(100iter)= 12.32\n",
      "accuracy= 73.622\n",
      "\n",
      "iteration= 1300, loss(100iter)= 0.526, lr= 0.00048000, time(100iter)= 10.72\n",
      "accuracy= 75.304\n",
      "\n",
      "iteration= 1400, loss(100iter)= 0.506, lr= 0.00048000, time(100iter)= 10.90\n",
      "accuracy= 75.543\n",
      "\n",
      "iteration= 1500, loss(100iter)= 0.548, lr= 0.00038400, time(100iter)= 10.93\n",
      "accuracy= 73.963\n",
      "\n",
      "iteration= 1600, loss(100iter)= 0.482, lr= 0.00038400, time(100iter)= 11.23\n",
      "accuracy= 75.178\n",
      "\n",
      "iteration= 1700, loss(100iter)= 0.529, lr= 0.00030720, time(100iter)= 11.14\n",
      "accuracy= 73.873\n",
      "\n",
      "iteration= 1800, loss(100iter)= 0.524, lr= 0.00024576, time(100iter)= 11.06\n",
      "accuracy= 74.025\n",
      "\n",
      "iteration= 1900, loss(100iter)= 0.522, lr= 0.00019661, time(100iter)= 10.70\n",
      "accuracy= 74.831\n",
      "\n",
      "iteration= 2000, loss(100iter)= 0.450, lr= 0.00019661, time(100iter)= 11.12\n",
      "accuracy= 77.495\n",
      "\n",
      "iteration= 2100, loss(100iter)= 0.489, lr= 0.00015729, time(100iter)= 11.23\n",
      "accuracy= 75.052\n",
      "\n",
      "iteration= 2200, loss(100iter)= 0.493, lr= 0.00012583, time(100iter)= 10.57\n",
      "accuracy= 75.294\n",
      "\n",
      "iteration= 2300, loss(100iter)= 0.466, lr= 0.00012583, time(100iter)= 10.85\n",
      "accuracy= 75.838\n",
      "\n",
      "iteration= 2400, loss(100iter)= 0.490, lr= 0.00010066, time(100iter)= 10.87\n",
      "accuracy= 74.986\n",
      "\n",
      "iteration= 2500, loss(100iter)= 0.496, lr= 0.00008053, time(100iter)= 11.31\n",
      "accuracy= 75.401\n",
      "\n",
      "iteration= 2600, loss(100iter)= 0.464, lr= 0.00008053, time(100iter)= 10.81\n",
      "accuracy= 75.339\n",
      "\n",
      "iteration= 2700, loss(100iter)= 0.445, lr= 0.00008053, time(100iter)= 10.79\n",
      "accuracy= 76.265\n",
      "\n",
      "iteration= 2800, loss(100iter)= 0.420, lr= 0.00008053, time(100iter)= 11.17\n",
      "accuracy= 78.075\n",
      "\n",
      "iteration= 2900, loss(100iter)= 0.512, lr= 0.00006442, time(100iter)= 10.48\n",
      "accuracy= 74.488\n",
      "\n",
      "iteration= 3000, loss(100iter)= 0.484, lr= 0.00006442, time(100iter)= 10.58\n",
      "accuracy= 76.304\n",
      "\n",
      "iteration= 3100, loss(100iter)= 0.474, lr= 0.00006442, time(100iter)= 10.75\n",
      "accuracy= 75.509\n",
      "\n",
      "iteration= 3200, loss(100iter)= 0.471, lr= 0.00005154, time(100iter)= 11.67\n",
      "accuracy= 75.440\n",
      "\n",
      "iteration= 3300, loss(100iter)= 0.436, lr= 0.00005154, time(100iter)= 11.28\n",
      "accuracy= 76.223\n",
      "\n",
      "iteration= 3400, loss(100iter)= 0.471, lr= 0.00004123, time(100iter)= 10.94\n",
      "accuracy= 76.275\n",
      "\n",
      "iteration= 3500, loss(100iter)= 0.452, lr= 0.00004123, time(100iter)= 11.00\n",
      "accuracy= 75.767\n",
      "\n",
      "iteration= 3600, loss(100iter)= 0.514, lr= 0.00003299, time(100iter)= 10.54\n",
      "accuracy= 74.422\n",
      "\n",
      "iteration= 3700, loss(100iter)= 0.461, lr= 0.00003299, time(100iter)= 10.66\n",
      "accuracy= 75.832\n",
      "\n",
      "iteration= 3800, loss(100iter)= 0.448, lr= 0.00003299, time(100iter)= 11.58\n",
      "accuracy= 76.367\n",
      "\n",
      "iteration= 3900, loss(100iter)= 0.447, lr= 0.00002639, time(100iter)= 10.79\n",
      "accuracy= 76.152\n",
      "\n",
      "iteration= 4000, loss(100iter)= 0.487, lr= 0.00002111, time(100iter)= 11.80\n",
      "accuracy= 75.250\n",
      "\n",
      "iteration= 4100, loss(100iter)= 0.464, lr= 0.00002111, time(100iter)= 10.54\n",
      "accuracy= 76.238\n",
      "\n",
      "iteration= 4200, loss(100iter)= 0.479, lr= 0.00001689, time(100iter)= 11.43\n",
      "accuracy= 75.530\n",
      "\n",
      "iteration= 4300, loss(100iter)= 0.397, lr= 0.00001689, time(100iter)= 11.25\n",
      "accuracy= 78.194\n",
      "\n",
      "iteration= 4400, loss(100iter)= 0.441, lr= 0.00001351, time(100iter)= 11.00\n",
      "accuracy= 76.632\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "iteration= 4500, loss(100iter)= 0.501, lr= 0.00001081, time(100iter)= 10.70\n",
      "accuracy= 74.700\n",
      "\n",
      "iteration= 4600, loss(100iter)= 0.486, lr= 0.00001081, time(100iter)= 11.21\n",
      "accuracy= 74.789\n",
      "\n",
      "iteration= 4700, loss(100iter)= 0.446, lr= 0.00001081, time(100iter)= 10.98\n",
      "accuracy= 76.223\n",
      "\n",
      "iteration= 4800, loss(100iter)= 0.450, lr= 0.00000865, time(100iter)= 11.64\n",
      "accuracy= 75.884\n",
      "\n",
      "iteration= 4900, loss(100iter)= 0.425, lr= 0.00000865, time(100iter)= 11.96\n",
      "accuracy= 76.462\n",
      "\n",
      "loss(100 pre-saved data)= 0.455, accuracy(100 pre-saved data)= 76.070\n"
     ]
    }
   ],
   "source": [
    " \n",
    "#########################\n",
    "# Graph convnet function\n",
    "#########################\n",
    "def our_graph_convnets(task_parameters,net_parameters,opt_parameters):\n",
    "\n",
    "\n",
    "    # Delete existing network if exists\n",
    "    try:\n",
    "        del net\n",
    "        print('Delete existing network\\n')\n",
    "    except NameError:\n",
    "        print('No existing network to delete\\n')\n",
    "\n",
    "\n",
    "    # instantiate\n",
    "    net = Graph_OurConvNet(net_parameters)\n",
    "    if torch.cuda.is_available():\n",
    "        net.cuda()\n",
    "    print(net)\n",
    "    \n",
    "    \n",
    "    # number of network parameters\n",
    "    nb_param = 0\n",
    "    for param in net.parameters():\n",
    "        nb_param += np.prod(list(param.data.size()))\n",
    "    print('nb_param=',nb_param,' L=',net_parameters['L'])\n",
    "    \n",
    "\n",
    "    # task parameters\n",
    "    flag_task = task_parameters['flag_task']\n",
    "    # network parameters\n",
    "    Voc = net_parameters['Voc']\n",
    "    D = net_parameters['D']\n",
    "    nb_clusters_target = net_parameters['nb_clusters_target']\n",
    "    H = net_parameters['H']\n",
    "    L = net_parameters['L']\n",
    "    # optimization parameters\n",
    "    learning_rate = opt_parameters['learning_rate']\n",
    "    max_iters = opt_parameters['max_iters']\n",
    "    batch_iters = opt_parameters['batch_iters']\n",
    "    decay_rate = opt_parameters['decay_rate']\n",
    "    \n",
    "    \n",
    "    # Optimizer\n",
    "    global_lr = learning_rate\n",
    "    global_step = 0\n",
    "    lr = learning_rate\n",
    "    optimizer = net.update(lr) \n",
    "\n",
    "    \n",
    "    #############\n",
    "    # loop over epochs\n",
    "    #############\n",
    "    t_start = time.time()\n",
    "    t_start_total = time.time()\n",
    "    average_loss_old = 1e10\n",
    "    running_loss = 0.0\n",
    "    running_total = 0\n",
    "    running_conf_mat = 0\n",
    "    running_accuracy = 0\n",
    "    tab_results = []\n",
    "    for iteration in range(1*max_iters):  # loop over the dataset multiple times\n",
    "\n",
    "        # generate one train graph\n",
    "        if flag_task=='matching': # subgraph matching\n",
    "            train_x = g.variable_size_graph(task_parameters)\n",
    "        elif flag_task=='clustering': # semi supervised clustering\n",
    "            train_x = g.graph_semi_super_clu(task_parameters)\n",
    "        train_y = train_x.target\n",
    "        train_y = Variable( torch.LongTensor(train_y).type(dtypeLong) , requires_grad=False) \n",
    "\n",
    "        # forward, loss\n",
    "        y = net.forward(train_x)\n",
    "        # compute loss weigth\n",
    "        labels = train_y.data.cpu().numpy()\n",
    "        V = labels.shape[0]\n",
    "        nb_classes = len(np.unique(labels)) \n",
    "        cluster_sizes = np.zeros(nb_classes)\n",
    "        for r in range(nb_classes):\n",
    "            cluster = np.where(labels==r)[0]\n",
    "            cluster_sizes[r] = len(cluster)    \n",
    "        weight = torch.zeros(nb_classes)\n",
    "        for r in range(nb_classes):\n",
    "            sumj = 0\n",
    "            for j in range(nb_classes):\n",
    "                if j!=r:\n",
    "                    sumj += cluster_sizes[j]\n",
    "            weight[r] = sumj/ V \n",
    "        loss = net.loss(y,train_y,weight)\n",
    "        loss_train = loss.data[0]\n",
    "        running_loss += loss_train\n",
    "        running_total += 1\n",
    "\n",
    "        # confusion matrix\n",
    "        S = train_y.data.cpu().numpy()\n",
    "        C = np.argmax( torch.nn.Softmax(dim=0)(y).data.cpu().numpy() , axis=1)\n",
    "        CM = confusion_matrix(S,C).astype(np.float32)\n",
    "        nb_classes = CM.shape[0]\n",
    "        train_y = train_y.data.cpu().numpy()\n",
    "        for r in range(nb_classes):\n",
    "            cluster = np.where(train_y==r)[0]\n",
    "            CM[r,:] /= cluster.shape[0]\n",
    "        running_conf_mat += CM\n",
    "        running_accuracy += np.sum(np.diag(CM))/ nb_classes\n",
    "\n",
    "        # backward, update\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # learning rate, print results\n",
    "        if not iteration%batch_iters:\n",
    "\n",
    "            # time\n",
    "            t_stop = time.time() - t_start\n",
    "            t_start = time.time()\n",
    "\n",
    "            # confusion matrix\n",
    "            average_conf_mat = running_conf_mat/ running_total\n",
    "            running_conf_mat = 0\n",
    "\n",
    "            # accuracy\n",
    "            average_accuracy = running_accuracy/ running_total\n",
    "            running_accuracy = 0\n",
    "\n",
    "            # update learning rate \n",
    "            average_loss = running_loss/ running_total\n",
    "            if average_loss > 0.99* average_loss_old:\n",
    "                lr /= decay_rate\n",
    "            average_loss_old = average_loss\n",
    "            optimizer = net.update_learning_rate(optimizer, lr)\n",
    "            running_loss = 0.0\n",
    "            running_total = 0\n",
    "\n",
    "            # save intermediate results\n",
    "            tab_results.append([iteration,average_loss,100* average_accuracy,time.time()-t_start_total])\n",
    "\n",
    "            # print results\n",
    "            if 1==1:\n",
    "                print('\\niteration= %d, loss(%diter)= %.3f, lr= %.8f, time(%diter)= %.2f' % \n",
    "                      (iteration, batch_iters, average_loss, lr, batch_iters, t_stop))\n",
    "                #print('Confusion matrix= \\n', 100* average_conf_mat)\n",
    "                print('accuracy= %.3f' % (100* average_accuracy))\n",
    "\n",
    "                \n",
    "                \n",
    "                \n",
    "                \n",
    "                \n",
    "    ############            \n",
    "    # Evaluation on 100 pre-saved data\n",
    "    ############\n",
    "    running_loss = 0.0\n",
    "    running_total = 0\n",
    "    running_conf_mat = 0\n",
    "    running_accuracy = 0\n",
    "    for iteration in range(100):\n",
    "        \n",
    "        # generate one data\n",
    "        if flag_task == 'matching':\n",
    "            train_x = g.variable_size_graph(task_parameters)\n",
    "        if flag_task == 'clustering':\n",
    "            train_x = task_parameters['all_trainx'][iteration][1]\n",
    "        train_y = train_x.target\n",
    "        train_y = Variable( torch.LongTensor(train_y).type(dtypeLong) , requires_grad=False) \n",
    "        \n",
    "        # forward, loss\n",
    "        y = net.forward(train_x)\n",
    "        # compute loss weigth\n",
    "        labels = train_y.data.cpu().numpy()\n",
    "        V = labels.shape[0]\n",
    "        nb_classes = len(np.unique(labels)) \n",
    "        cluster_sizes = np.zeros(nb_classes)\n",
    "        for r in range(nb_classes):\n",
    "            cluster = np.where(labels==r)[0]\n",
    "            cluster_sizes[r] = len(cluster)    \n",
    "        weight = torch.zeros(nb_classes)\n",
    "        for r in range(nb_classes):\n",
    "            sumj = 0\n",
    "            for j in range(nb_classes):\n",
    "                if j!=r:\n",
    "                    sumj += cluster_sizes[j]\n",
    "            weight[r] = sumj/ V \n",
    "        loss = net.loss(y,train_y,weight)\n",
    "        loss_train = loss.data[0]\n",
    "        running_loss += loss_train\n",
    "        running_total += 1\n",
    "        \n",
    "        # confusion matrix\n",
    "        S = train_y.data.cpu().numpy()\n",
    "        C = np.argmax( torch.nn.Softmax(dim=0)(y).data.cpu().numpy() , axis=1)\n",
    "        CM = confusion_matrix(S,C).astype(np.float32)\n",
    "        nb_classes = CM.shape[0]\n",
    "        train_y = train_y.data.cpu().numpy()\n",
    "        for r in range(nb_classes):\n",
    "            cluster = np.where(train_y==r)[0]\n",
    "            CM[r,:] /= cluster.shape[0]\n",
    "        running_conf_mat += CM\n",
    "        running_accuracy += np.sum(np.diag(CM))/ nb_classes\n",
    "\n",
    "        # confusion matrix\n",
    "        average_conf_mat = running_conf_mat/ running_total\n",
    "        average_accuracy = running_accuracy/ running_total\n",
    "        average_loss = running_loss/ running_total\n",
    "        \n",
    "    # print results\n",
    "    print('\\nloss(100 pre-saved data)= %.3f, accuracy(100 pre-saved data)= %.3f' % (average_loss,100* average_accuracy))\n",
    "\n",
    "        \n",
    "        \n",
    "        \n",
    "    #############\n",
    "    # output\n",
    "    #############\n",
    "    result = {}\n",
    "    result['final_loss'] = average_loss\n",
    "    result['final_acc'] = 100* average_accuracy\n",
    "    result['final_CM'] = 100* average_conf_mat\n",
    "    result['final_batch_time'] = t_stop\n",
    "    result['nb_param_nn'] = nb_param\n",
    "    result['plot_all_epochs'] = tab_results\n",
    "    #print(result)\n",
    "    \n",
    "    \n",
    "    return result\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#run it\n",
    "result = our_graph_convnets(task_parameters,net_parameters,opt_parameters)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}