02_graph_convnet_lenet5_mnist_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Graph Convolutional Neural Networks\n",
    "## Graph LeNet5 with PyTorch\n",
    "### Xavier Bresson, Oct. 2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of spectral graph ConvNets<br>\n",
    "Convolutional Neural Networks on Graphs with Fast Localized Spectral Filtering<br>\n",
    "M Defferrard, X Bresson, P Vandergheynst<br>\n",
    "Advances in Neural Information Processing Systems, 3844-3852, 2016<br>\n",
    "ArXiv preprint: [arXiv:1606.09375](https://arxiv.org/pdf/1606.09375.pdf) <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda available\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import pdb #pdb.set_trace()\n",
    "import collections\n",
    "import time\n",
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, 'lib/')\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print('cuda available')\n",
    "    dtypeFloat = torch.cuda.FloatTensor\n",
    "    dtypeLong = torch.cuda.LongTensor\n",
    "    torch.cuda.manual_seed(1)\n",
    "else:\n",
    "    print('cuda not available')\n",
    "    dtypeFloat = torch.FloatTensor\n",
    "    dtypeLong = torch.LongTensor\n",
    "    torch.manual_seed(1)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting datasets/train-images-idx3-ubyte.gz\n",
      "Extracting datasets/train-labels-idx1-ubyte.gz\n",
      "Extracting datasets/t10k-images-idx3-ubyte.gz\n",
      "Extracting datasets/t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n",
      "(55000,)\n",
      "(5000, 784)\n",
      "(5000,)\n",
      "(10000, 784)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('datasets', one_hot=False) # load data in folder datasets/\n",
    "\n",
    "train_data = mnist.train.images.astype(np.float32)\n",
    "val_data = mnist.validation.images.astype(np.float32)\n",
    "test_data = mnist.test.images.astype(np.float32)\n",
    "train_labels = mnist.train.labels\n",
    "val_labels = mnist.validation.labels\n",
    "test_labels = mnist.test.labels\n",
    "print(train_data.shape)\n",
    "print(train_labels.shape)\n",
    "print(val_data.shape)\n",
    "print(val_labels.shape)\n",
    "print(test_data.shape)\n",
    "print(test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nb edges:  6396\n",
      "Heavy Edge Matching coarsening with Xavier version\n",
      "Layer 0: M_0 = |V| = 928 nodes (144 added), |E| = 3198 edges\n",
      "Layer 1: M_1 = |V| = 464 nodes (61 added), |E| = 1592 edges\n",
      "Layer 2: M_2 = |V| = 232 nodes (22 added), |E| = 772 edges\n",
      "Layer 3: M_3 = |V| = 116 nodes (6 added), |E| = 370 edges\n",
      "Layer 4: M_4 = |V| = 58 nodes (0 added), |E| = 189 edges\n",
      "lmax: [1.3857549, 1.3441432, 1.219873, 0.99999928]\n",
      "(55000, 928)\n",
      "(5000, 928)\n",
      "(10000, 928)\n",
      "Execution time: 1.06s\n"
     ]
    }
   ],
   "source": [
    "from lib.grid_graph import grid_graph\n",
    "from lib.coarsening import coarsen\n",
    "from lib.coarsening import lmax_L\n",
    "from lib.coarsening import perm_data\n",
    "from lib.coarsening import rescale_L\n",
    "\n",
    "# Construct graph\n",
    "t_start = time.time()\n",
    "grid_side = 28\n",
    "number_edges = 8\n",
    "metric = 'euclidean'\n",
    "A = grid_graph(grid_side,number_edges,metric) # create graph of Euclidean grid\n",
    "\n",
    "# Compute coarsened graphs\n",
    "coarsening_levels = 4\n",
    "L, perm = coarsen(A, coarsening_levels)\n",
    "\n",
    "# Compute max eigenvalue of graph Laplacians\n",
    "lmax = []\n",
    "for i in range(coarsening_levels):\n",
    "    lmax.append(lmax_L(L[i]))\n",
    "print('lmax: ' + str([lmax[i] for i in range(coarsening_levels)]))\n",
    "\n",
    "# Reindex nodes to satisfy a binary tree structure\n",
    "train_data = perm_data(train_data, perm)\n",
    "val_data = perm_data(val_data, perm)\n",
    "test_data = perm_data(test_data, perm)\n",
    "\n",
    "print(train_data.shape)\n",
    "print(val_data.shape)\n",
    "print(test_data.shape)\n",
    "\n",
    "print('Execution time: {:.2f}s'.format(time.time() - t_start))\n",
    "del perm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Graph ConvNet LeNet5\n",
    "### Layers: CL32-MP4-CL64-MP4-FC512-FC10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# class definitions\n",
    "\n",
    "class my_sparse_mm(torch.autograd.Function):\n",
    "    \"\"\"\n",
    "    Implementation of a new autograd function for sparse variables, \n",
    "    called \"my_sparse_mm\", by subclassing torch.autograd.Function \n",
    "    and implementing the forward and backward passes.\n",
    "    \"\"\"\n",
    "    \n",
    "    def forward(self, W, x):  # W is SPARSE\n",
    "        self.save_for_backward(W, x)\n",
    "        y = torch.mm(W, x)\n",
    "        return y\n",
    "    \n",
    "    def backward(self, grad_output):\n",
    "        W, x = self.saved_tensors \n",
    "        grad_input = grad_output.clone()\n",
    "        grad_input_dL_dW = torch.mm(grad_input, x.t()) \n",
    "        grad_input_dL_dx = torch.mm(W.t(), grad_input )\n",
    "        return grad_input_dL_dW, grad_input_dL_dx\n",
    "    \n",
    "    \n",
    "class Graph_ConvNet_LeNet5(nn.Module):\n",
    "    \n",
    "    def __init__(self, net_parameters):\n",
    "        \n",
    "        print('Graph ConvNet: LeNet5')\n",
    "        \n",
    "        super(Graph_ConvNet_LeNet5, self).__init__()\n",
    "        \n",
    "        # parameters\n",
    "        D, CL1_F, CL1_K, CL2_F, CL2_K, FC1_F, FC2_F = net_parameters\n",
    "        FC1Fin = CL2_F*(D//16)\n",
    "        \n",
    "        # graph CL1\n",
    "        self.cl1 = nn.Linear(CL1_K, CL1_F) \n",
    "        Fin = CL1_K; Fout = CL1_F;\n",
    "        scale = np.sqrt( 2.0/ (Fin+Fout) )\n",
    "        self.cl1.weight.data.uniform_(-scale, scale)\n",
    "        self.cl1.bias.data.fill_(0.0)\n",
    "        self.CL1_K = CL1_K; self.CL1_F = CL1_F; \n",
    "        \n",
    "        # graph CL2\n",
    "        self.cl2 = nn.Linear(CL2_K*CL1_F, CL2_F) \n",
    "        Fin = CL2_K*CL1_F; Fout = CL2_F;\n",
    "        scale = np.sqrt( 2.0/ (Fin+Fout) )\n",
    "        self.cl2.weight.data.uniform_(-scale, scale)\n",
    "        self.cl2.bias.data.fill_(0.0)\n",
    "        self.CL2_K = CL2_K; self.CL2_F = CL2_F; \n",
    "\n",
    "        # FC1\n",
    "        self.fc1 = nn.Linear(FC1Fin, FC1_F) \n",
    "        Fin = FC1Fin; Fout = FC1_F;\n",
    "        scale = np.sqrt( 2.0/ (Fin+Fout) )\n",
    "        self.fc1.weight.data.uniform_(-scale, scale)\n",
    "        self.fc1.bias.data.fill_(0.0)\n",
    "        self.FC1Fin = FC1Fin\n",
    "        \n",
    "        # FC2\n",
    "        self.fc2 = nn.Linear(FC1_F, FC2_F)\n",
    "        Fin = FC1_F; Fout = FC2_F;\n",
    "        scale = np.sqrt( 2.0/ (Fin+Fout) )\n",
    "        self.fc2.weight.data.uniform_(-scale, scale)\n",
    "        self.fc2.bias.data.fill_(0.0)\n",
    "\n",
    "        # nb of parameters\n",
    "        nb_param = CL1_K* CL1_F + CL1_F          # CL1\n",
    "        nb_param += CL2_K* CL1_F* CL2_F + CL2_F  # CL2\n",
    "        nb_param += FC1Fin* FC1_F + FC1_F        # FC1\n",
    "        nb_param += FC1_F* FC2_F + FC2_F         # FC2\n",
    "        print('nb of parameters=',nb_param,'\\n')\n",
    "        \n",
    "        \n",
    "    def init_weights(self, W, Fin, Fout):\n",
    "\n",
    "        scale = np.sqrt( 2.0/ (Fin+Fout) )\n",
    "        W.uniform_(-scale, scale)\n",
    "\n",
    "        return W\n",
    "        \n",
    "        \n",
    "    def graph_conv_cheby(self, x, cl, L, lmax, Fout, K):\n",
    "\n",
    "        # parameters\n",
    "        # B = batch size\n",
    "        # V = nb vertices\n",
    "        # Fin = nb input features\n",
    "        # Fout = nb output features\n",
    "        # K = Chebyshev order & support size\n",
    "        B, V, Fin = x.size(); B, V, Fin = int(B), int(V), int(Fin) \n",
    "\n",
    "        # rescale Laplacian\n",
    "        lmax = lmax_L(L)\n",
    "        L = rescale_L(L, lmax) \n",
    "        \n",
    "        # convert scipy sparse matric L to pytorch\n",
    "        L = L.tocoo()\n",
    "        indices = np.column_stack((L.row, L.col)).T \n",
    "        indices = indices.astype(np.int64)\n",
    "        indices = torch.from_numpy(indices)\n",
    "        indices = indices.type(torch.LongTensor)\n",
    "        L_data = L.data.astype(np.float32)\n",
    "        L_data = torch.from_numpy(L_data) \n",
    "        L_data = L_data.type(torch.FloatTensor)\n",
    "        L = torch.sparse.FloatTensor(indices, L_data, torch.Size(L.shape))\n",
    "        L = Variable( L , requires_grad=False)\n",
    "        if torch.cuda.is_available():\n",
    "            L = L.cuda()\n",
    "        \n",
    "        # transform to Chebyshev basis\n",
    "        x0 = x.permute(1,2,0).contiguous()  # V x Fin x B\n",
    "        x0 = x0.view([V, Fin*B])            # V x Fin*B\n",
    "        x = x0.unsqueeze(0)                 # 1 x V x Fin*B\n",
    "        \n",
    "        def concat(x, x_):\n",
    "            x_ = x_.unsqueeze(0)            # 1 x V x Fin*B\n",
    "            return torch.cat((x, x_), 0)    # K x V x Fin*B  \n",
    "             \n",
    "        if K > 1: \n",
    "            x1 = my_sparse_mm()(L,x0)              # V x Fin*B\n",
    "            x = torch.cat((x, x1.unsqueeze(0)),0)  # 2 x V x Fin*B\n",
    "        for k in range(2, K):\n",
    "            x2 = 2 * my_sparse_mm()(L,x1) - x0  \n",
    "            x = torch.cat((x, x2.unsqueeze(0)),0)  # M x Fin*B\n",
    "            x0, x1 = x1, x2  \n",
    "        \n",
    "        x = x.view([K, V, Fin, B])           # K x V x Fin x B     \n",
    "        x = x.permute(3,1,2,0).contiguous()  # B x V x Fin x K       \n",
    "        x = x.view([B*V, Fin*K])             # B*V x Fin*K\n",
    "        \n",
    "        # Compose linearly Fin features to get Fout features\n",
    "        x = cl(x)                            # B*V x Fout  \n",
    "        x = x.view([B, V, Fout])             # B x V x Fout\n",
    "        \n",
    "        return x\n",
    "        \n",
    "        \n",
    "    # Max pooling of size p. Must be a power of 2.\n",
    "    def graph_max_pool(self, x, p): \n",
    "        if p > 1: \n",
    "            x = x.permute(0,2,1).contiguous()  # x = B x F x V\n",
    "            x = nn.MaxPool1d(p)(x)             # B x F x V/p          \n",
    "            x = x.permute(0,2,1).contiguous()  # x = B x V/p x F\n",
    "            return x  \n",
    "        else:\n",
    "            return x    \n",
    "        \n",
    "        \n",
    "    def forward(self, x, d, L, lmax):\n",
    "        \n",
    "        # graph CL1\n",
    "        x = x.unsqueeze(2) # B x V x Fin=1  \n",
    "        x = self.graph_conv_cheby(x, self.cl1, L[0], lmax[0], self.CL1_F, self.CL1_K)\n",
    "        x = F.relu(x)\n",
    "        x = self.graph_max_pool(x, 4)\n",
    "        \n",
    "        # graph CL2\n",
    "        x = self.graph_conv_cheby(x, self.cl2, L[2], lmax[2], self.CL2_F, self.CL2_K)\n",
    "        x = F.relu(x)\n",
    "        x = self.graph_max_pool(x, 4)\n",
    "        \n",
    "        # FC1\n",
    "        x = x.view(-1, self.FC1Fin)\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x  = nn.Dropout(d)(x)\n",
    "        \n",
    "        # FC2\n",
    "        x = self.fc2(x)\n",
    "            \n",
    "        return x\n",
    "        \n",
    "        \n",
    "    def loss(self, y, y_target, l2_regularization):\n",
    "    \n",
    "        loss = nn.CrossEntropyLoss()(y,y_target)\n",
    "\n",
    "        l2_loss = 0.0\n",
    "        for param in self.parameters():\n",
    "            data = param* param\n",
    "            l2_loss += data.sum()\n",
    "           \n",
    "        loss += 0.5* l2_regularization* l2_loss\n",
    "            \n",
    "        return loss\n",
    "    \n",
    "    \n",
    "    def update(self, lr):\n",
    "                \n",
    "        update = torch.optim.SGD( self.parameters(), lr=lr, momentum=0.9 )\n",
    "        \n",
    "        return update\n",
    "        \n",
    "        \n",
    "    def update_learning_rate(self, optimizer, lr):\n",
    "   \n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n",
    "\n",
    "        return optimizer\n",
    "\n",
    "    \n",
    "    def evaluation(self, y_predicted, test_l):\n",
    "    \n",
    "        _, class_predicted = torch.max(y_predicted.data, 1)\n",
    "        return 100.0* (class_predicted == test_l).sum()/ y_predicted.size(0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delete existing network\n",
      "\n",
      "Graph ConvNet: LeNet5\n",
      "nb of parameters= 1958282 \n",
      "\n",
      "Graph_ConvNet_LeNet5 (\n",
      "  (cl1): Linear (25 -> 32)\n",
      "  (cl2): Linear (800 -> 64)\n",
      "  (fc1): Linear (3712 -> 512)\n",
      "  (fc2): Linear (512 -> 10)\n",
      ")\n",
      "num_epochs= 20 , train_size= 55000 , nb_iter= 11000\n",
      "epoch= 1, i=  100, loss(batch)= 0.4129, accuray(batch)= 91.00\n",
      "epoch= 1, i=  200, loss(batch)= 0.3374, accuray(batch)= 92.00\n",
      "epoch= 1, i=  300, loss(batch)= 0.1941, accuray(batch)= 97.00\n",
      "epoch= 1, i=  400, loss(batch)= 0.2339, accuray(batch)= 96.00\n",
      "epoch= 1, i=  500, loss(batch)= 0.1828, accuray(batch)= 97.00\n",
      "epoch= 1, loss(train)= 0.396, accuracy(train)= 90.804, time= 100.878, lr= 0.05000\n",
      "  accuracy(test) = 97.730 %, time= 9.023\n",
      "epoch= 2, i=  100, loss(batch)= 0.1696, accuray(batch)= 98.00\n",
      "epoch= 2, i=  200, loss(batch)= 0.2761, accuray(batch)= 94.00\n",
      "epoch= 2, i=  300, loss(batch)= 0.1676, accuray(batch)= 98.00\n",
      "epoch= 2, i=  400, loss(batch)= 0.1188, accuray(batch)= 100.00\n",
      "epoch= 2, i=  500, loss(batch)= 0.2046, accuray(batch)= 95.00\n",
      "epoch= 2, loss(train)= 0.187, accuracy(train)= 97.549, time= 100.835, lr= 0.04750\n",
      "  accuracy(test) = 98.270 %, time= 8.987\n",
      "epoch= 3, i=  100, loss(batch)= 0.1839, accuray(batch)= 96.00\n",
      "epoch= 3, i=  200, loss(batch)= 0.1674, accuray(batch)= 98.00\n",
      "epoch= 3, i=  300, loss(batch)= 0.1397, accuray(batch)= 98.00\n",
      "epoch= 3, i=  400, loss(batch)= 0.1966, accuray(batch)= 98.00\n",
      "epoch= 3, i=  500, loss(batch)= 0.1546, accuray(batch)= 98.00\n",
      "epoch= 3, loss(train)= 0.155, accuracy(train)= 98.253, time= 101.344, lr= 0.04512\n",
      "  accuracy(test) = 98.650 %, time= 9.043\n",
      "epoch= 4, i=  100, loss(batch)= 0.2077, accuray(batch)= 97.00\n",
      "epoch= 4, i=  200, loss(batch)= 0.1644, accuray(batch)= 98.00\n",
      "epoch= 4, i=  300, loss(batch)= 0.1275, accuray(batch)= 99.00\n",
      "epoch= 4, i=  400, loss(batch)= 0.1430, accuray(batch)= 98.00\n",
      "epoch= 4, i=  500, loss(batch)= 0.1855, accuray(batch)= 99.00\n",
      "epoch= 4, loss(train)= 0.138, accuracy(train)= 98.542, time= 101.353, lr= 0.04287\n",
      "  accuracy(test) = 98.650 %, time= 9.012\n",
      "epoch= 5, i=  100, loss(batch)= 0.1073, accuray(batch)= 99.00\n",
      "epoch= 5, i=  200, loss(batch)= 0.1340, accuray(batch)= 98.00\n",
      "epoch= 5, i=  300, loss(batch)= 0.1362, accuray(batch)= 99.00\n",
      "epoch= 5, i=  400, loss(batch)= 0.1478, accuray(batch)= 98.00\n",
      "epoch= 5, i=  500, loss(batch)= 0.1501, accuray(batch)= 98.00\n",
      "epoch= 5, loss(train)= 0.122, accuracy(train)= 98.795, time= 101.034, lr= 0.04073\n",
      "  accuracy(test) = 98.740 %, time= 9.012\n",
      "epoch= 6, i=  100, loss(batch)= 0.1457, accuray(batch)= 98.00\n",
      "epoch= 6, i=  200, loss(batch)= 0.1073, accuray(batch)= 98.00\n",
      "epoch= 6, i=  300, loss(batch)= 0.1079, accuray(batch)= 99.00\n",
      "epoch= 6, i=  400, loss(batch)= 0.0891, accuray(batch)= 99.00\n",
      "epoch= 6, i=  500, loss(batch)= 0.0995, accuray(batch)= 99.00\n",
      "epoch= 6, loss(train)= 0.111, accuracy(train)= 98.978, time= 101.218, lr= 0.03869\n",
      "  accuracy(test) = 98.690 %, time= 9.030\n",
      "epoch= 7, i=  100, loss(batch)= 0.1062, accuray(batch)= 99.00\n",
      "epoch= 7, i=  200, loss(batch)= 0.0986, accuray(batch)= 99.00\n",
      "epoch= 7, i=  300, loss(batch)= 0.1089, accuray(batch)= 99.00\n",
      "epoch= 7, i=  400, loss(batch)= 0.0914, accuray(batch)= 99.00\n",
      "epoch= 7, i=  500, loss(batch)= 0.1146, accuray(batch)= 98.00\n",
      "epoch= 7, loss(train)= 0.102, accuracy(train)= 99.120, time= 101.632, lr= 0.03675\n",
      "  accuracy(test) = 98.990 %, time= 9.076\n",
      "epoch= 8, i=  100, loss(batch)= 0.0832, accuray(batch)= 100.00\n",
      "epoch= 8, i=  200, loss(batch)= 0.0797, accuray(batch)= 100.00\n",
      "epoch= 8, i=  300, loss(batch)= 0.0762, accuray(batch)= 100.00\n",
      "epoch= 8, i=  400, loss(batch)= 0.2062, accuray(batch)= 97.00\n",
      "epoch= 8, i=  500, loss(batch)= 0.0970, accuray(batch)= 99.00\n",
      "epoch= 8, loss(train)= 0.095, accuracy(train)= 99.164, time= 101.151, lr= 0.03492\n",
      "  accuracy(test) = 98.990 %, time= 9.021\n",
      "epoch= 9, i=  100, loss(batch)= 0.1012, accuray(batch)= 99.00\n",
      "epoch= 9, i=  200, loss(batch)= 0.0793, accuray(batch)= 100.00\n",
      "epoch= 9, i=  300, loss(batch)= 0.0771, accuray(batch)= 100.00\n",
      "epoch= 9, i=  400, loss(batch)= 0.0786, accuray(batch)= 100.00\n",
      "epoch= 9, i=  500, loss(batch)= 0.0899, accuray(batch)= 99.00\n",
      "epoch= 9, loss(train)= 0.089, accuracy(train)= 99.264, time= 101.987, lr= 0.03317\n",
      "  accuracy(test) = 99.180 %, time= 9.014\n",
      "epoch= 10, i=  100, loss(batch)= 0.0718, accuray(batch)= 99.00\n",
      "epoch= 10, i=  200, loss(batch)= 0.0646, accuray(batch)= 100.00\n",
      "epoch= 10, i=  300, loss(batch)= 0.1157, accuray(batch)= 99.00\n",
      "epoch= 10, i=  400, loss(batch)= 0.0718, accuray(batch)= 100.00\n",
      "epoch= 10, i=  500, loss(batch)= 0.1252, accuray(batch)= 98.00\n",
      "epoch= 10, loss(train)= 0.082, accuracy(train)= 99.367, time= 101.591, lr= 0.03151\n",
      "  accuracy(test) = 99.110 %, time= 9.033\n",
      "epoch= 11, i=  100, loss(batch)= 0.0681, accuray(batch)= 100.00\n",
      "epoch= 11, i=  200, loss(batch)= 0.0846, accuray(batch)= 99.00\n",
      "epoch= 11, i=  300, loss(batch)= 0.0703, accuray(batch)= 99.00\n",
      "epoch= 11, i=  400, loss(batch)= 0.0682, accuray(batch)= 100.00\n",
      "epoch= 11, i=  500, loss(batch)= 0.0870, accuray(batch)= 100.00\n",
      "epoch= 11, loss(train)= 0.079, accuracy(train)= 99.378, time= 101.530, lr= 0.02994\n",
      "  accuracy(test) = 99.060 %, time= 9.017\n",
      "epoch= 12, i=  100, loss(batch)= 0.0997, accuray(batch)= 98.00\n",
      "epoch= 12, i=  200, loss(batch)= 0.1114, accuray(batch)= 98.00\n",
      "epoch= 12, i=  300, loss(batch)= 0.0691, accuray(batch)= 99.00\n",
      "epoch= 12, i=  400, loss(batch)= 0.0671, accuray(batch)= 100.00\n",
      "epoch= 12, i=  500, loss(batch)= 0.0946, accuray(batch)= 98.00\n",
      "epoch= 12, loss(train)= 0.075, accuracy(train)= 99.458, time= 101.616, lr= 0.02844\n",
      "  accuracy(test) = 99.120 %, time= 9.091\n",
      "epoch= 13, i=  100, loss(batch)= 0.0869, accuray(batch)= 98.00\n",
      "epoch= 13, i=  200, loss(batch)= 0.0619, accuray(batch)= 100.00\n",
      "epoch= 13, i=  300, loss(batch)= 0.0930, accuray(batch)= 99.00\n",
      "epoch= 13, i=  400, loss(batch)= 0.0799, accuray(batch)= 99.00\n",
      "epoch= 13, i=  500, loss(batch)= 0.0586, accuray(batch)= 100.00\n",
      "epoch= 13, loss(train)= 0.072, accuracy(train)= 99.458, time= 101.749, lr= 0.02702\n",
      "  accuracy(test) = 99.100 %, time= 9.008\n",
      "epoch= 14, i=  100, loss(batch)= 0.0618, accuray(batch)= 100.00\n",
      "epoch= 14, i=  200, loss(batch)= 0.0577, accuray(batch)= 100.00\n",
      "epoch= 14, i=  300, loss(batch)= 0.0583, accuray(batch)= 100.00\n",
      "epoch= 14, i=  400, loss(batch)= 0.0607, accuray(batch)= 100.00\n",
      "epoch= 14, i=  500, loss(batch)= 0.0760, accuray(batch)= 99.00\n",
      "epoch= 14, loss(train)= 0.069, accuracy(train)= 99.516, time= 101.536, lr= 0.02567\n",
      "  accuracy(test) = 99.180 %, time= 9.018\n",
      "epoch= 15, i=  100, loss(batch)= 0.0555, accuray(batch)= 100.00\n",
      "epoch= 15, i=  200, loss(batch)= 0.0765, accuray(batch)= 99.00\n",
      "epoch= 15, i=  300, loss(batch)= 0.0752, accuray(batch)= 99.00\n",
      "epoch= 15, i=  400, loss(batch)= 0.0618, accuray(batch)= 100.00\n",
      "epoch= 15, i=  500, loss(batch)= 0.0734, accuray(batch)= 100.00\n",
      "epoch= 15, loss(train)= 0.066, accuracy(train)= 99.591, time= 101.262, lr= 0.02438\n",
      "  accuracy(test) = 99.280 %, time= 9.052\n",
      "epoch= 16, i=  100, loss(batch)= 0.0780, accuray(batch)= 99.00\n",
      "epoch= 16, i=  200, loss(batch)= 0.0516, accuray(batch)= 100.00\n",
      "epoch= 16, i=  300, loss(batch)= 0.0634, accuray(batch)= 100.00\n",
      "epoch= 16, i=  400, loss(batch)= 0.0531, accuray(batch)= 100.00\n",
      "epoch= 16, i=  500, loss(batch)= 0.0635, accuray(batch)= 99.00\n",
      "epoch= 16, loss(train)= 0.064, accuracy(train)= 99.600, time= 100.793, lr= 0.02316\n",
      "  accuracy(test) = 99.220 %, time= 9.053\n",
      "epoch= 17, i=  100, loss(batch)= 0.0884, accuray(batch)= 99.00\n",
      "epoch= 17, i=  200, loss(batch)= 0.0694, accuray(batch)= 99.00\n",
      "epoch= 17, i=  300, loss(batch)= 0.0541, accuray(batch)= 100.00\n",
      "epoch= 17, i=  400, loss(batch)= 0.0574, accuray(batch)= 100.00\n",
      "epoch= 17, i=  500, loss(batch)= 0.0516, accuray(batch)= 100.00\n",
      "epoch= 17, loss(train)= 0.062, accuracy(train)= 99.596, time= 100.811, lr= 0.02201\n",
      "  accuracy(test) = 99.200 %, time= 9.063\n",
      "epoch= 18, i=  100, loss(batch)= 0.0531, accuray(batch)= 100.00\n",
      "epoch= 18, i=  200, loss(batch)= 0.0755, accuray(batch)= 98.00\n",
      "epoch= 18, i=  300, loss(batch)= 0.0521, accuray(batch)= 100.00\n",
      "epoch= 18, i=  400, loss(batch)= 0.0612, accuray(batch)= 100.00\n",
      "epoch= 18, i=  500, loss(batch)= 0.0561, accuray(batch)= 100.00\n",
      "epoch= 18, loss(train)= 0.061, accuracy(train)= 99.615, time= 100.540, lr= 0.02091\n",
      "  accuracy(test) = 99.180 %, time= 9.043\n",
      "epoch= 19, i=  100, loss(batch)= 0.0559, accuray(batch)= 100.00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch= 19, i=  200, loss(batch)= 0.0570, accuray(batch)= 99.00\n",
      "epoch= 19, i=  300, loss(batch)= 0.0497, accuray(batch)= 100.00\n",
      "epoch= 19, i=  400, loss(batch)= 0.0596, accuray(batch)= 99.00\n",
      "epoch= 19, i=  500, loss(batch)= 0.0499, accuray(batch)= 100.00\n",
      "epoch= 19, loss(train)= 0.059, accuracy(train)= 99.629, time= 100.761, lr= 0.01986\n",
      "  accuracy(test) = 99.210 %, time= 9.012\n",
      "epoch= 20, i=  100, loss(batch)= 0.0495, accuray(batch)= 100.00\n",
      "epoch= 20, i=  200, loss(batch)= 0.0549, accuray(batch)= 100.00\n",
      "epoch= 20, i=  300, loss(batch)= 0.0477, accuray(batch)= 100.00\n",
      "epoch= 20, i=  400, loss(batch)= 0.0674, accuray(batch)= 100.00\n",
      "epoch= 20, i=  500, loss(batch)= 0.0522, accuray(batch)= 100.00\n",
      "epoch= 20, loss(train)= 0.058, accuracy(train)= 99.664, time= 100.785, lr= 0.01887\n",
      "  accuracy(test) = 99.190 %, time= 9.055\n"
     ]
    }
   ],
   "source": [
    "# Delete existing network if exists\n",
    "try:\n",
    "    del net\n",
    "    print('Delete existing network\\n')\n",
    "except NameError:\n",
    "    print('No existing network to delete\\n')\n",
    "\n",
    "\n",
    "\n",
    "# network parameters\n",
    "D = train_data.shape[1]\n",
    "CL1_F = 32\n",
    "CL1_K = 25\n",
    "CL2_F = 64\n",
    "CL2_K = 25\n",
    "FC1_F = 512\n",
    "FC2_F = 10\n",
    "net_parameters = [D, CL1_F, CL1_K, CL2_F, CL2_K, FC1_F, FC2_F]\n",
    "\n",
    "\n",
    "# instantiate the object net of the class \n",
    "net = Graph_ConvNet_LeNet5(net_parameters)\n",
    "if torch.cuda.is_available():\n",
    "    net.cuda()\n",
    "print(net)\n",
    "\n",
    "\n",
    "# Weights\n",
    "L_net = list(net.parameters())\n",
    "\n",
    "\n",
    "# learning parameters\n",
    "learning_rate = 0.05\n",
    "dropout_value = 0.5\n",
    "l2_regularization = 5e-4 \n",
    "batch_size = 100\n",
    "num_epochs = 20\n",
    "train_size = train_data.shape[0]\n",
    "nb_iter = int(num_epochs * train_size) // batch_size\n",
    "print('num_epochs=',num_epochs,', train_size=',train_size,', nb_iter=',nb_iter)\n",
    "\n",
    "\n",
    "# Optimizer\n",
    "global_lr = learning_rate\n",
    "global_step = 0\n",
    "decay = 0.95\n",
    "decay_steps = train_size\n",
    "lr = learning_rate\n",
    "optimizer = net.update(lr) \n",
    "\n",
    "\n",
    "# loop over epochs\n",
    "indices = collections.deque()\n",
    "for epoch in range(num_epochs):  # loop over the dataset multiple times\n",
    "\n",
    "    # reshuffle \n",
    "    indices.extend(np.random.permutation(train_size)) # rand permutation\n",
    "    \n",
    "    # reset time\n",
    "    t_start = time.time()\n",
    "    \n",
    "    # extract batches\n",
    "    running_loss = 0.0\n",
    "    running_accuray = 0\n",
    "    running_total = 0\n",
    "    while len(indices) >= batch_size:\n",
    "        \n",
    "        # extract batches\n",
    "        batch_idx = [indices.popleft() for i in range(batch_size)]\n",
    "        train_x, train_y = train_data[batch_idx,:], train_labels[batch_idx]\n",
    "        train_x = Variable( torch.FloatTensor(train_x).type(dtypeFloat) , requires_grad=False) \n",
    "        train_y = train_y.astype(np.int64)\n",
    "        train_y = torch.LongTensor(train_y).type(dtypeLong)\n",
    "        train_y = Variable( train_y , requires_grad=False) \n",
    "            \n",
    "        # Forward \n",
    "        y = net.forward(train_x, dropout_value, L, lmax)\n",
    "        loss = net.loss(y,train_y,l2_regularization) \n",
    "        loss_train = loss.data[0]\n",
    "        \n",
    "        # Accuracy\n",
    "        acc_train = net.evaluation(y,train_y.data)\n",
    "        \n",
    "        # backward\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update \n",
    "        global_step += batch_size # to update learning rate\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # loss, accuracy\n",
    "        running_loss += loss_train\n",
    "        running_accuray += acc_train\n",
    "        running_total += 1\n",
    "        \n",
    "        # print        \n",
    "        if not running_total%100: # print every x mini-batches\n",
    "            print('epoch= %d, i= %4d, loss(batch)= %.4f, accuray(batch)= %.2f' % (epoch+1, running_total, loss_train, acc_train))\n",
    "          \n",
    "       \n",
    "    # print \n",
    "    t_stop = time.time() - t_start\n",
    "    print('epoch= %d, loss(train)= %.3f, accuracy(train)= %.3f, time= %.3f, lr= %.5f' % \n",
    "          (epoch+1, running_loss/running_total, running_accuray/running_total, t_stop, lr))\n",
    " \n",
    "\n",
    "    # update learning rate \n",
    "    lr = global_lr * pow( decay , float(global_step// decay_steps) )\n",
    "    optimizer = net.update_learning_rate(optimizer, lr)\n",
    "    \n",
    "    \n",
    "    # Test set\n",
    "    running_accuray_test = 0\n",
    "    running_total_test = 0\n",
    "    indices_test = collections.deque()\n",
    "    indices_test.extend(range(test_data.shape[0]))\n",
    "    t_start_test = time.time()\n",
    "    while len(indices_test) >= batch_size:\n",
    "        batch_idx_test = [indices_test.popleft() for i in range(batch_size)]\n",
    "        test_x, test_y = test_data[batch_idx_test,:], test_labels[batch_idx_test]\n",
    "        test_x = Variable( torch.FloatTensor(test_x).type(dtypeFloat) , requires_grad=False) \n",
    "        y = net.forward(test_x, 0.0, L, lmax) \n",
    "        test_y = test_y.astype(np.int64)\n",
    "        test_y = torch.LongTensor(test_y).type(dtypeLong)\n",
    "        test_y = Variable( test_y , requires_grad=False) \n",
    "        acc_test = net.evaluation(y,test_y.data)\n",
    "        running_accuray_test += acc_test\n",
    "        running_total_test += 1\n",
    "    t_stop_test = time.time() - t_start_test\n",
    "    print('  accuracy(test) = %.3f %%, time= %.3f' % (running_accuray_test / running_total_test, t_stop_test))  \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}