Pruning_Method_Attention-based_Feature.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attention-based Feature Pruning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Dataset CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if torch.cuda.is_available() else {}\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "            datasets.CIFAR10('./data.cifar10', train=False, download=True, transform=transforms.Compose([\n",
    "                transforms.ToTensor(),\n",
    "                transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])),\n",
    "            batch_size=32, shuffle=True, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1 one minimum iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 3, 32, 32]), torch.Size([32]))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx, data = next(enumerate(test_loader))\n",
    "value, label = data[0].shape, data[1].shape\n",
    "value, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 32, 32])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0][0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2 one data, one batch of iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 32, 32])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_idx, data_idx = 0, 6\n",
    "\n",
    "data1 = data[batch_idx][data_idx].clone().unsqueeze(0)\n",
    "data1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Model VGG19_BN\n",
    "\n",
    "#### 2.1 pretrained ImageNet VGG19 Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torchvision.models as models\n",
    "\n",
    "# model = models.vgg19_bn(pretrained=True)\n",
    "# model.eval()\n",
    "# model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2 define empty VGG19 Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vgg(\n",
       "  (feature): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU(inplace=True)\n",
       "    (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (9): ReLU(inplace=True)\n",
       "    (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (12): ReLU(inplace=True)\n",
       "    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (16): ReLU(inplace=True)\n",
       "    (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (19): ReLU(inplace=True)\n",
       "    (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (22): ReLU(inplace=True)\n",
       "    (23): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (25): ReLU(inplace=True)\n",
       "    (26): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (27): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (29): ReLU(inplace=True)\n",
       "    (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (32): ReLU(inplace=True)\n",
       "    (33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (35): ReLU(inplace=True)\n",
       "    (36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (38): ReLU(inplace=True)\n",
       "    (39): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (42): ReLU(inplace=True)\n",
       "    (43): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (44): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (45): ReLU(inplace=True)\n",
       "    (46): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (47): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (48): ReLU(inplace=True)\n",
       "    (49): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (50): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (51): ReLU(inplace=True)\n",
       "  )\n",
       "  (classifier): Linear(in_features=512, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "\n",
    "__all__ = ['vgg']\n",
    "\n",
    "defaultcfg = {\n",
    "    11: [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512],\n",
    "    13: [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512],\n",
    "    16: [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512],\n",
    "    19: [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512],\n",
    "}\n",
    "\n",
    "\n",
    "class vgg(nn.Module):\n",
    "    def __init__(self, dataset='cifar10', depth=19, init_weights=True, cfg=None, batch_norm=True):\n",
    "        super(vgg, self).__init__()\n",
    "        if cfg is None:\n",
    "            cfg = defaultcfg[depth]\n",
    "\n",
    "        self.feature = self.make_layers(cfg, batch_norm)\n",
    "\n",
    "        if dataset == 'cifar10':\n",
    "            num_classes = 10\n",
    "        elif dataset == 'cifar100':\n",
    "            num_classes = 100\n",
    "        self.classifier = nn.Linear(cfg[-1], num_classes)\n",
    "        if init_weights:\n",
    "            self._initialize_weights()\n",
    "\n",
    "    def make_layers(self, cfg, batch_norm=False):\n",
    "        layers = []\n",
    "        in_channels = 3\n",
    "        for v in cfg:\n",
    "            if v == 'M':\n",
    "                layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n",
    "            else:\n",
    "                conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, bias=False)\n",
    "                if batch_norm:\n",
    "                    layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]\n",
    "                else:\n",
    "                    layers += [conv2d, nn.ReLU(inplace=True)]\n",
    "                in_channels = v\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.feature(x)\n",
    "        x = nn.AvgPool2d(2)(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        y = self.classifier(x)\n",
    "        return y\n",
    "\n",
    "    def _initialize_weights(self):\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))  # mean, std\n",
    "                if m.bias is not None:\n",
    "                    m.bias.data.zero_()\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                m.weight.data.fill_(0.5)\n",
    "                m.bias.data.zero_()\n",
    "            elif isinstance(m, nn.Linear):\n",
    "                m.weight.data.normal_(0, 0.01)\n",
    "                m.bias.data.zero_()\n",
    "\n",
    "model = vgg(dataset='cifar10', depth=19)\n",
    "model.eval()\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.3 initialize model weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[[ 1.1444e-01, -1.1256e-01, -1.7101e-02],\n",
       "           [ 1.0275e-01,  4.0381e-02,  1.8850e-02],\n",
       "           [ 7.1216e-02, -5.0391e-02, -3.7566e-02]],\n",
       " \n",
       "          [[-3.7260e-02,  6.9098e-02, -4.6946e-02],\n",
       "           [ 6.3294e-02, -6.7656e-02, -3.5001e-02],\n",
       "           [ 5.7546e-02, -5.7186e-02,  8.9768e-03]],\n",
       " \n",
       "          [[-4.8316e-02, -7.1742e-02,  7.4178e-02],\n",
       "           [ 3.2971e-02,  2.6361e-02, -2.7924e-02],\n",
       "           [ 2.1197e-02, -8.2348e-02, -8.2991e-02]]],\n",
       " \n",
       " \n",
       "         [[[-4.0529e-02, -1.1625e-02,  3.0071e-02],\n",
       "           [-2.3976e-02,  1.6124e-01, -4.0124e-02],\n",
       "           [ 3.5073e-02,  1.2587e-01, -1.6371e-02]],\n",
       " \n",
       "          [[ 2.9429e-02, -6.0992e-02, -5.7987e-03],\n",
       "           [ 8.0695e-03,  1.0201e-01,  6.1032e-02],\n",
       "           [-1.2790e-01,  3.9012e-02,  1.1109e-03]],\n",
       " \n",
       "          [[-2.4212e-02, -6.3374e-03,  7.4965e-02],\n",
       "           [-6.5757e-02, -2.1136e-02,  1.0336e-02],\n",
       "           [-5.3650e-02,  7.9492e-02,  6.1771e-02]]],\n",
       " \n",
       " \n",
       "         [[[-9.4683e-03,  6.0007e-02,  3.2684e-02],\n",
       "           [ 5.1721e-02,  1.8992e-02,  1.1097e-01],\n",
       "           [ 1.0383e-02, -8.7037e-02,  9.4582e-02]],\n",
       " \n",
       "          [[ 7.7745e-02,  2.0789e-02, -5.6989e-02],\n",
       "           [-8.1887e-03,  1.9440e-02,  8.3429e-02],\n",
       "           [-8.1701e-02, -1.8910e-01, -5.5092e-02]],\n",
       " \n",
       "          [[-1.1640e-02, -1.5547e-03, -3.7031e-02],\n",
       "           [ 8.1964e-02,  4.4215e-02,  7.0960e-03],\n",
       "           [ 2.1378e-02,  7.6074e-02, -8.7929e-03]]],\n",
       " \n",
       " \n",
       "         ...,\n",
       " \n",
       " \n",
       "         [[[-1.9458e-02, -1.7693e-02, -7.1944e-02],\n",
       "           [ 1.1616e-02,  1.5546e-01,  2.7025e-02],\n",
       "           [-5.0056e-02, -1.3098e-02,  3.0034e-02]],\n",
       " \n",
       "          [[-2.6959e-02, -4.5793e-02, -6.3095e-03],\n",
       "           [-4.7469e-02,  4.7532e-02, -2.9739e-02],\n",
       "           [-5.0613e-02,  1.3453e-02, -3.2628e-02]],\n",
       " \n",
       "          [[ 2.9021e-02,  4.9304e-02, -4.0513e-02],\n",
       "           [ 4.2125e-02, -1.6866e-02, -6.9319e-04],\n",
       "           [-1.2481e-02,  5.0570e-02, -1.0737e-02]]],\n",
       " \n",
       " \n",
       "         [[[-6.2285e-03,  4.5438e-02,  7.1942e-02],\n",
       "           [ 5.1030e-02, -8.8825e-02,  3.6739e-03],\n",
       "           [ 6.9477e-02, -9.3080e-02,  4.9839e-02]],\n",
       " \n",
       "          [[ 7.8999e-02,  2.8067e-02,  2.8242e-02],\n",
       "           [-1.5971e-02,  2.3739e-02, -4.7151e-02],\n",
       "           [ 1.0540e-01,  7.4806e-02, -2.1054e-02]],\n",
       " \n",
       "          [[ 7.8172e-03, -6.7527e-02, -9.9103e-02],\n",
       "           [ 3.2311e-02,  4.1776e-02, -1.7824e-01],\n",
       "           [ 6.4510e-02,  1.4584e-02,  5.2022e-02]]],\n",
       " \n",
       " \n",
       "         [[[-5.5574e-02,  2.6219e-02,  1.9360e-02],\n",
       "           [-1.1866e-04, -1.5860e-02,  8.9002e-04],\n",
       "           [-6.6317e-03, -3.1114e-02, -1.3376e-02]],\n",
       " \n",
       "          [[-2.7513e-02,  7.1972e-02,  1.4600e-02],\n",
       "           [ 1.4670e-02, -6.2651e-02, -5.3783e-02],\n",
       "           [-6.1402e-02,  2.7671e-02, -5.5628e-03]],\n",
       " \n",
       "          [[-3.4380e-02, -1.7548e-02,  2.1389e-02],\n",
       "           [ 6.6367e-02, -1.5818e-01, -1.8787e-02],\n",
       "           [-9.6387e-02,  3.6190e-02, -1.5641e-01]]]]),\n",
       " tensor([0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "         0.5000]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model._initialize_weights()\n",
    "model.feature[0].weight.data, model.feature[1].weight.data, "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.4 load weight data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'D:/Project/Pycharm/network-slimming/logs/sparsity_vgg19_cifar10_s_1e-4/model_best.pth.tar'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-dc5c81bdfd30>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[0mmodel_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0murl_path\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msparsity_path\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'/model_best.pth.tar'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mcheckpoint\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[0mbest_prec1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'best_prec1'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[0mepoch1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'epoch'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[0;32m    577\u001b[0m         \u001b[0mpickle_load_args\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'encoding'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    578\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 579\u001b[1;33m     \u001b[1;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    580\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0m_is_zipfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    581\u001b[0m             \u001b[1;31m# The zipfile reader is going to advance the current file position.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[1;34m(name_or_buffer, mode)\u001b[0m\n\u001b[0;32m    228\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 230\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    231\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    232\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;34m'w'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, name, mode)\u001b[0m\n\u001b[0;32m    209\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    210\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 211\u001b[1;33m         \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    212\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    213\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'D:/Project/Pycharm/network-slimming/logs/sparsity_vgg19_cifar10_s_1e-4/model_best.pth.tar'"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "url_path = 'D:/Project/Pycharm/network-slimming/logs/'\n",
    "\n",
    "baseline_path = 'baseline_vgg19_cifar10'\n",
    "sparsity_path = 'sparsity_vgg19_cifar10_s_1e-4'\n",
    "# fine_tune_path = 'attention_fine_tune_feature_vgg19_percent_0.7'\n",
    "\n",
    "model_path = url_path + sparsity_path + '/model_best.pth.tar'\n",
    "\n",
    "checkpoint = torch.load(model_path)\n",
    "best_prec1 = checkpoint['best_prec1']\n",
    "epoch1 = checkpoint['epoch']\n",
    "if checkpoint['state_dict'] is not None:\n",
    "    model.load_state_dict(checkpoint['state_dict'])\n",
    "    \n",
    "epoch1, best_prec1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.feature[0].weight.data, model.feature[1].weight.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.5 the first conv2d converting of model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "value1 = model.feature[0](data1)\n",
    "value1, value1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.6 find BatchNorm2d & nn.ReLU converting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "relu = nn.ReLU()\n",
    "data_item = data[0].clone()\n",
    "for idx, m in enumerate(model.feature):\n",
    "    data_item = m(data_item)\n",
    "    if isinstance(m, nn.BatchNorm2d):\n",
    "        relu_data = relu(data_item)\n",
    "        print(idx, relu_data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Activation-based Gramma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "def activation_based_gamma(weight_data):\n",
    "    d1, d2 = weight_data.shape[0], weight_data.shape[1]\n",
    "    \n",
    "    # 1. A: feature map data\n",
    "    A = weight_data.view(d1, d2, -1).abs()\n",
    "    c, h, w = A.shape\n",
    "    \n",
    "    # 2. Fsum(A): sum of values along the channel direction\n",
    "    FsumA = torch.zeros(h, w)\n",
    "    for i in range(c):\n",
    "        FsumA.add_(A[i])\n",
    "        \n",
    "    # 3. ||Fsum(A)||2: two norm\n",
    "    FsumA_norm = torch.linalg.norm(FsumA)\n",
    "    \n",
    "    # 4. F(A) / ||F(A)||2: normalize weight data\n",
    "    F_all = FsumA / FsumA_norm\n",
    "    \n",
    "    # 5. F(Aj) / ||F(Aj)||^2 & gamma = ∑ | F(A) / ||F(A)||2 - F(Aj) / ||F(Aj)||2 |\n",
    "    gamma = torch.zeros(c)\n",
    "    for j in range(c):\n",
    "        FAj = FsumA - A[j]\n",
    "        FAj_norm = torch.linalg.norm(FAj)\n",
    "        Fj = FAj / FAj_norm\n",
    "#         gamma[j] = (F_all - Fj).abs().sum()\n",
    "        gamma[j] = torch.linalg.norm(F_all - Fj, ord=1)  # ord=1, 1 norm; ord=2, 2 norm\n",
    "\n",
    "    return gamma"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1 batch size activation-based gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([0.7776, 0.7880, 0.7887, 0.7836, 0.7847, 0.7891, 0.7933, 0.7884, 0.7913,\n",
       "         0.7850, 0.7819, 0.7854, 0.7833, 0.7919, 0.7918, 0.7847, 0.7936, 0.7922,\n",
       "         0.7825, 0.7785, 0.8020, 0.7883, 0.7828, 0.8008, 0.7868, 0.7877, 0.7890,\n",
       "         0.7871, 0.7846, 0.7816, 0.7879, 0.7784, 0.7841, 0.7943, 0.7811, 0.7804,\n",
       "         0.7904, 0.7834, 0.7956, 0.7758, 0.7870, 0.7908, 0.8050, 0.7822, 0.7815,\n",
       "         0.7771, 0.7773, 0.7838, 0.7838, 0.7897, 0.7899, 0.7870, 0.7913, 0.7804,\n",
       "         0.7902, 0.7861, 0.7758, 0.7879, 0.7847, 0.7839, 0.7829, 0.7883, 0.7826,\n",
       "         0.7874]),\n",
       " tensor([0.0121, 0.0123, 0.0123, 0.0122, 0.0123, 0.0123, 0.0124, 0.0123, 0.0124,\n",
       "         0.0123, 0.0122, 0.0123, 0.0122, 0.0124, 0.0124, 0.0123, 0.0124, 0.0124,\n",
       "         0.0122, 0.0122, 0.0125, 0.0123, 0.0122, 0.0125, 0.0123, 0.0123, 0.0123,\n",
       "         0.0123, 0.0123, 0.0122, 0.0123, 0.0122, 0.0123, 0.0124, 0.0122, 0.0122,\n",
       "         0.0123, 0.0122, 0.0124, 0.0121, 0.0123, 0.0124, 0.0126, 0.0122, 0.0122,\n",
       "         0.0121, 0.0121, 0.0122, 0.0122, 0.0123, 0.0123, 0.0123, 0.0124, 0.0122,\n",
       "         0.0123, 0.0123, 0.0121, 0.0123, 0.0123, 0.0122, 0.0122, 0.0123, 0.0122,\n",
       "         0.0123]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(64, 64, 32, 32)\n",
    "\n",
    "B, C, H, W = x.shape\n",
    "\n",
    "gamma = torch.zeros(B)\n",
    "for i in range(B):\n",
    "    data = x[i].clone().squeeze(0)\n",
    "    gamma += activation_based_gamma(data)\n",
    "gamma_mean = gamma / B\n",
    "gamma, gamma_mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Prune\n",
    "#### 4.1 number of channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5504"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_total = 0\n",
    "for m in model.modules():\n",
    "    if isinstance(m, nn.BatchNorm2d):\n",
    "        num_total += m.weight.data.shape[0]\n",
    "num_total"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.2 all channels' gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "gamma_list = torch.zeros(num_total)\n",
    "gamma_record = []\n",
    "\n",
    "index = 0\n",
    "one_batch = data1.clone()\n",
    "for k, m in enumerate(model.feature):\n",
    "    with torch.no_grad():\n",
    "        one_batch = m(one_batch)\n",
    "    if isinstance(m, nn.BatchNorm2d):\n",
    "        value = one_batch.clone().squeeze(0)\n",
    "        gamma = activation_based_gamma(value)\n",
    "        gamma_record.append(gamma)\n",
    "        size = value.shape[0]\n",
    "        gamma_list[index:(index+size)] = gamma.clone()\n",
    "        index += size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gamma_list[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gamma_record"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.3 threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3852, tensor(0.0019))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pruning_rate = 0.7\n",
    "y, i = torch.sort(gamma_list)\n",
    "thre_idx = int(num_total * pruning_rate)\n",
    "thre = y[thre_idx]\n",
    "\n",
    "thre_idx, thre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.4 prune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer index: 1 \t total channel: 64 \t remaining channel: 64\n",
      "layer index: 4 \t total channel: 64 \t remaining channel: 64\n",
      "layer index: 8 \t total channel: 128 \t remaining channel: 128\n",
      "layer index: 11 \t total channel: 128 \t remaining channel: 128\n",
      "layer index: 15 \t total channel: 256 \t remaining channel: 156\n",
      "layer index: 18 \t total channel: 256 \t remaining channel: 174\n",
      "layer index: 21 \t total channel: 256 \t remaining channel: 184\n",
      "layer index: 24 \t total channel: 256 \t remaining channel: 197\n",
      "layer index: 28 \t total channel: 512 \t remaining channel: 40\n",
      "layer index: 31 \t total channel: 512 \t remaining channel: 72\n",
      "layer index: 34 \t total channel: 512 \t remaining channel: 62\n",
      "layer index: 37 \t total channel: 512 \t remaining channel: 61\n",
      "layer index: 41 \t total channel: 512 \t remaining channel: 68\n",
      "layer index: 44 \t total channel: 512 \t remaining channel: 73\n",
      "layer index: 47 \t total channel: 512 \t remaining channel: 65\n",
      "layer index: 50 \t total channel: 512 \t remaining channel: 72\n",
      "Pre-processing Successful! Pruned ratio:  tensor(0.7078)\n"
     ]
    }
   ],
   "source": [
    "num_pruned = 0\n",
    "num_cfg = []\n",
    "mask_cfg = []\n",
    "\n",
    "one_batch = data1.clone()\n",
    "for k, m in enumerate(model.feature):\n",
    "    one_batch = m(one_batch)\n",
    "    if isinstance(m, nn.BatchNorm2d):\n",
    "        value = one_batch.clone().squeeze(0)\n",
    "        gamma = activation_based_gamma(value)\n",
    "        mask = gamma.gt(thre).float()\n",
    "        m.weight.data.mul_(mask)\n",
    "        m.bias.data.mul_(mask)\n",
    "        num_cfg.append(int(torch.sum(mask)))\n",
    "        mask_cfg.append(mask.clone())\n",
    "        num_pruned += mask.shape[0] - torch.sum(mask)\n",
    "        print('layer index: {:d} \\t total channel: {:d} \\t remaining channel: {:d}'.\n",
    "                  format(k, mask.shape[0], int(torch.sum(mask))))\n",
    "\n",
    "    elif isinstance(m, nn.MaxPool2d):\n",
    "        mask_cfg.append('M')\n",
    "    elif isinstance(m, nn.Linear):\n",
    "        print(k, 'linear over')\n",
    "        break\n",
    "    \n",
    "pruned_ratio = num_pruned / num_total\n",
    "print('Pre-processing Successful! Pruned ratio: ', pruned_ratio)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:python36] *",
   "language": "python",
   "name": "conda-env-python36-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}