Model_VGG19_modules_params.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model VGG19: FLOPs & Parameters\n",
    "\n",
    "\n",
    "#### 0. 1 cuda is available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "USE_CUDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 0.2 pytorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.8.1+cu102'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "torch.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Pretrained VGG19 Model with Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VGG(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU(inplace=True)\n",
       "    (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (9): ReLU(inplace=True)\n",
       "    (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (12): ReLU(inplace=True)\n",
       "    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (16): ReLU(inplace=True)\n",
       "    (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (19): ReLU(inplace=True)\n",
       "    (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (22): ReLU(inplace=True)\n",
       "    (23): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (25): ReLU(inplace=True)\n",
       "    (26): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (27): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (29): ReLU(inplace=True)\n",
       "    (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (32): ReLU(inplace=True)\n",
       "    (33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (35): ReLU(inplace=True)\n",
       "    (36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (38): ReLU(inplace=True)\n",
       "    (39): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (42): ReLU(inplace=True)\n",
       "    (43): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (44): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (45): ReLU(inplace=True)\n",
       "    (46): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (47): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (48): ReLU(inplace=True)\n",
       "    (49): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (50): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (51): ReLU(inplace=True)\n",
       "    (52): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=25088, out_features=4096, bias=True)\n",
       "    (1): ReLU(inplace=True)\n",
       "    (2): Dropout(p=0.5, inplace=False)\n",
       "    (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "    (4): ReLU(inplace=True)\n",
       "    (5): Dropout(p=0.5, inplace=False)\n",
       "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torchvision.models as models\n",
    "\n",
    "model = models.vgg19_bn(pretrained=True)\n",
    "# model = torch.hub.load('pytorch/vision:v0.9.0', 'vgg19_bn', pretrained=True)\n",
    "\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1 VGG19 Weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[1].weight.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.5763e-07,  9.8841e-08,  2.4442e-01,  2.2678e-01,  2.4756e-01,\n",
       "         5.9402e-07,  1.5344e-01,  5.5348e-01,  1.8775e-01,  4.0541e-08,\n",
       "         7.3613e-08,  6.7899e-07,  1.9412e-08,  3.7765e-07, -1.5930e-07,\n",
       "         4.6782e-01, -5.3255e-07,  1.8139e-02,  1.9039e-08,  2.0468e-07,\n",
       "        -3.8896e-05,  8.2928e-07,  3.6763e-01, -5.7934e-08,  1.0634e-07,\n",
       "         7.7692e-09, -1.2172e-07,  1.0209e-07,  1.0130e-07,  3.9670e-08,\n",
       "         4.9515e-08,  6.8137e-07,  1.0767e-07,  6.3167e-08,  5.1813e-01,\n",
       "         3.9662e-01,  4.0801e-06,  3.2676e-01,  3.0591e-08,  1.4233e-05,\n",
       "        -4.0759e-07,  2.1676e-04,  1.8606e-06,  2.0022e-08,  6.8042e-07,\n",
       "         1.0519e-06,  3.0301e-07,  1.7505e-08,  1.4339e-07,  2.6455e-08,\n",
       "         3.7188e-01, -1.0622e-06,  1.9643e-01,  6.8300e-01,  1.8162e-01,\n",
       "        -1.1994e-07,  3.4379e-01,  1.2878e-04,  6.3663e-01,  6.3070e-01,\n",
       "         2.0438e-07,  1.0555e-08,  1.4975e-08,  3.4700e-06])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[1].weight.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2 VGG19 Convolution Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 3, 3, 3])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[0].weight.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[-3.2130e-08, -3.1717e-08, -6.7120e-08],\n",
       "          [-2.9321e-08, -2.1596e-08, -5.3207e-08],\n",
       "          [-5.3304e-08, -4.0832e-08, -6.6434e-08]],\n",
       "\n",
       "         [[ 1.3710e-07,  1.2848e-07,  7.1270e-08],\n",
       "          [ 1.5487e-07,  1.5186e-07,  8.3582e-08],\n",
       "          [ 9.3392e-08,  9.9451e-08,  4.1284e-08]],\n",
       "\n",
       "         [[ 1.7435e-07,  1.7840e-07,  1.0561e-07],\n",
       "          [ 1.9644e-07,  1.9769e-07,  1.2560e-07],\n",
       "          [ 1.1935e-07,  1.3447e-07,  5.9638e-08]]],\n",
       "\n",
       "\n",
       "        [[[-6.3467e-08, -5.2714e-08, -3.8511e-08],\n",
       "          [-1.4349e-08, -9.0426e-09, -1.7164e-08],\n",
       "          [ 2.2282e-08,  1.9084e-08,  1.8150e-09]],\n",
       "\n",
       "         [[-1.2191e-08, -6.1933e-09,  8.5022e-09],\n",
       "          [ 3.8618e-08,  5.8869e-08,  2.2082e-08],\n",
       "          [ 6.2501e-08,  7.0528e-08,  2.7185e-08]],\n",
       "\n",
       "         [[ 3.0045e-08,  3.8003e-08,  3.5389e-08],\n",
       "          [ 8.6743e-08,  1.1626e-07,  8.0502e-08],\n",
       "          [ 7.9653e-08,  1.0057e-07,  6.7894e-08]]],\n",
       "\n",
       "\n",
       "        [[[-1.3565e-01, -2.9960e-01, -1.5151e-01],\n",
       "          [-2.6804e-01, -3.0505e-01, -9.4600e-02],\n",
       "          [-1.4124e-01, -8.1448e-02,  1.7112e-01]],\n",
       "\n",
       "         [[ 2.0333e-01,  4.0123e-01,  1.3747e-01],\n",
       "          [ 4.2157e-01,  6.7595e-01,  3.2725e-01],\n",
       "          [ 1.3103e-01,  3.2654e-01,  6.4284e-02]],\n",
       "\n",
       "         [[-4.1092e-02,  4.6723e-02, -9.7236e-03],\n",
       "          [ 2.9558e-02,  3.5226e-02, -1.2338e-01],\n",
       "          [-9.1504e-02, -1.7836e-01, -3.1635e-01]]],\n",
       "\n",
       "\n",
       "        ...,\n",
       "\n",
       "\n",
       "        [[[-5.0743e-08, -6.6759e-08, -7.8941e-08],\n",
       "          [-4.6060e-08, -7.9228e-08, -7.3020e-08],\n",
       "          [-7.2354e-08, -7.5888e-08, -7.5602e-08]],\n",
       "\n",
       "         [[-1.8643e-08, -5.4891e-08, -5.5029e-08],\n",
       "          [-2.9716e-08, -6.0283e-08, -5.9039e-08],\n",
       "          [-5.4538e-08, -5.7847e-08, -5.4300e-08]],\n",
       "\n",
       "         [[-9.0202e-09, -4.0853e-08, -4.0702e-08],\n",
       "          [-2.0145e-08, -4.0434e-08, -4.7582e-08],\n",
       "          [-4.9317e-08, -4.9057e-08, -4.8955e-08]]],\n",
       "\n",
       "\n",
       "        [[[-7.8860e-09,  4.4172e-09,  4.1371e-08],\n",
       "          [-9.1843e-09,  4.9512e-09,  6.3214e-08],\n",
       "          [ 1.2759e-09,  2.6705e-08,  6.1890e-08]],\n",
       "\n",
       "         [[-1.6242e-09, -5.1753e-10,  3.5820e-08],\n",
       "          [-1.5325e-08,  9.5484e-09,  6.6321e-08],\n",
       "          [ 5.8824e-09,  3.6904e-08,  7.9135e-08]],\n",
       "\n",
       "         [[ 7.1371e-09,  3.5890e-09,  2.7016e-08],\n",
       "          [-2.0624e-09,  1.1004e-08,  5.9306e-08],\n",
       "          [ 2.0382e-08,  3.8491e-08,  6.5097e-08]]],\n",
       "\n",
       "\n",
       "        [[[-5.1288e-07,  4.4387e-06,  2.8007e-06],\n",
       "          [ 1.4936e-06,  6.4549e-06,  4.6987e-06],\n",
       "          [ 1.3737e-06,  3.8875e-06,  3.2473e-06]],\n",
       "\n",
       "         [[-2.3703e-06,  3.5919e-06,  1.5149e-06],\n",
       "          [ 6.3111e-07,  6.6647e-06,  4.2479e-06],\n",
       "          [-2.4147e-07,  2.5924e-06,  1.3272e-06]],\n",
       "\n",
       "         [[ 4.0485e-06,  1.1047e-05,  7.5509e-06],\n",
       "          [ 7.4615e-06,  1.4459e-05,  1.0637e-05],\n",
       "          [ 4.2098e-06,  7.7080e-06,  5.1014e-06]]]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[0].weight.data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Pretrained VGG19 Model without Batch Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/vgg19-dcbb9e9d.pth\" to C:\\Users\\UPC/.cache\\torch\\hub\\checkpoints\\vgg19-dcbb9e9d.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c4d9604615f4f9b83226d41f984311f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/548M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-472aa8126235>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtorchvision\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvgg19\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpretrained\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;31m# model = torch.hub.load('pytorch/vision:v0.9.0', 'vgg19', pretrained=True)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torchvision\\models\\vgg.py\u001b[0m in \u001b[0;36mvgg19\u001b[1;34m(pretrained, progress, **kwargs)\u001b[0m\n\u001b[0;32m    177\u001b[0m         \u001b[0mprogress\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mIf\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdisplays\u001b[0m \u001b[0ma\u001b[0m \u001b[0mprogress\u001b[0m \u001b[0mbar\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mdownload\u001b[0m \u001b[0mto\u001b[0m \u001b[0mstderr\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    178\u001b[0m     \"\"\"\n\u001b[1;32m--> 179\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_vgg\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'vgg19'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'E'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpretrained\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprogress\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    180\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    181\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torchvision\\models\\vgg.py\u001b[0m in \u001b[0;36m_vgg\u001b[1;34m(arch, cfg, batch_norm, pretrained, progress, **kwargs)\u001b[0m\n\u001b[0;32m     98\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mpretrained\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     99\u001b[0m         state_dict = load_state_dict_from_url(model_urls[arch],\n\u001b[1;32m--> 100\u001b[1;33m                                               progress=progress)\n\u001b[0m\u001b[0;32m    101\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    102\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torch\\hub.py\u001b[0m in \u001b[0;36mload_state_dict_from_url\u001b[1;34m(url, model_dir, map_location, progress, check_hash, file_name)\u001b[0m\n\u001b[0;32m    522\u001b[0m             \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mHASH_REGEX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# r is Optional[Match[str]]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    523\u001b[0m             \u001b[0mhash_prefix\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mr\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 524\u001b[1;33m         \u001b[0mdownload_url_to_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcached_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhash_prefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprogress\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mprogress\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    525\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    526\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0m_is_legacy_zip_format\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcached_file\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\site-packages\\torch\\hub.py\u001b[0m in \u001b[0;36mdownload_url_to_file\u001b[1;34m(url, dst, hash_prefix, progress)\u001b[0m\n\u001b[0;32m    414\u001b[0m                   unit='B', unit_scale=True, unit_divisor=1024) as pbar:\n\u001b[0;32m    415\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 416\u001b[1;33m                 \u001b[0mbuffer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mu\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m8192\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    417\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    418\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\http\\client.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, amt)\u001b[0m\n\u001b[0;32m    461\u001b[0m             \u001b[1;31m# Amount is given, implement using readinto\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    462\u001b[0m             \u001b[0mb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbytearray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 463\u001b[1;33m             \u001b[0mn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreadinto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    464\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtobytes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    465\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\http\\client.py\u001b[0m in \u001b[0;36mreadinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m    505\u001b[0m         \u001b[1;31m# connection, and the user is reading more bytes than will be provided\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    506\u001b[0m         \u001b[1;31m# (for example, reading in 1k chunks)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 507\u001b[1;33m         \u001b[0mn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreadinto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    508\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mn\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    509\u001b[0m             \u001b[1;31m# Ideally, we would raise IncompleteRead if the content-length\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m    584\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    585\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 586\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    587\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    588\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[1;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[0;32m   1010\u001b[0m                   \u001b[1;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1011\u001b[0m                   self.__class__)\n\u001b[1;32m-> 1012\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1013\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1014\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0msocket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\ssl.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, len, buffer)\u001b[0m\n\u001b[0;32m    872\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Read on closed or unwrapped SSL socket.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    873\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 874\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    875\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mSSLError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    876\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mSSL_ERROR_EOF\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msuppress_ragged_eofs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\Anaconda3\\envs\\python36\\lib\\ssl.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, len, buffer)\u001b[0m\n\u001b[0;32m    629\u001b[0m         \"\"\"\n\u001b[0;32m    630\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 631\u001b[1;33m             \u001b[0mv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    632\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    633\u001b[0m             \u001b[0mv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision.models as models\n",
    "\n",
    "model = models.vgg19(pretrained=True)\n",
    "# model = torch.hub.load('pytorch/vision:v0.9.0', 'vgg19', pretrained=True)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 features[0]: Conv2d\n",
    "\n",
    "- 输入通道为RGB彩色图像三通道，输出通道为64，卷积核大小为3×3\n",
    "- Conv2d weight：3个卷积核为一组，一共有64组卷积核，即 `model.features[0].weight.shape = [64, 3, 3, 3]`\n",
    "- Conv2d bias：3个卷积核为一组，卷积核的结果相加，产生一个通道的结果。一共有64组，使用偏置对其调整，即 `model.features[0].bias.shape=[64]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 3, 3, 3])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[0].weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[0].bias.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 features[2]: Conv2d \n",
    "\n",
    "- 输入通道为64，输出通道为64，卷积核大小为3×3\n",
    "- Conv2d weight：64个3×3的卷积核为一组，一共有64组，即 `model.features[2].weight.shape=[64, 64, 3, 3]`\n",
    "- Conv2d bias: 64个3×3的卷积核为一组，一共有64组，每一组内对应位置求和，即`model.features[2].bias.shape=[64]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 64, 3, 3])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[2].weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.features[2].bias.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Prune Conv2d Bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-5.8394e-02, -1.4754e-01,  1.8017e-01, -2.8633e-01,  1.4053e-02,\n",
       "         -3.9571e-02,  1.0259e-01, -8.8048e-04, -7.7598e-02, -1.8744e-01,\n",
       "         -3.4763e-02,  2.2564e-02, -6.1512e-02, -3.9896e-01, -2.3920e-01,\n",
       "          1.1534e-01, -1.1614e-01, -2.2889e-01, -1.6896e-01, -2.4117e-01,\n",
       "          5.9063e-01,  4.4781e-04, -3.8955e-02, -8.2188e-01, -1.5366e-01,\n",
       "          2.2464e-02,  4.2782e-01, -5.8604e-02, -4.8700e-02,  3.0054e-01,\n",
       "         -5.6536e-02,  5.2338e-02,  2.4208e-01, -3.2136e-03,  4.8011e-01,\n",
       "         -8.3752e-03,  1.7856e-01,  3.9528e-01, -2.2767e-02,  5.7821e-02,\n",
       "          6.5798e-04, -1.3895e-01, -1.6600e-01,  7.9103e-03,  1.1497e-01,\n",
       "          1.0045e-01,  2.3931e-01,  4.0163e-01, -1.3901e-01, -4.5015e-01,\n",
       "          1.9804e-01, -1.4634e-01, -1.1509e-01,  7.2199e-02, -1.0609e-02,\n",
       "          3.5647e-01,  9.0835e-02, -1.5250e-01,  2.1137e-01, -2.0729e-01,\n",
       "          1.1017e-02,  2.3209e-01,  9.1408e-02,  2.5651e-02],\n",
       "        grad_fn=<CloneBackward>),\n",
       " tensor([0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 0.,\n",
       "         0., 0., 1., 1., 0., 0., 0., 1., 1., 0., 0., 1., 0., 1., 1., 0., 1., 0.,\n",
       "         1., 1., 0., 1., 1., 0., 0., 1., 1., 1., 1., 1., 0., 0., 1., 0., 0., 1.,\n",
       "         0., 1., 1., 0., 1., 0., 1., 1., 1., 1.]),\n",
       " tensor([-0.0000e+00, -0.0000e+00, 1.8017e-01, -0.0000e+00, 1.4053e-02, -0.0000e+00,\n",
       "         1.0259e-01, -0.0000e+00, -0.0000e+00, -0.0000e+00, -0.0000e+00, 2.2564e-02,\n",
       "         -0.0000e+00, -0.0000e+00, -0.0000e+00, 1.1534e-01, -0.0000e+00, -0.0000e+00,\n",
       "         -0.0000e+00, -0.0000e+00, 5.9063e-01, 4.4781e-04, -0.0000e+00, -0.0000e+00,\n",
       "         -0.0000e+00, 2.2464e-02, 4.2782e-01, -0.0000e+00, -0.0000e+00, 3.0054e-01,\n",
       "         -0.0000e+00, 5.2338e-02, 2.4208e-01, -0.0000e+00, 4.8011e-01, -0.0000e+00,\n",
       "         1.7856e-01, 3.9528e-01, -0.0000e+00, 5.7821e-02, 6.5798e-04, -0.0000e+00,\n",
       "         -0.0000e+00, 7.9103e-03, 1.1497e-01, 1.0045e-01, 2.3931e-01, 4.0163e-01,\n",
       "         -0.0000e+00, -0.0000e+00, 1.9804e-01, -0.0000e+00, -0.0000e+00, 7.2199e-02,\n",
       "         -0.0000e+00, 3.5647e-01, 9.0835e-02, -0.0000e+00, 2.1137e-01, -0.0000e+00,\n",
       "         1.1017e-02, 2.3209e-01, 9.1408e-02, 2.5651e-02],\n",
       "        grad_fn=<MulBackward0>))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bias = model.features[2].bias.clone()\n",
    "mask = bias.gt(0.).float()\n",
    "bias_mask = bias.mul(mask)\n",
    "bias, mask, bias_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 Prune Conv2d Weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 64, 3, 3])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([1., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1., 0.,\n",
       "        0., 0., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1., 1., 0., 1., 1., 1., 0.,\n",
       "        0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 0., 0., 1., 0., 0., 1., 1., 0.,\n",
       "        1., 0., 0., 0., 0., 0., 1., 0., 0., 1.])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weight = model.features[2].weight.clone()\n",
    "num_channel = weight.shape[0]\n",
    "mask = torch.randn(num_channel).gt(0.).float()\n",
    "for i in range(num_channel):\n",
    "    if mask[i] <= 0:\n",
    "        weight[i] = torch.zeros()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Handle Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vgg(\n",
       "  (feature): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU(inplace=True)\n",
       "    (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (9): ReLU(inplace=True)\n",
       "    (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (12): ReLU(inplace=True)\n",
       "    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (16): ReLU(inplace=True)\n",
       "    (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (19): ReLU(inplace=True)\n",
       "    (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (22): ReLU(inplace=True)\n",
       "    (23): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (25): ReLU(inplace=True)\n",
       "    (26): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (27): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (29): ReLU(inplace=True)\n",
       "    (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (32): ReLU(inplace=True)\n",
       "    (33): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (34): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (35): ReLU(inplace=True)\n",
       "    (36): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (37): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (38): ReLU(inplace=True)\n",
       "    (39): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (42): ReLU(inplace=True)\n",
       "    (43): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (44): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (45): ReLU(inplace=True)\n",
       "    (46): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (47): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (48): ReLU(inplace=True)\n",
       "    (49): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (50): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (51): ReLU(inplace=True)\n",
       "  )\n",
       "  (classifier): Linear(in_features=512, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "\n",
    "__all__ = ['vgg']\n",
    "\n",
    "defaultcfg = {\n",
    "    11: [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512],\n",
    "    13: [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512],\n",
    "    16: [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512],\n",
    "    19: [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512],\n",
    "}\n",
    "\n",
    "\n",
    "class vgg(nn.Module):\n",
    "    def __init__(self, dataset='cifar10', depth=19, init_weights=True, cfg=None, batch_norm=True, conv_cfg=None):\n",
    "        super(vgg, self).__init__()\n",
    "\n",
    "        self.conv_cfg = conv_cfg\n",
    "        if cfg is None:\n",
    "            cfg = defaultcfg[depth]\n",
    "\n",
    "        self.feature = self.make_layers(cfg, batch_norm)\n",
    "\n",
    "        num_classes = 10\n",
    "        if dataset == 'cifar100':\n",
    "            num_classes = 100\n",
    "        self.classifier = nn.Linear(cfg[-1], num_classes)\n",
    "\n",
    "        if init_weights:\n",
    "            self._initialize_weights()\n",
    "\n",
    "    def make_layers(self, cfg, batch_norm=False):\n",
    "        layers = []\n",
    "        in_channels = 3\n",
    "        for v in cfg:\n",
    "            if v == 'M':\n",
    "                layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n",
    "            else:\n",
    "                conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, bias=False)\n",
    "                if batch_norm:\n",
    "                    layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]\n",
    "                else:\n",
    "                    layers += [conv2d, nn.ReLU(inplace=True)]\n",
    "                in_channels = v\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        conv_value = []\n",
    "        if self.conv_cfg:\n",
    "            conv_idx = 0\n",
    "            for k, m in enumerate(self.feature):\n",
    "                x = m(x)\n",
    "                if isinstance(m, nn.Conv2d):\n",
    "                    conv_idx += 1\n",
    "                    if conv_idx in self.conv_cfg:\n",
    "                        conv_value.append(x)\n",
    "        else:\n",
    "            x = self.feature(x)\n",
    "\n",
    "        x = nn.AvgPool2d(2)(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        y = self.classifier(x)\n",
    "\n",
    "        if len(conv_value):\n",
    "            return y, conv_value\n",
    "        else:\n",
    "            return y\n",
    "\n",
    "    def _initialize_weights(self):\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "                if m.bias is not None:\n",
    "                    m.bias.data.zero_()\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                m.weight.data.fill_(0.5)\n",
    "                m.bias.data.zero_()\n",
    "            elif isinstance(m, nn.Linear):\n",
    "                m.weight.data.normal_(0, 0.01)\n",
    "                m.bias.data.zero_()\n",
    "\n",
    "\n",
    "model = vgg(dataset='cifar10', depth=19)\n",
    "model.eval()\n",
    "# model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.  Pytorch统计FLOPs和Parameters\n",
    "\n",
    "#### 4.1 thop\n",
    "    `input` 与 `model` 需要位于同一device，比如 model.cuda() 和 input.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_bn() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "\u001b[91m[WARN] Cannot find rule for <class 'torch.nn.modules.container.Sequential'>. Treat it as zero Macs and zero Params.\u001b[00m\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "\u001b[91m[WARN] Cannot find rule for <class '__main__.vgg'>. Treat it as zero Macs and zero Params.\u001b[00m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "G:\\Anaconda3\\envs\\python37\\lib\\site-packages\\torch\\nn\\functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  ..\\c10/core/TensorImpl.h:1156.)\n",
      "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(398742528.0, 20035018.0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from thop import profile\n",
    "from thop import clever_format\n",
    "\n",
    "input = torch.randn(1, 3, 32, 32)\n",
    "flops, params = profile(model, inputs=(input, ))\n",
    "flops, params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('398.74M', '20.04M')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flops, params = clever_format([flops, params], \"%.2f\")\n",
    "flops, params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.2 torchstat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      module name  input shape output shape      params memory(MB)           MAdd          Flops  MemRead(B)  MemWrite(B) duration[%]   MemR+W(B)\n",
      "0       feature.0    3  32  32   64  32  32      1728.0       0.25    3,473,408.0    1,769,472.0     19200.0     262144.0       0.00%    281344.0\n",
      "1       feature.1   64  32  32   64  32  32       128.0       0.25      262,144.0      131,072.0    262656.0     262144.0       0.00%    524800.0\n",
      "2       feature.2   64  32  32   64  32  32         0.0       0.25       65,536.0       65,536.0    262144.0     262144.0       0.00%    524288.0\n",
      "3       feature.3   64  32  32   64  32  32     36864.0       0.25   75,431,936.0   37,748,736.0    409600.0     262144.0      33.33%    671744.0\n",
      "4       feature.4   64  32  32   64  32  32       128.0       0.25      262,144.0      131,072.0    262656.0     262144.0       0.00%    524800.0\n",
      "5       feature.5   64  32  32   64  32  32         0.0       0.25       65,536.0       65,536.0    262144.0     262144.0       0.00%    524288.0\n",
      "6       feature.6   64  32  32   64  16  16         0.0       0.06       49,152.0       65,536.0    262144.0      65536.0       0.00%    327680.0\n",
      "7       feature.7   64  16  16  128  16  16     73728.0       0.12   37,715,968.0   18,874,368.0    360448.0     131072.0       0.00%    491520.0\n",
      "8       feature.8  128  16  16  128  16  16       256.0       0.12      131,072.0       65,536.0    132096.0     131072.0       0.00%    263168.0\n",
      "9       feature.9  128  16  16  128  16  16         0.0       0.12       32,768.0       32,768.0    131072.0     131072.0       0.00%    262144.0\n",
      "10     feature.10  128  16  16  128  16  16    147456.0       0.12   75,464,704.0   37,748,736.0    720896.0     131072.0       0.00%    851968.0\n",
      "11     feature.11  128  16  16  128  16  16       256.0       0.12      131,072.0       65,536.0    132096.0     131072.0       0.00%    263168.0\n",
      "12     feature.12  128  16  16  128  16  16         0.0       0.12       32,768.0       32,768.0    131072.0     131072.0       0.00%    262144.0\n",
      "13     feature.13  128  16  16  128   8   8         0.0       0.03       24,576.0       32,768.0    131072.0      32768.0       0.00%    163840.0\n",
      "14     feature.14  128   8   8  256   8   8    294912.0       0.06   37,732,352.0   18,874,368.0   1212416.0      65536.0       0.00%   1277952.0\n",
      "15     feature.15  256   8   8  256   8   8       512.0       0.06       65,536.0       32,768.0     67584.0      65536.0       0.00%    133120.0\n",
      "16     feature.16  256   8   8  256   8   8         0.0       0.06       16,384.0       16,384.0     65536.0      65536.0       0.00%    131072.0\n",
      "17     feature.17  256   8   8  256   8   8    589824.0       0.06   75,481,088.0   37,748,736.0   2424832.0      65536.0       0.00%   2490368.0\n",
      "18     feature.18  256   8   8  256   8   8       512.0       0.06       65,536.0       32,768.0     67584.0      65536.0       0.00%    133120.0\n",
      "19     feature.19  256   8   8  256   8   8         0.0       0.06       16,384.0       16,384.0     65536.0      65536.0       0.00%    131072.0\n",
      "20     feature.20  256   8   8  256   8   8    589824.0       0.06   75,481,088.0   37,748,736.0   2424832.0      65536.0       0.00%   2490368.0\n",
      "21     feature.21  256   8   8  256   8   8       512.0       0.06       65,536.0       32,768.0     67584.0      65536.0       0.00%    133120.0\n",
      "22     feature.22  256   8   8  256   8   8         0.0       0.06       16,384.0       16,384.0     65536.0      65536.0       0.00%    131072.0\n",
      "23     feature.23  256   8   8  256   8   8    589824.0       0.06   75,481,088.0   37,748,736.0   2424832.0      65536.0       0.00%   2490368.0\n",
      "24     feature.24  256   8   8  256   8   8       512.0       0.06       65,536.0       32,768.0     67584.0      65536.0       0.00%    133120.0\n",
      "25     feature.25  256   8   8  256   8   8         0.0       0.06       16,384.0       16,384.0     65536.0      65536.0       0.00%    131072.0\n",
      "26     feature.26  256   8   8  256   4   4         0.0       0.02       12,288.0       16,384.0     65536.0      16384.0       0.00%     81920.0\n",
      "27     feature.27  256   4   4  512   4   4   1179648.0       0.03   37,740,544.0   18,874,368.0   4734976.0      32768.0       0.00%   4767744.0\n",
      "28     feature.28  512   4   4  512   4   4      1024.0       0.03       32,768.0       16,384.0     36864.0      32768.0       0.00%     69632.0\n",
      "29     feature.29  512   4   4  512   4   4         0.0       0.03        8,192.0        8,192.0     32768.0      32768.0       0.00%     65536.0\n",
      "30     feature.30  512   4   4  512   4   4   2359296.0       0.03   75,489,280.0   37,748,736.0   9469952.0      32768.0      33.34%   9502720.0\n",
      "31     feature.31  512   4   4  512   4   4      1024.0       0.03       32,768.0       16,384.0     36864.0      32768.0       0.00%     69632.0\n",
      "32     feature.32  512   4   4  512   4   4         0.0       0.03        8,192.0        8,192.0     32768.0      32768.0       0.00%     65536.0\n",
      "33     feature.33  512   4   4  512   4   4   2359296.0       0.03   75,489,280.0   37,748,736.0   9469952.0      32768.0       0.00%   9502720.0\n",
      "34     feature.34  512   4   4  512   4   4      1024.0       0.03       32,768.0       16,384.0     36864.0      32768.0       0.00%     69632.0\n",
      "35     feature.35  512   4   4  512   4   4         0.0       0.03        8,192.0        8,192.0     32768.0      32768.0       0.00%     65536.0\n",
      "36     feature.36  512   4   4  512   4   4   2359296.0       0.03   75,489,280.0   37,748,736.0   9469952.0      32768.0       0.00%   9502720.0\n",
      "37     feature.37  512   4   4  512   4   4      1024.0       0.03       32,768.0       16,384.0     36864.0      32768.0       0.00%     69632.0\n",
      "38     feature.38  512   4   4  512   4   4         0.0       0.03        8,192.0        8,192.0     32768.0      32768.0       0.00%     65536.0\n",
      "39     feature.39  512   4   4  512   2   2         0.0       0.01        6,144.0        8,192.0     32768.0       8192.0       0.00%     40960.0\n",
      "40     feature.40  512   2   2  512   2   2   2359296.0       0.01   18,872,320.0    9,437,184.0   9445376.0       8192.0       0.00%   9453568.0\n",
      "41     feature.41  512   2   2  512   2   2      1024.0       0.01        8,192.0        4,096.0     12288.0       8192.0       0.00%     20480.0\n",
      "42     feature.42  512   2   2  512   2   2         0.0       0.01        2,048.0        2,048.0      8192.0       8192.0       0.00%     16384.0\n",
      "43     feature.43  512   2   2  512   2   2   2359296.0       0.01   18,872,320.0    9,437,184.0   9445376.0       8192.0       0.00%   9453568.0\n",
      "44     feature.44  512   2   2  512   2   2      1024.0       0.01        8,192.0        4,096.0     12288.0       8192.0       0.00%     20480.0\n",
      "45     feature.45  512   2   2  512   2   2         0.0       0.01        2,048.0        2,048.0      8192.0       8192.0       0.00%     16384.0\n",
      "46     feature.46  512   2   2  512   2   2   2359296.0       0.01   18,872,320.0    9,437,184.0   9445376.0       8192.0       0.00%   9453568.0\n",
      "47     feature.47  512   2   2  512   2   2      1024.0       0.01        8,192.0        4,096.0     12288.0       8192.0       0.00%     20480.0\n",
      "48     feature.48  512   2   2  512   2   2         0.0       0.01        2,048.0        2,048.0      8192.0       8192.0       0.00%     16384.0\n",
      "49     feature.49  512   2   2  512   2   2   2359296.0       0.01   18,872,320.0    9,437,184.0   9445376.0       8192.0       0.00%   9453568.0\n",
      "50     feature.50  512   2   2  512   2   2      1024.0       0.01        8,192.0        4,096.0     12288.0       8192.0       0.00%     20480.0\n",
      "51     feature.51  512   2   2  512   2   2         0.0       0.01        2,048.0        2,048.0      8192.0       8192.0       0.00%     16384.0\n",
      "52     classifier          512           10      5130.0       0.00       10,230.0        5,120.0     22568.0         40.0      33.33%     22608.0\n",
      "total                                        20035018.0       3.59  797,577,206.0  399,168,512.0     22568.0         40.0     100.00%  87666512.0\n",
      "=================================================================================================================================================\n",
      "Total params: 20,035,018\n",
      "-------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "Total memory: 3.59MB\n",
      "Total MAdd: 797.58MMAdd\n",
      "Total Flops: 399.17MFlops\n",
      "Total MemR+W: 83.61MB\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from torchstat import stat\n",
    "\n",
    "stat(model, (3, 32, 32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.3  ptflops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: module vgg is treated as a zero-op.\n",
      "vgg(\n",
      "  20.035 M, 100.000% Params, 0.399 GMac, 100.000% MACs, \n",
      "  (feature): Sequential(\n",
      "    20.03 M, 99.974% Params, 0.399 GMac, 99.999% MACs, \n",
      "    (0): Conv2d(0.002 M, 0.009% Params, 0.002 GMac, 0.443% MACs, 3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (1): BatchNorm2d(0.0 M, 0.001% Params, 0.0 GMac, 0.033% MACs, 64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (2): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.016% MACs, inplace=True)\n",
      "    (3): Conv2d(0.037 M, 0.184% Params, 0.038 GMac, 9.457% MACs, 64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (4): BatchNorm2d(0.0 M, 0.001% Params, 0.0 GMac, 0.033% MACs, 64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (5): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.016% MACs, inplace=True)\n",
      "    (6): MaxPool2d(0.0 M, 0.000% Params, 0.0 GMac, 0.016% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (7): Conv2d(0.074 M, 0.368% Params, 0.019 GMac, 4.728% MACs, 64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (8): BatchNorm2d(0.0 M, 0.001% Params, 0.0 GMac, 0.016% MACs, 128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (9): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.008% MACs, inplace=True)\n",
      "    (10): Conv2d(0.147 M, 0.736% Params, 0.038 GMac, 9.457% MACs, 128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (11): BatchNorm2d(0.0 M, 0.001% Params, 0.0 GMac, 0.016% MACs, 128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (12): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.008% MACs, inplace=True)\n",
      "    (13): MaxPool2d(0.0 M, 0.000% Params, 0.0 GMac, 0.008% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (14): Conv2d(0.295 M, 1.472% Params, 0.019 GMac, 4.728% MACs, 128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (15): BatchNorm2d(0.001 M, 0.003% Params, 0.0 GMac, 0.008% MACs, 256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (16): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.004% MACs, inplace=True)\n",
      "    (17): Conv2d(0.59 M, 2.944% Params, 0.038 GMac, 9.457% MACs, 256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (18): BatchNorm2d(0.001 M, 0.003% Params, 0.0 GMac, 0.008% MACs, 256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (19): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.004% MACs, inplace=True)\n",
      "    (20): Conv2d(0.59 M, 2.944% Params, 0.038 GMac, 9.457% MACs, 256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (21): BatchNorm2d(0.001 M, 0.003% Params, 0.0 GMac, 0.008% MACs, 256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (22): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.004% MACs, inplace=True)\n",
      "    (23): Conv2d(0.59 M, 2.944% Params, 0.038 GMac, 9.457% MACs, 256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (24): BatchNorm2d(0.001 M, 0.003% Params, 0.0 GMac, 0.008% MACs, 256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (25): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.004% MACs, inplace=True)\n",
      "    (26): MaxPool2d(0.0 M, 0.000% Params, 0.0 GMac, 0.004% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (27): Conv2d(1.18 M, 5.888% Params, 0.019 GMac, 4.728% MACs, 256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (28): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.004% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (29): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.002% MACs, inplace=True)\n",
      "    (30): Conv2d(2.359 M, 11.776% Params, 0.038 GMac, 9.457% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (31): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.004% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (32): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.002% MACs, inplace=True)\n",
      "    (33): Conv2d(2.359 M, 11.776% Params, 0.038 GMac, 9.457% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (34): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.004% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (35): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.002% MACs, inplace=True)\n",
      "    (36): Conv2d(2.359 M, 11.776% Params, 0.038 GMac, 9.457% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (37): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.004% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (38): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.002% MACs, inplace=True)\n",
      "    (39): MaxPool2d(0.0 M, 0.000% Params, 0.0 GMac, 0.002% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (40): Conv2d(2.359 M, 11.776% Params, 0.009 GMac, 2.364% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (41): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.001% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (42): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.001% MACs, inplace=True)\n",
      "    (43): Conv2d(2.359 M, 11.776% Params, 0.009 GMac, 2.364% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (44): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.001% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (45): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.001% MACs, inplace=True)\n",
      "    (46): Conv2d(2.359 M, 11.776% Params, 0.009 GMac, 2.364% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (47): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.001% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (48): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.001% MACs, inplace=True)\n",
      "    (49): Conv2d(2.359 M, 11.776% Params, 0.009 GMac, 2.364% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (50): BatchNorm2d(0.001 M, 0.005% Params, 0.0 GMac, 0.001% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (51): ReLU(0.0 M, 0.000% Params, 0.0 GMac, 0.001% MACs, inplace=True)\n",
      "  )\n",
      "  (classifier): Linear(0.005 M, 0.026% Params, 0.0 GMac, 0.001% MACs, in_features=512, out_features=10, bias=True)\n",
      ")\n",
      "Computational complexity:       0.4 GMac\n",
      "Number of parameters:           20.04 M \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('0.4 GMac', '20.04 M')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torchvision.models as models\n",
    "import torch\n",
    "from ptflops import get_model_complexity_info\n",
    "\n",
    "with torch.cuda.device(0):\n",
    "    macs, params = get_model_complexity_info(model, (3, 32, 32), as_strings=True,\n",
    "                                           print_per_layer_stat=True, verbose=True)\n",
    "print('{:<30}  {:<8}'.format('Computational complexity: ', macs))\n",
    "print('{:<30}  {:<8}'.format('Number of parameters: ', params))\n",
    "macs, params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Pruned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "\n",
    "def resume_model(resume_file):\n",
    "    if not os.path.isfile(resume_file):\n",
    "        raise ValueError(\"Resume model file is not found at '{}'\".format(resume_file))\n",
    "    print(\"=> loading checkpoint '{}'\".format(resume_file))\n",
    "    checkpoint = torch.load(resume_file)\n",
    "    start_epoch = checkpoint['epoch']\n",
    "    best_prec1 = checkpoint['best_prec1']\n",
    "    state_dict = checkpoint['state_dict']\n",
    "    opti_dict = checkpoint['optimizer']\n",
    "    if 'cfg' in checkpoint:\n",
    "        cfg = checkpoint['cfg']\n",
    "    else:\n",
    "        cfg = None\n",
    "    print(\"cfg: {}\".format(cfg))\n",
    "    print(\"=> loaded checkpoint '{}' (epoch {}) Prec1: {:f}\"\n",
    "          .format(resume_file, start_epoch, best_prec1))\n",
    "    return state_dict, opti_dict, start_epoch, best_prec1, cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=> loading checkpoint 'D:\\Project\\Pycharm\\network-slimming\\logs\\ft_inherit_at_prune_vgg19_cifar100_percent_0.5_seed_2\\model_best.pth.tar'\n",
      "cfg: [21, 60, 'M', 107, 125, 'M', 240, 229, 201, 204, 'M', 315, 241, 141, 171, 'M', 149, 144, 165, 238]\n",
      "=> loaded checkpoint 'D:\\Project\\Pycharm\\network-slimming\\logs\\ft_inherit_at_prune_vgg19_cifar100_percent_0.5_seed_2\\model_best.pth.tar' (epoch 151) Prec1: 0.726200\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(151,\n",
       " tensor(0.7262),\n",
       " \"[21, 60, 'M', 107, 125, 'M', 240, 229, 201, 204, 'M', 315, 241, 141, 171, 'M', 149, 144, 165, 238]\")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filepath = r\"G:\\lizulin\\network-slimming\\logs\\ft_inherit_bn_prune_vgg19_cifar10_percent_0.7_seed_2\\model_best.pth.tar\"\n",
    "filepath2 = r\"D:\\Project\\Pycharm\\network-slimming\\logs\\attention_fine_tune_feature_vgg19_percent_0.7\\model_best.pth.tar\"\n",
    "filepath3 = r\"D:\\Project\\Pycharm\\network-slimming\\logs\\ft_inherit_at_prune_vgg19_cifar100_percent_0.5_seed_2\\model_best.pth.tar\"\n",
    "\n",
    "state_dict, opti_dict, start_epoch, best_prec1, cfg = resume_model(filepath3)\n",
    "start_epoch, best_prec1, str(list(cfg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = vgg(dataset='cifar10', depth=19, cfg=cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'torchstat'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-f11ab1e6da58>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtorchstat\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mstat\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mstat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'torchstat'"
     ]
    }
   ],
   "source": [
    "from torchstat import stat\n",
    "\n",
    "stat(model, (3, 32, 32))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}