WeightWatcher-SVDSmoothing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# WeightWatcher - SVDSmoothing\n",
    "\n",
    "### Example of SVDSmoothing using the VGG11 model \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-05T23:51:16.544327Z",
     "start_time": "2019-03-05T23:51:16.531457Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Suppress the powerlaw package warnings\n",
    "# \"powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\"\n",
    "# \"powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\"\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=RuntimeWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import tensorflow\n",
    "import torchvision.models as models\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-06T00:16:43.249725Z",
     "start_time": "2019-03-06T00:16:43.240611Z"
    }
   },
   "source": [
    "### Import WeightWatcher\n",
    "\n",
    "set custom Logging at WARN Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(level=logging.DEBUG)\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.setLevel(logging.DEBUG)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.4.9'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import weightwatcher as ww\n",
    "logger = logging.getLogger(ww.__name__)\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "ww.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:\n",
      "\n",
      "python      version 3.8.5 (default, Sep  4 2020, 02:22:02) \n",
      "[Clang 10.0.0 ]\n",
      "numpy       version 1.18.5\n",
      "tensforflow version 2.3.0\n",
      "keras       version 2.4.0\n",
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': []}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 2, channels = 4 \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 1 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 64x3 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=3 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 2 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (64, 3, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 3 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 4 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 128x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=64 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 5 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (128, 64, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 7 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=128 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 8 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (256, 128, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 9 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 10 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (256, 256, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 11 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 12 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=256 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 13 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 256, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 15 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 512, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 16 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 17 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 18 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 512, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 19 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 20 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 512, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 21 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 22 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 23 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 24 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 25 4  layer type <class 'torch.nn.modules.linear.Linear'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 25088)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 26 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 27 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 28 4  layer type <class 'torch.nn.modules.linear.Linear'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 4096)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 29 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 30 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 31 4  layer type <class 'torch.nn.modules.linear.Linear'>\n",
      "DEBUG:weightwatcher:weights shape : (1000, 4096)  max size None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>layer_id</th>\n",
       "      <th>name</th>\n",
       "      <th>M</th>\n",
       "      <th>N</th>\n",
       "      <th>layer_type</th>\n",
       "      <th>num_evals</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>3.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>27.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>64.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>576.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>128.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>1152.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>256.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>None</td>\n",
       "      <td>256.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>15</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>18</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>25</td>\n",
       "      <td>None</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>25088.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>28</td>\n",
       "      <td>None</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>31</td>\n",
       "      <td>None</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   layer_id  name       M        N         layer_type  num_evals   rf\n",
       "0         2  None     3.0     64.0  LAYER_TYPE.CONV2D       27.0  9.0\n",
       "1         5  None    64.0    128.0  LAYER_TYPE.CONV2D      576.0  9.0\n",
       "2         8  None   128.0    256.0  LAYER_TYPE.CONV2D     1152.0  9.0\n",
       "3        10  None   256.0    256.0  LAYER_TYPE.CONV2D     2304.0  9.0\n",
       "4        13  None   256.0    512.0  LAYER_TYPE.CONV2D     2304.0  9.0\n",
       "5        15  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "6        18  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "7        20  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "8        25  None  4096.0  25088.0   LAYER_TYPE.DENSE     4096.0  1.0\n",
       "9        28  None  4096.0   4096.0   LAYER_TYPE.DENSE     4096.0  1.0\n",
       "10       31  None  1000.0   4096.0   LAYER_TYPE.DENSE     1000.0  1.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vgg11 = models.vgg11(pretrained=True)\n",
    "watcher = ww.WeightWatcher(model=vgg11)\n",
    "watcher.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [31], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 2, channels = 4 \n",
      "INFO:weightwatcher:Filtering layer by id 31\n",
      "INFO:weightwatcher:skipping layer 0 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 0 None is skipped\n",
      "INFO:weightwatcher:skipping layer 1 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 1 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 64x3 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=3 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 2 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:Layer 2 None is skipped\n",
      "INFO:weightwatcher:skipping layer 3 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 3 None is skipped\n",
      "INFO:weightwatcher:skipping layer 4 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 4 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 128x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=64 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 5 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:Layer 5 None is skipped\n",
      "INFO:weightwatcher:skipping layer 6 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 6 None is skipped\n",
      "INFO:weightwatcher:skipping layer 7 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 7 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=128 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 8 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 8 None is skipped\n",
      "INFO:weightwatcher:skipping layer 9 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 9 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 10 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 10 None is skipped\n",
      "INFO:weightwatcher:skipping layer 11 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 11 None is skipped\n",
      "INFO:weightwatcher:skipping layer 12 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 12 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=256 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 13 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 13 None is skipped\n",
      "INFO:weightwatcher:skipping layer 14 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 14 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 15 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 15 None is skipped\n",
      "INFO:weightwatcher:skipping layer 16 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 16 None is skipped\n",
      "INFO:weightwatcher:skipping layer 17 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 17 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 18 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 18 None is skipped\n",
      "INFO:weightwatcher:skipping layer 19 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 19 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 20 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 20 None is skipped\n",
      "INFO:weightwatcher:skipping layer 21 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 21 None is skipped\n",
      "INFO:weightwatcher:skipping layer 22 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 22 None is skipped\n",
      "INFO:weightwatcher:skipping layer 23 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 23 None is skipped\n",
      "INFO:weightwatcher:skipping layer 24 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 24 None is skipped\n",
      "INFO:weightwatcher:skipping layer 25 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "DEBUG:weightwatcher:Layer 25 None is skipped\n",
      "INFO:weightwatcher:skipping layer 26 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 26 None is skipped\n",
      "INFO:weightwatcher:skipping layer 27 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 27 None is skipped\n",
      "INFO:weightwatcher:skipping layer 28 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:Layer 28 None is skipped\n",
      "INFO:weightwatcher:skipping layer 29 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 29 None is skipped\n",
      "INFO:weightwatcher:skipping layer 30 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 30 None is skipped\n",
      "INFO:weightwatcher:keeping layer 31 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 31 4  : <class 'torch.nn.modules.linear.Linear'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 31 None, keeping 0.20% percent , or ncomp=200 out of 1000. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (1000, 4096) -> (1000, 4096) n_comp=200\n",
      "INFO:weightwatcher:Returning smoothed model\n"
     ]
    }
   ],
   "source": [
    "smoothed_vgg11 = watcher.SVDSmoothing( layers=[31])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 2, channels = 4 \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 1 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 64x3 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=3 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 2 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (64, 3, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 3 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 4 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 128x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=64 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 5 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (128, 64, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 7 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=128 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 8 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (256, 128, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 9 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 10 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (256, 256, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 11 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 12 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=256 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 13 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 256, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 15 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 512, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 16 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 17 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 18 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 512, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 19 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 20 16  layer type <class 'torch.nn.modules.conv.Conv2d'>\n",
      "DEBUG:weightwatcher:weights shape : (512, 512, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 21 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 22 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 23 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 24 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 25 4  layer type <class 'torch.nn.modules.linear.Linear'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 25088)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 26 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 27 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 28 4  layer type <class 'torch.nn.modules.linear.Linear'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 4096)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 29 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 30 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 31 4  layer type <class 'torch.nn.modules.linear.Linear'>\n",
      "DEBUG:weightwatcher:weights shape : (1000, 4096)  max size None\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 2, channels = 4 \n",
      "INFO:weightwatcher:Filtering layer by id 31\n",
      "INFO:weightwatcher:skipping layer 0 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 0 None is skipped\n",
      "INFO:weightwatcher:skipping layer 1 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 1 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 64x3 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=3 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 2 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:Layer 2 None is skipped\n",
      "INFO:weightwatcher:skipping layer 3 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 3 None is skipped\n",
      "INFO:weightwatcher:skipping layer 4 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 4 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 128x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=64 rf= 9 channels = 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:skipping layer 5 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:Layer 5 None is skipped\n",
      "INFO:weightwatcher:skipping layer 6 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 6 None is skipped\n",
      "INFO:weightwatcher:skipping layer 7 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 7 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=128 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 8 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 8 None is skipped\n",
      "INFO:weightwatcher:skipping layer 9 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 9 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 10 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 10 None is skipped\n",
      "INFO:weightwatcher:skipping layer 11 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 11 None is skipped\n",
      "INFO:weightwatcher:skipping layer 12 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 12 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=256 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 13 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 13 None is skipped\n",
      "INFO:weightwatcher:skipping layer 14 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 14 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 15 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 15 None is skipped\n",
      "INFO:weightwatcher:skipping layer 16 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 16 None is skipped\n",
      "INFO:weightwatcher:skipping layer 17 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 17 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 18 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 18 None is skipped\n",
      "INFO:weightwatcher:skipping layer 19 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 19 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 20 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 20 None is skipped\n",
      "INFO:weightwatcher:skipping layer 21 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 21 None is skipped\n",
      "INFO:weightwatcher:skipping layer 22 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 22 None is skipped\n",
      "INFO:weightwatcher:skipping layer 23 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 23 None is skipped\n",
      "INFO:weightwatcher:skipping layer 24 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 24 None is skipped\n",
      "INFO:weightwatcher:skipping layer 25 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "DEBUG:weightwatcher:Layer 25 None is skipped\n",
      "INFO:weightwatcher:skipping layer 26 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 26 None is skipped\n",
      "INFO:weightwatcher:skipping layer 27 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 27 None is skipped\n",
      "INFO:weightwatcher:skipping layer 28 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:Layer 28 None is skipped\n",
      "INFO:weightwatcher:skipping layer 29 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 29 None is skipped\n",
      "INFO:weightwatcher:skipping layer 30 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 30 None is skipped\n",
      "INFO:weightwatcher:keeping layer 31 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:Getting ESD for layer 31 \n",
      "DEBUG:weightwatcher:apply ESD  on Layer 31 None \n",
      "DEBUG:weightwatcher:running SVD on Layer 31 None \n",
      "DEBUG:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2} \n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(1000, 4096)  n_comp = 1000\n",
      "INFO:weightwatcher:Found 1000 eiganvalues for 31 None\n"
     ]
    }
   ],
   "source": [
    "smoother_esd_31 = watcher.get_ESD(model=smoothed_vgg11, layer=31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 2, channels = 4 \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 1 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 64x3 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=3 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "INFO:weightwatcher:LAYER: 2 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 2 None, keeping 0.20% percent , or ncomp=0 out of 3. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 2 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 3 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 4 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 128x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=64 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "INFO:weightwatcher:LAYER: 5 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 5 None, keeping 0.20% percent , or ncomp=12 out of 64. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 5 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 7 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=128 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "INFO:weightwatcher:LAYER: 8 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 8 None, keeping 0.20% percent , or ncomp=25 out of 128. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 8 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 9 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "INFO:weightwatcher:LAYER: 10 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 10 None, keeping 0.20% percent , or ncomp=51 out of 256. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 10 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 11 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 12 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=256 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 13 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 13 None, keeping 0.20% percent , or ncomp=51 out of 256. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 13 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 15 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 15 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 15 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 16 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 17 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 18 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 18 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 18 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 19 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 20 16  : <class 'torch.nn.modules.conv.Conv2d'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 20 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 20 None skipped \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 21 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 22 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 23 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 24 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "INFO:weightwatcher:LAYER: 25 4  : <class 'torch.nn.modules.linear.Linear'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 25 None, keeping 0.20% percent , or ncomp=819 out of 4096. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 25088) -> (4096, 25088) n_comp=819\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 26 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 27 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 28 4  : <class 'torch.nn.modules.linear.Linear'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 28 None, keeping 0.20% percent , or ncomp=819 out of 4096. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 4096) -> (4096, 4096) n_comp=819\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 29 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 30 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 31 4  : <class 'torch.nn.modules.linear.Linear'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 31 None, keeping 0.20% percent , or ncomp=200 out of 1000. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (1000, 4096) -> (1000, 4096) n_comp=200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:Returning smoothed model\n"
     ]
    }
   ],
   "source": [
    "smoothed_vgg11 = watcher.SVDSmoothing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keras Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:\n",
      "\n",
      "python      version 3.8.5 (default, Sep  4 2020, 02:22:02) \n",
      "[Clang 10.0.0 ]\n",
      "numpy       version 1.18.5\n",
      "tensforflow version 2.3.0\n",
      "keras       version 2.4.0\n",
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 4, channels = 2 \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 3x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=64 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 1 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 3, 64)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 64x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=64 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 2 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 64, 64)  max size None\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 3 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 64x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=128 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 4 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 64, 128)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 128x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=128 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 5 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 128, 128)  max size None\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 128x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=256 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 7 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 128, 256)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 8 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 256, 256)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 9 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 256, 256)  max size None\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 10 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 11 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 256, 512)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 12 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 512, 512)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 13 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 512, 512)  max size None\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 15 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 512, 512)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 16 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 512, 512)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 17 16  layer type <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "DEBUG:weightwatcher:weights shape : (3, 3, 512, 512)  max size None\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 18 None has no weights\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.FLATTENED not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 19 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 20 4  layer type <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "DEBUG:weightwatcher:weights shape : (25088, 4096)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 21 4  layer type <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 4096)  max size None\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 22 4  layer type <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 1000)  max size None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>layer_id</th>\n",
       "      <th>name</th>\n",
       "      <th>M</th>\n",
       "      <th>N</th>\n",
       "      <th>layer_type</th>\n",
       "      <th>num_evals</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>64.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>576.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>64.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>576.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>128.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>1152.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>128.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>1152.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>256.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>256.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9</td>\n",
       "      <td>None</td>\n",
       "      <td>256.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>13</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>15</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>16</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>17</td>\n",
       "      <td>None</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>4608.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>25088.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>21</td>\n",
       "      <td>None</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>22</td>\n",
       "      <td>None</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   layer_id  name       M        N         layer_type  num_evals   rf\n",
       "0         1  None    64.0      3.0  LAYER_TYPE.CONV2D      576.0  9.0\n",
       "1         2  None    64.0     64.0  LAYER_TYPE.CONV2D      576.0  9.0\n",
       "2         4  None   128.0     64.0  LAYER_TYPE.CONV2D     1152.0  9.0\n",
       "3         5  None   128.0    128.0  LAYER_TYPE.CONV2D     1152.0  9.0\n",
       "4         7  None   256.0    128.0  LAYER_TYPE.CONV2D     2304.0  9.0\n",
       "5         8  None   256.0    256.0  LAYER_TYPE.CONV2D     2304.0  9.0\n",
       "6         9  None   256.0    256.0  LAYER_TYPE.CONV2D     2304.0  9.0\n",
       "7        11  None   512.0    256.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "8        12  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "9        13  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "10       15  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "11       16  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "12       17  None   512.0    512.0  LAYER_TYPE.CONV2D     4608.0  9.0\n",
       "13       20  None  4096.0  25088.0   LAYER_TYPE.DENSE     4096.0  1.0\n",
       "14       21  None  4096.0   4096.0   LAYER_TYPE.DENSE     4096.0  1.0\n",
       "15       22  None  1000.0   4096.0   LAYER_TYPE.DENSE     1000.0  1.0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.applications.vgg16 import VGG16\n",
    "vgg16 = VGG16()\n",
    "watcher = ww.WeightWatcher(model=vgg16)\n",
    "watcher.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [22], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 4, channels = 2 \n",
      "INFO:weightwatcher:Filtering layer by id 22\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "INFO:weightwatcher:skipping layer 0 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 0 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 3x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=64 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 1 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "DEBUG:weightwatcher:Layer 1 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 64x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=64 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 2 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:Layer 2 None is skipped\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "INFO:weightwatcher:skipping layer 3 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 3 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 64x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=128 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 4 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "DEBUG:weightwatcher:Layer 4 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 128x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=128 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 5 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:Layer 5 None is skipped\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "INFO:weightwatcher:skipping layer 6 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 6 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 128x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=256 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 7 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "DEBUG:weightwatcher:Layer 7 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 8 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 8 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 9 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 9 None is skipped\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "INFO:weightwatcher:skipping layer 10 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 10 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=512 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 11 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "DEBUG:weightwatcher:Layer 11 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 12 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 12 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 13 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 13 None is skipped\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "INFO:weightwatcher:skipping layer 14 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 14 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 15 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 15 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 16 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 16 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "INFO:weightwatcher:skipping layer 17 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "DEBUG:weightwatcher:Layer 17 None is skipped\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "INFO:weightwatcher:skipping layer 18 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 18 None is skipped\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.FLATTENED not found \n",
      "INFO:weightwatcher:skipping layer 19 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 19 None is skipped\n",
      "INFO:weightwatcher:skipping layer 20 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "DEBUG:weightwatcher:Layer 20 None is skipped\n",
      "INFO:weightwatcher:skipping layer 21 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:Layer 21 None is skipped\n",
      "INFO:weightwatcher:keeping layer 22 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 22 4  : <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 22 None, keeping 0.20% percent , or ncomp=200 out of 1000. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 1000) -> (4096, 1000) n_comp=200\n",
      "INFO:weightwatcher:Returning smoothed model\n"
     ]
    }
   ],
   "source": [
    "smoothed_vgg16 = watcher.SVDSmoothing(layers = [22])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 4, channels = 2 \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 3x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=64 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "INFO:weightwatcher:LAYER: 1 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 1 None, keeping 0.20% percent , or ncomp=12 out of 64. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 1 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 64x64 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=64 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "INFO:weightwatcher:LAYER: 2 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 2 None, keeping 0.20% percent , or ncomp=12 out of 64. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 2 None skipped \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 3 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 64x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=64 M=128 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 64 max evals None\n",
      "INFO:weightwatcher:LAYER: 4 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 4 None, keeping 0.20% percent , or ncomp=25 out of 128. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 4 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 128x128 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=128 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "INFO:weightwatcher:LAYER: 5 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 5 None, keeping 0.20% percent , or ncomp=25 out of 128. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 5 None skipped \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 128x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=128 M=256 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 128 max evals None\n",
      "INFO:weightwatcher:LAYER: 7 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 7 None, keeping 0.20% percent , or ncomp=51 out of 256. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 7 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "INFO:weightwatcher:LAYER: 8 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 8 None, keeping 0.20% percent , or ncomp=51 out of 256. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 8 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x256 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=256 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "INFO:weightwatcher:LAYER: 9 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 9 None, keeping 0.20% percent , or ncomp=51 out of 256. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 9 None skipped \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 10 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 256x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=256 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 256 max evals None\n",
      "INFO:weightwatcher:LAYER: 11 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 11 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 11 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 12 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 12 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 12 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 13 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 13 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 13 None skipped \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None has no weights\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 15 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 15 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 15 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 16 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 16 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 16 None skipped \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels First shape: 512x512 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=512 M=512 rf= 9 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 512 max evals None\n",
      "INFO:weightwatcher:LAYER: 17 16  : <class 'tensorflow.python.keras.layers.convolutional.Conv2D'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 17 None, keeping 0.20% percent , or ncomp=102 out of 512. of the singular vectors\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 17 None skipped \n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.UNKNOWN not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 18 None has no weights\n",
      "WARNING:weightwatcher:layer type LAYER_TYPE.FLATTENED not found \n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 19 None has no weights\n",
      "DEBUG:weightwatcher:layer_supported  N 25088 max evals None\n",
      "INFO:weightwatcher:LAYER: 20 4  : <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 20 None, keeping 0.20% percent , or ncomp=819 out of 4096. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (25088, 4096) -> (25088, 4096) n_comp=819\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 21 4  : <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 21 None, keeping 0.20% percent , or ncomp=819 out of 4096. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 4096) -> (4096, 4096) n_comp=819\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 22 4  : <class 'tensorflow.python.keras.layers.core.Dense'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 22 None, keeping 0.20% percent , or ncomp=200 out of 1000. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 1000) -> (4096, 1000) n_comp=200\n",
      "INFO:weightwatcher:Returning smoothed model\n"
     ]
    }
   ],
   "source": [
    "smoothed_vgg16 = watcher.SVDSmoothing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ONNX Example\n",
    "\n",
    "download latest alexnet.onnx from\n",
    "\n",
    "https://github.com/onnx/models/tree/master/vision/classification/alexnet\n",
    "\n",
    "and rename to alexnet.onnx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import onnx\n",
    "from onnx import numpy_helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:\n",
      "\n",
      "python      version 3.8.5 (default, Sep  4 2020, 02:22:02) \n",
      "[Clang 10.0.0 ]\n",
      "numpy       version 1.18.5\n",
      "tensforflow version 2.3.0\n",
      "keras       version 2.4.0\n",
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 8, channels = 2 \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [96]\n",
      "INFO:weightwatcher:Layer id 0  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 96x3 ()x11\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=11 M=11 rf= 288 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 11 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 1 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (96, 3, 11, 11)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [256]\n",
      "INFO:weightwatcher:Layer id 2  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 2 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 256x48 ()x5\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=5 M=5 rf= 12288 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 5 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 3 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (256, 48, 5, 5)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [384]\n",
      "INFO:weightwatcher:Layer id 4  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 4 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 384x256 ()x3\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=3 rf= 98304 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 5 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (384, 256, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [384]\n",
      "INFO:weightwatcher:Layer id 6  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 384x192 ()x3\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=3 rf= 73728 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 7 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (384, 192, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [256]\n",
      "INFO:weightwatcher:Layer id 8  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 8 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 256x192 ()x3\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=3 rf= 49152 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 9 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (256, 192, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [4096]\n",
      "INFO:weightwatcher:Layer id 10  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 10 None type 1 unknown\n",
      "DEBUG:weightwatcher:layer_supported  N 9216 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 11 4  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 9216)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [4096]\n",
      "INFO:weightwatcher:Layer id 12  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 12 None type 1 unknown\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 13 4  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (4096, 4096)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [1000]\n",
      "INFO:weightwatcher:Layer id 14  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None type 1 unknown\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 15 4  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (1000, 4096)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [2]\n",
      "INFO:weightwatcher:Layer id 16  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 16 None type 1 unknown\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>layer_id</th>\n",
       "      <th>name</th>\n",
       "      <th>M</th>\n",
       "      <th>N</th>\n",
       "      <th>layer_type</th>\n",
       "      <th>num_evals</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>11.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>3168.0</td>\n",
       "      <td>288.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>61440.0</td>\n",
       "      <td>12288.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>294912.0</td>\n",
       "      <td>98304.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>221184.0</td>\n",
       "      <td>73728.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>None</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>147456.0</td>\n",
       "      <td>49152.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11</td>\n",
       "      <td>None</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>9216.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13</td>\n",
       "      <td>None</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>15</td>\n",
       "      <td>None</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>4096.0</td>\n",
       "      <td>LAYER_TYPE.DENSE</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  layer_id  name       M       N         layer_type  num_evals       rf\n",
       "0        1  None    11.0    11.0  LAYER_TYPE.CONV2D     3168.0    288.0\n",
       "1        3  None     5.0     5.0  LAYER_TYPE.CONV2D    61440.0  12288.0\n",
       "2        5  None     3.0     3.0  LAYER_TYPE.CONV2D   294912.0  98304.0\n",
       "3        7  None     3.0     3.0  LAYER_TYPE.CONV2D   221184.0  73728.0\n",
       "4        9  None     3.0     3.0  LAYER_TYPE.CONV2D   147456.0  49152.0\n",
       "5       11  None  4096.0  9216.0   LAYER_TYPE.DENSE     4096.0      1.0\n",
       "6       13  None  4096.0  4096.0   LAYER_TYPE.DENSE     4096.0      1.0\n",
       "7       15  None  1000.0  4096.0   LAYER_TYPE.DENSE     1000.0      1.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alexnet = onnx.load(\"alexnet.onnx\")\n",
    "watcher = ww.WeightWatcher(model=alexnet)\n",
    "watcher.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 8, channels = 2 \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [96]\n",
      "INFO:weightwatcher:Layer id 0  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 0 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 96x3 ()x11\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=11 M=11 rf= 288 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 11 max evals None\n",
      "INFO:weightwatcher:LAYER: 1 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 1 None, keeping 0.20% percent , or ncomp=2 out of 11. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 1 None skipped \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [256]\n",
      "INFO:weightwatcher:Layer id 2  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 2 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 256x48 ()x5\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=5 M=5 rf= 12288 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 5 max evals None\n",
      "INFO:weightwatcher:LAYER: 3 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 3 None, keeping 0.20% percent , or ncomp=1 out of 5. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 3 None skipped \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [384]\n",
      "INFO:weightwatcher:Layer id 4  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 4 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 384x256 ()x3\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=3 rf= 98304 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "INFO:weightwatcher:LAYER: 5 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 5 None, keeping 0.20% percent , or ncomp=0 out of 3. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 5 None skipped \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [384]\n",
      "INFO:weightwatcher:Layer id 6  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 384x192 ()x3\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=3 rf= 73728 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "INFO:weightwatcher:LAYER: 7 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 7 None, keeping 0.20% percent , or ncomp=0 out of 3. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 7 None skipped \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [256]\n",
      "INFO:weightwatcher:Layer id 8  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 8 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "WARNING:weightwatcher:warning, expected channels FIRST,  detected channels LAST\n",
      "DEBUG:weightwatcher:Flipping LAST to FIRST Channel, 256x192 ()x3\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=3 M=3 rf= 49152 channels = 2\n",
      "DEBUG:weightwatcher:layer_supported  N 3 max evals None\n",
      "INFO:weightwatcher:LAYER: 9 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 9 None, keeping 0.20% percent , or ncomp=0 out of 3. of the singular vectors\n",
      "WARNING:weightwatcher:Something went wrong,UNKNOWN layer 9 None skipped \n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [4096]\n",
      "INFO:weightwatcher:Layer id 10  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 10 None type 1 unknown\n",
      "DEBUG:weightwatcher:layer_supported  N 9216 max evals None\n",
      "INFO:weightwatcher:LAYER: 11 4  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 11 None, keeping 0.20% percent , or ncomp=819 out of 4096. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 9216) -> (4096, 9216) n_comp=819\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [4096]\n",
      "INFO:weightwatcher:Layer id 12  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 12 None type 1 unknown\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 13 4  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 13 None, keeping 0.20% percent , or ncomp=819 out of 4096. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (4096, 4096) -> (4096, 4096) n_comp=819\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [1000]\n",
      "INFO:weightwatcher:Layer id 14  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None type 1 unknown\n",
      "DEBUG:weightwatcher:layer_supported  N 4096 max evals None\n",
      "INFO:weightwatcher:LAYER: 15 4  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 15 None, keeping 0.20% percent , or ncomp=200 out of 1000. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (1000, 4096) -> (1000, 4096) n_comp=200\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [2]\n",
      "INFO:weightwatcher:Layer id 16  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 16 None type 1 unknown\n",
      "INFO:weightwatcher:Returning smoothed model\n"
     ]
    }
   ],
   "source": [
    "smoothed_alexnet = watcher.SVDSmoothing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}