WeightWatcher-SVDSmoothing-ONNX-DEBUG.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# WeightWatcher - SVDSmoothing\n",
    "\n",
    "### Example of SVDSmoothing using the VGG11 model \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-05T23:51:16.544327Z",
     "start_time": "2019-03-05T23:51:16.531457Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Suppress the powerlaw package warnings\n",
    "# \"powerlaw.py:700: RuntimeWarning: divide by zero encountered in true_divide\"\n",
    "# \"powerlaw.py:700: RuntimeWarning: invalid value encountered in true_divide\"\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=RuntimeWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import tensorflow\n",
    "import torchvision.models as models\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-06T00:16:43.249725Z",
     "start_time": "2019-03-06T00:16:43.240611Z"
    }
   },
   "source": [
    "### Import WeightWatcher\n",
    "\n",
    "set custom Logging at WARN Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(level=logging.DEBUG)\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.setLevel(logging.DEBUG)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.4.9'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import weightwatcher as ww\n",
    "logger = logging.getLogger(ww.__name__)\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "ww.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ONNX Example\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import onnx\n",
    "from onnx import numpy_helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:\n",
      "\n",
      "python      version 3.8.5 (default, Sep  4 2020, 02:22:02) \n",
      "[Clang 10.0.0 ]\n",
      "numpy       version 1.18.5\n",
      "tensforflow version 2.3.0\n",
      "keras       version 2.4.0\n",
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': 'last', 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': []}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 8, channels = 4 \n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 1x1 (NxM), 1x1 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=1 M=1 rf= 1 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 1 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 0 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (1, 1, 1, 1)  max size None\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 8x1 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=8 M=1 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 8 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 1 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (8, 1, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 2  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 2 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 3  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 3 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 4  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 4 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 5  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 5 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 6  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 6 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 16x8 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=16 M=8 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 16 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 7 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (16, 8, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 8  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 8 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 9  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 9 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 10  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 10 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 11  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 11 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 12  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 12 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 32x16 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=32 M=16 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 32 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 13 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (32, 16, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 14  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 14 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 15  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 15 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 16  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 16 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 17  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 17 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 18  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 18 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 32x32 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=32 M=32 rf= 9 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 32 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 19 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (32, 32, 3, 3)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 20  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 20 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 21  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 21 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 22  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 22 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 23  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 23 None type 1 unknown\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 24  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:weightwatcher:layer not supported: Layer 24 None type 1 unknown\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 1x32 (NxM), 7x7 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=1 M=32 rf= 49 channels = 4\n",
      "DEBUG:weightwatcher:layer_supported  N 1 max evals None\n",
      "DEBUG:weightwatcher:LAYER TYPE: 25 16  layer type <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:weights shape : (1, 32, 7, 7)  max size None\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [1]\n",
      "INFO:weightwatcher:Layer id 26  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:layer not supported: Layer 26 None type 1 unknown\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>layer_id</th>\n",
       "      <th>name</th>\n",
       "      <th>M</th>\n",
       "      <th>N</th>\n",
       "      <th>layer_type</th>\n",
       "      <th>num_evals</th>\n",
       "      <th>rf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>72.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>None</td>\n",
       "      <td>16.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>144.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19</td>\n",
       "      <td>None</td>\n",
       "      <td>32.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>288.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>25</td>\n",
       "      <td>None</td>\n",
       "      <td>32.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>LAYER_TYPE.CONV2D</td>\n",
       "      <td>1568.0</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  layer_id  name     M     N         layer_type  num_evals    rf\n",
       "0        0  None   1.0   1.0  LAYER_TYPE.CONV2D        1.0   1.0\n",
       "1        1  None   1.0   8.0  LAYER_TYPE.CONV2D        9.0   9.0\n",
       "2        7  None   8.0  16.0  LAYER_TYPE.CONV2D       72.0   9.0\n",
       "3       13  None  16.0  32.0  LAYER_TYPE.CONV2D      144.0   9.0\n",
       "4       19  None  32.0  32.0  LAYER_TYPE.CONV2D      288.0   9.0\n",
       "5       25  None  32.0   1.0  LAYER_TYPE.CONV2D     1568.0  49.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = onnx.load(\"example.onnx\")\n",
    "watcher = ww.WeightWatcher(model=model)\n",
    "watcher.describe(channels='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "watcher.analyze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [1, 7, 13, 19]}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 8, channels = 4 \n",
      "INFO:weightwatcher:Filtering layer by id 1\n",
      "INFO:weightwatcher:Filtering layer by id 7\n",
      "INFO:weightwatcher:Filtering layer by id 13\n",
      "INFO:weightwatcher:Filtering layer by id 19\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 1x1 (NxM), 1x1 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=1 M=1 rf= 1 channels = 4\n",
      "INFO:weightwatcher:skipping layer 0 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 1 max evals None\n",
      "DEBUG:weightwatcher:Layer 0 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 8x1 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=8 M=1 rf= 9 channels = 4\n",
      "INFO:weightwatcher:keeping layer 1 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 8 max evals None\n",
      "INFO:weightwatcher:LAYER: 1 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:apply ESD  on Layer 1 None \n",
      "DEBUG:weightwatcher:running SVD on Layer 1 None \n",
      "DEBUG:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [1, 7, 13, 19]} \n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(8, 1)  n_comp = 1\n",
      "DEBUG:weightwatcher:fitting power law on 9 eigenvalues\n",
      "DEBUG:weightwatcher:finding best distribution for fit\n",
      "Assuming nested distributions\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 2  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 2 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 2 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 3  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 3 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 3 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 4  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 4 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 4 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 5  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 5 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 5 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 6  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 6 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 6 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 16x8 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=16 M=8 rf= 9 channels = 4\n",
      "INFO:weightwatcher:keeping layer 7 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 16 max evals None\n",
      "INFO:weightwatcher:LAYER: 7 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:apply ESD  on Layer 7 None \n",
      "DEBUG:weightwatcher:running SVD on Layer 7 None \n",
      "DEBUG:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [1, 7, 13, 19]} \n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(16, 8)  n_comp = 8\n",
      "DEBUG:weightwatcher:fitting power law on 72 eigenvalues\n",
      "DEBUG:weightwatcher:finding best distribution for fit\n",
      "Assuming nested distributions\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 8  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 8 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 8 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 9  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 9 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 9 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 10  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 10 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 10 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 11  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 11 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 11 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 12  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 12 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 12 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 32x16 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=32 M=16 rf= 9 channels = 4\n",
      "INFO:weightwatcher:keeping layer 13 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 32 max evals None\n",
      "INFO:weightwatcher:LAYER: 13 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:apply ESD  on Layer 13 None \n",
      "DEBUG:weightwatcher:running SVD on Layer 13 None \n",
      "DEBUG:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [1, 7, 13, 19]} \n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 16)  n_comp = 16\n",
      "DEBUG:weightwatcher:fitting power law on 144 eigenvalues\n",
      "DEBUG:weightwatcher:finding best distribution for fit\n",
      "Assuming nested distributions\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 14  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 14 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 14 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 15  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 15 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 15 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 16  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 16 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 16 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 17  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 17 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 17 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 18  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 18 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 18 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 32x32 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=32 M=32 rf= 9 channels = 4\n",
      "INFO:weightwatcher:keeping layer 19 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 32 max evals None\n",
      "INFO:weightwatcher:LAYER: 19 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "DEBUG:weightwatcher:apply ESD  on Layer 19 None \n",
      "DEBUG:weightwatcher:running SVD on Layer 19 None \n",
      "DEBUG:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [1, 7, 13, 19]} \n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:Running full SVD:  W.shape=(32, 32)  n_comp = 32\n",
      "DEBUG:weightwatcher:fitting power law on 288 eigenvalues\n",
      "DEBUG:weightwatcher:finding best distribution for fit\n",
      "Assuming nested distributions\n",
      "INFO:weightwatcher:compare dist=lognormal R=0.2056875284453196 p=0.8370350011211934\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 20  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 20 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 20 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 21  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 21 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 21 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 22  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 22 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 22 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 23  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 23 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 23 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 24  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 24 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 24 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 1x32 (NxM), 7x7 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=1 M=32 rf= 49 channels = 4\n",
      "INFO:weightwatcher:skipping layer 25 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 1 max evals None\n",
      "DEBUG:weightwatcher:Layer 25 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [1]\n",
      "INFO:weightwatcher:Layer id 26  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 26 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 26 None is skipped\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>layer_id</th>\n",
       "      <th>name</th>\n",
       "      <th>D</th>\n",
       "      <th>M</th>\n",
       "      <th>N</th>\n",
       "      <th>alpha</th>\n",
       "      <th>alpha_weighted</th>\n",
       "      <th>best_fit</th>\n",
       "      <th>entropy</th>\n",
       "      <th>has_esd</th>\n",
       "      <th>...</th>\n",
       "      <th>num_evals</th>\n",
       "      <th>num_pl_spikes</th>\n",
       "      <th>rank_loss</th>\n",
       "      <th>rf</th>\n",
       "      <th>sigma</th>\n",
       "      <th>spectral_norm</th>\n",
       "      <th>stable_rank</th>\n",
       "      <th>sv_max</th>\n",
       "      <th>xmax</th>\n",
       "      <th>xmin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>0.259648</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>3.031012</td>\n",
       "      <td>0.802709</td>\n",
       "      <td>truncated_power_law</td>\n",
       "      <td>0.935927</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.767650</td>\n",
       "      <td>1.840061</td>\n",
       "      <td>4.396896</td>\n",
       "      <td>1.356488</td>\n",
       "      <td>1.840061</td>\n",
       "      <td>0.577397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "      <td>0.131396</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>6.060119</td>\n",
       "      <td>0.213827</td>\n",
       "      <td>truncated_power_law</td>\n",
       "      <td>0.901188</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>72.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.686706</td>\n",
       "      <td>1.084637</td>\n",
       "      <td>15.190852</td>\n",
       "      <td>1.041459</td>\n",
       "      <td>1.084637</td>\n",
       "      <td>0.566951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>None</td>\n",
       "      <td>0.142098</td>\n",
       "      <td>16.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>2.505632</td>\n",
       "      <td>-0.042734</td>\n",
       "      <td>truncated_power_law</td>\n",
       "      <td>0.913796</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>144.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.191215</td>\n",
       "      <td>0.961490</td>\n",
       "      <td>27.225922</td>\n",
       "      <td>0.980556</td>\n",
       "      <td>0.961490</td>\n",
       "      <td>0.153311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19</td>\n",
       "      <td>None</td>\n",
       "      <td>0.155170</td>\n",
       "      <td>32.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>2.851987</td>\n",
       "      <td>-0.502200</td>\n",
       "      <td>lognormal</td>\n",
       "      <td>0.890765</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>288.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.209696</td>\n",
       "      <td>0.666672</td>\n",
       "      <td>48.606331</td>\n",
       "      <td>0.816500</td>\n",
       "      <td>0.666672</td>\n",
       "      <td>0.160011</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  layer_id  name         D     M     N     alpha  alpha_weighted  \\\n",
       "0        1  None  0.259648   1.0   8.0  3.031012        0.802709   \n",
       "1        7  None  0.131396   8.0  16.0  6.060119        0.213827   \n",
       "2       13  None  0.142098  16.0  32.0  2.505632       -0.042734   \n",
       "3       19  None  0.155170  32.0  32.0  2.851987       -0.502200   \n",
       "\n",
       "              best_fit   entropy  has_esd  ...  num_evals num_pl_spikes  \\\n",
       "0  truncated_power_law  0.935927      1.0  ...        9.0           7.0   \n",
       "1  truncated_power_law  0.901188      1.0  ...       72.0           9.0   \n",
       "2  truncated_power_law  0.913796      1.0  ...      144.0          62.0   \n",
       "3            lognormal  0.890765      1.0  ...      288.0          78.0   \n",
       "\n",
       "   rank_loss   rf     sigma  spectral_norm  stable_rank    sv_max      xmax  \\\n",
       "0        0.0  9.0  0.767650       1.840061     4.396896  1.356488  1.840061   \n",
       "1        0.0  9.0  1.686706       1.084637    15.190852  1.041459  1.084637   \n",
       "2        0.0  9.0  0.191215       0.961490    27.225922  0.980556  0.961490   \n",
       "3        0.0  9.0  0.209696       0.666672    48.606331  0.816500  0.666672   \n",
       "\n",
       "       xmin  \n",
       "0  0.577397  \n",
       "1  0.566951  \n",
       "2  0.153311  \n",
       "3  0.160011  \n",
       "\n",
       "[4 rows x 27 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "watcher.analyze(layers=[1,7,13,19])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:\n",
      "\n",
      "python      version 3.8.5 (default, Sep  4 2020, 02:22:02) \n",
      "[Clang 10.0.0 ]\n",
      "numpy       version 1.18.5\n",
      "tensforflow version 2.3.0\n",
      "keras       version 2.4.0\n",
      "INFO:weightwatcher:params {'glorot_fix': False, 'normalize': False, 'conv2d_norm': True, 'randomize': False, 'savefig': False, 'rescale': True, 'deltaEs': False, 'intra': False, 'channels': None, 'conv2d_fft': False, 'min_evals': 0, 'max_evals': None, 'plot': False, 'mp_fit': False, 'ww2x': False, 'layers': [7, 13, 19], 'percent': 0.2}\n",
      "DEBUG:weightwatcher:FRAMEWORKS: KERAS = 4  PYTORCH = 2 ONNX = 8 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:FIRST = 2  LAST = 4 UNKNOWN = 1 \n",
      "DEBUG:weightwatcher:MODEL ITERATOR, framework = 8, channels = 4 \n",
      "INFO:weightwatcher:Filtering layer by id 7\n",
      "INFO:weightwatcher:Filtering layer by id 13\n",
      "INFO:weightwatcher:Filtering layer by id 19\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 1x1 (NxM), 1x1 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=1 M=1 rf= 1 channels = 4\n",
      "INFO:weightwatcher:skipping layer 0 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 1 max evals None\n",
      "DEBUG:weightwatcher:Layer 0 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 8x1 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=8 M=1 rf= 9 channels = 4\n",
      "INFO:weightwatcher:skipping layer 1 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 8 max evals None\n",
      "DEBUG:weightwatcher:Layer 1 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 2  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 2 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 2 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 3  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 3 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 3 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 4  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 4 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 4 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 5  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 5 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 5 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [8]\n",
      "INFO:weightwatcher:Layer id 6  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 6 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 6 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 16x8 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=16 M=8 rf= 9 channels = 4\n",
      "INFO:weightwatcher:keeping layer 7 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 16 max evals None\n",
      "INFO:weightwatcher:LAYER: 7 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 7 None, keeping 0.20% percent , or ncomp=1 out of 8. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:smoothed W (16, 8) -> (16, 8) n_comp=1\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 8  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 8 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 8 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 9  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 9 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 9 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 10  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 10 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 10 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 11  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 11 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 11 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [16]\n",
      "INFO:weightwatcher:Layer id 12  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 12 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 12 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 32x16 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=32 M=16 rf= 9 channels = 4\n",
      "INFO:weightwatcher:keeping layer 13 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 32 max evals None\n",
      "INFO:weightwatcher:LAYER: 13 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 13 None, keeping 0.20% percent , or ncomp=3 out of 16. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:smoothed W (32, 16) -> (32, 16) n_comp=3\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 14  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 14 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 14 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 15  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 15 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 15 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 16  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 16 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 16 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 17  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 17 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 17 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 18  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 18 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 18 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 32x32 (NxM), 3x3 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=32 M=32 rf= 9 channels = 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:weightwatcher:keeping layer 19 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 32 max evals None\n",
      "INFO:weightwatcher:LAYER: 19 16  : <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:apply truncated SVD on Layer 19 None, keeping 0.20% percent , or ncomp=6 out of 32. of the singular vectors\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:smoothed W (32, 32) -> (32, 32) n_comp=6\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 20  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 20 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 20 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 21  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 21 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 21 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 22  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 22 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 22 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 23  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 23 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 23 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [32]\n",
      "INFO:weightwatcher:Layer id 24  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 24 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 24 None is skipped\n",
      "DEBUG:weightwatcher:conv2D_Wmats\n",
      "DEBUG:weightwatcher:Channels Last tensor shape: 1x32 (NxM), 7x7 (i,j)\n",
      "DEBUG:weightwatcher:get_conv2D_Wmats N=1 M=32 rf= 49 channels = 4\n",
      "INFO:weightwatcher:skipping layer 25 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 1 max evals None\n",
      "DEBUG:weightwatcher:Layer 25 None is skipped\n",
      "DEBUG:weightwatcher:Unsupported ONNX Layer, dims = [1]\n",
      "INFO:weightwatcher:Layer id 26  unknown type 1 layer  <class 'weightwatcher.weightwatcher.ONNXLayer'>\n",
      "INFO:weightwatcher:skipping layer 26 None by id\n",
      "DEBUG:weightwatcher:layer_supported  N 0 max evals None\n",
      "DEBUG:weightwatcher:Layer 26 None is skipped\n",
      "INFO:weightwatcher:Returning smoothed model\n"
     ]
    }
   ],
   "source": [
    "watcher = ww.WeightWatcher(model=model)\n",
    "smoothed_model = watcher.SVDSmoothing(layers=[7,13,19])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}