unit06-dl-tips/6.8-debugging/6.8-part3-overfit-batches.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "77b76d5f-d7cd-4a9e-b644-5d6104d95946",
   "metadata": {},
   "source": [
    "# Unit 6. Essential Deep Learning Tips & Tricks\n",
    "\n",
    "## 6.8 Debugging Deep Neural Networks\n",
    "\n",
    "## Part 3. Batch Overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eec68953",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch       : 1.13.0\n",
      "lightning   : 1.8.4.post0\n",
      "torchmetrics: 0.11.0\n",
      "\n",
      "conda environment: dl-fundamentals\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -p torch,lightning,torchmetrics --conda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e80fef60",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightning as L\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torchmetrics\n",
    "from lightning.pytorch.loggers import CSVLogger\n",
    "\n",
    "from shared_utilities import CustomDataModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b119b7c9-18ac-4c89-a574-a1c63783605c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PyTorchMLP(torch.nn.Module):\n",
    "    def __init__(self, num_features, num_classes):\n",
    "        super().__init__()\n",
    "\n",
    "        self.all_layers = torch.nn.Sequential(\n",
    "            # 1st hidden layer\n",
    "            torch.nn.Linear(num_features, 100),\n",
    "            torch.nn.BatchNorm1d(100),\n",
    "            #torch.nn.ReLU(),\n",
    "            \n",
    "            # 2nd hidden layer\n",
    "            torch.nn.Linear(100, 50),\n",
    "            torch.nn.BatchNorm1d(50),\n",
    "            #torch.nn.ReLU(),\n",
    "            \n",
    "            # output layer\n",
    "            torch.nn.Linear(50, num_classes),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.flatten(x, start_dim=1)\n",
    "        logits = self.all_layers(x)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26845f11-4a0d-48f3-818a-22e4093b79ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LightningModel(L.LightningModule):\n",
    "    def __init__(self, model, learning_rate):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.example_input_array = torch.Tensor(5, 100) # example batch\n",
    "\n",
    "        self.learning_rate = learning_rate\n",
    "        self.model = model\n",
    "\n",
    "        self.save_hyperparameters(ignore=[\"model\"])\n",
    "\n",
    "        self.train_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
    "        self.val_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
    "        self.test_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "\n",
    "    def _shared_step(self, batch):\n",
    "        features, true_labels = batch\n",
    "        logits = self(features)\n",
    "\n",
    "        loss = F.cross_entropy(logits, true_labels)\n",
    "        predicted_labels = torch.argmax(logits, dim=1)\n",
    "        return loss, true_labels, predicted_labels\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        loss, true_labels, predicted_labels = self._shared_step(batch)\n",
    "\n",
    "        self.log(\"train_loss\", loss)\n",
    "        self.train_acc(predicted_labels, true_labels)\n",
    "        self.log(\n",
    "            \"train_acc\", self.train_acc, prog_bar=True, on_epoch=True, on_step=False\n",
    "        )\n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        loss, true_labels, predicted_labels = self._shared_step(batch)\n",
    "\n",
    "        self.log(\"val_loss\", loss, prog_bar=True)\n",
    "        self.val_acc(predicted_labels, true_labels)\n",
    "        self.log(\"val_acc\", self.val_acc, prog_bar=True)\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        loss, true_labels, predicted_labels = self._shared_step(batch)\n",
    "        self.test_acc(predicted_labels, true_labels)\n",
    "        self.log(\"test_acc\", self.test_acc)\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        opt = torch.optim.SGD(self.parameters(), lr=self.learning_rate)\n",
    "        return opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aa7bcf14-8046-4213-accf-544b67e74883",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sebastian/miniforge3/envs/dl-fundamentals/lib/python3.9/site-packages/lightning/pytorch/utilities/seed.py:48: LightningDeprecationWarning: `lightning.pytorch.utilities.seed.seed_everything` has been deprecated in v1.8.0 and will be removed in v1.10.0. Please use `lightning.lite.utilities.seed.seed_everything` instead.\n",
      "  rank_zero_deprecation(\n",
      "Global seed set to 123\n"
     ]
    }
   ],
   "source": [
    "L.seed_everything(123)\n",
    "dm = CustomDataModule()\n",
    "\n",
    "pytorch_model = PyTorchMLP(num_features=100, num_classes=2)\n",
    "lightning_model = LightningModel(model=pytorch_model, learning_rate=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "91c7e6bc-a865-48c0-aa71-f8845af86270",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (mps), used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "`Trainer(overfit_batches=1)` was configured so 1 batch will be used.\n",
      "\n",
      "  | Name      | Type               | Params | In sizes | Out sizes\n",
      "------------------------------------------------------------------------\n",
      "0 | model     | PyTorchMLP         | 15.6 K | [5, 100] | [5, 2]   \n",
      "1 | train_acc | MulticlassAccuracy | 0      | ?        | ?        \n",
      "2 | val_acc   | MulticlassAccuracy | 0      | ?        | ?        \n",
      "3 | test_acc  | MulticlassAccuracy | 0      | ?        | ?        \n",
      "------------------------------------------------------------------------\n",
      "15.6 K    Trainable params\n",
      "0         Non-trainable params\n",
      "15.6 K    Total params\n",
      "0.062     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2ecaa600c1248eea4411e6bb58da8ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=10` reached.\n"
     ]
    }
   ],
   "source": [
    "%%capture --no-display\n",
    "\n",
    "\n",
    "trainer = L.Trainer(\n",
    "    overfit_batches=1,\n",
    "    max_epochs=10,\n",
    "    accelerator=\"cpu\",\n",
    "    devices=\"auto\",\n",
    "    deterministic=True,\n",
    ")\n",
    "\n",
    "trainer.fit(model=lightning_model, datamodule=dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86ee9ae1-9fef-4443-b1ec-4a4f1d86d59f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}