pytorch_nn_intro.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch_nn_intro.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MRqQ0IcigZXo"
      },
      "source": [
        "2021 Takahiro Shinozaki @ Tokyo Tech\n",
        "\n",
        "Quick introduction of neural networks\n",
        "\n",
        "References:\n",
        "\n",
        "    https://pytorch.org/docs/stable/nn.html#loss-functions\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0G6tLwfngN2A"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "# Install torchviz to visualize the network structure\n",
        "! pip install torchviz\n",
        "from torchviz import make_dot "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TlzbtIRBKhVf"
      },
      "source": [
        "# Define a linear layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4XlkPprgnRw"
      },
      "source": [
        "# Linear layer (ll)\n",
        "ll =  nn.Linear(2, 2)\n",
        "print('type(ll) =', type(ll))\n",
        "# parameters are randomly initialized\n",
        "print('weight= ', ll.weight)\n",
        "print('bias= ', ll.bias)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V0bUJeZZmc4V"
      },
      "source": [
        "Example affine transformation using the linear layer\n",
        "\n",
        "$\n",
        "y = \\mathrm{ll}(x) =\n",
        "\\left[\\begin{array}{cc}\n",
        "1 & 0 \\\\\n",
        "0 & 2 \\\\\n",
        "\\end{array}\\right]\n",
        "\\left[\\begin{array}{c}\n",
        "1 \\\\\n",
        "1 \\\\\n",
        "\\end{array}\\right]\n",
        "+\n",
        "\\left[\\begin{array}{c}\n",
        "10 \\\\\n",
        "-10 \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{c}\n",
        "11 \\\\\n",
        "-8 \\\\\n",
        "\\end{array}\\right]\n",
        "$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UW167zbDmJRg"
      },
      "source": [
        "# Explicitly setting the parameter values\n",
        "ll.weight = nn.Parameter(torch.tensor([[ 1.0, 0.0],[0.0, 2.0]]))\n",
        "print('weight= ', ll.weight)\n",
        "ll.bias = nn.Parameter(torch.tensor([10.0, -10.0]))\n",
        "print('bias= ', ll.bias)\n",
        "\n",
        "x = torch.tensor([1.0, 1.0])\n",
        "print('x= ', x)\n",
        "y = ll(x)\n",
        "print('y =', y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PyfmBhQJLOpm"
      },
      "source": [
        "# Introduce activation functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qnCODo0JjtMN"
      },
      "source": [
        "# an example input (two samples of three dimensinal vectors)\n",
        "x = torch.randn(2, 3)\n",
        "\n",
        "# sigmoid activation function\n",
        "h = nn.Sigmoid()\n",
        "print(type(h))\n",
        "# apply h to an input x\n",
        "print('sigmoid output =', h(x))\n",
        "\n",
        "# softmax activation function\n",
        "h = nn.Softmax(1) # 1 means 1-dimensional softmax\n",
        "print(type(h))\n",
        "# apply h to an input x\n",
        "print('sofmax output =', h(x))\n",
        "# check if the sum of the layer output is 1.0\n",
        "print('sums of layer outputs =', torch.sum(h(x), 1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "acBxR27aLcQh"
      },
      "source": [
        "# Define a neural network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pIsaO6jwFtoO"
      },
      "source": [
        "######## Advanced topic : __call__ method #############\n",
        "\n",
        "# nn.Sigmoid() etc. returns a class instance; e.g. h = nn.Sigmoid()\n",
        "# We use the returned instance as if it is a function; e.g. h(x)\n",
        "# If we write them in one line, it is: nn.Sigmoid()(x)\n",
        "# It is based on __call__ special method equipped in python\n",
        "\n",
        "# Example\n",
        "class callMethodTest():\n",
        "    def __init__(self):\n",
        "        self.coef = 2.5\n",
        "\n",
        "    # __call___ is a special method\n",
        "    # By defining this in a class, you can call the instance of this class something like as a function \n",
        "    def __call__(self, x):\n",
        "        return self.coef * x\n",
        "\n",
        "cmt = callMethodTest() # Make an instance\n",
        "y = cmt(5.0) # use the instance like a function.\n",
        "print(y)\n",
        "###############################################"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KQtUrMyTu6RN"
      },
      "source": [
        "# Define a neural network as a class\n",
        "class myNN(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(myNN, self).__init__()\n",
        "        self.layer1 = nn.Linear(3, 4) # input size = 3 and output size = 4\n",
        "        self.layer2 = nn.Linear(4, 2)\n",
        "\n",
        "    def forward(self, input):\n",
        "        l1out = nn.Sigmoid()(self.layer1(input))\n",
        "        l2out = nn.Softmax(1)(self.layer2(l1out))\n",
        "        return l2out\n",
        "\n",
        "# make a class instance\n",
        "model = myNN()\n",
        "\n",
        "print('model =', model)\n",
        "\n",
        "# obtain the model state as a dictionary\n",
        "params = model.state_dict()\n",
        "print('params =', params)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AFbLy5E_Mksd"
      },
      "source": [
        "# Use the network\n",
        "\n",
        "## Feed-forward\n",
        "\n",
        "The network parameters are not trained yet, so the output is random at this point"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pIFbu2fHeb1n"
      },
      "source": [
        "print(x)\n",
        "y = model(x)\n",
        "print(y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LAeybL4IPpLI"
      },
      "source": [
        "## Network strucure visualization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHQf7yjJOYId"
      },
      "source": [
        "print(dict(model.named_parameters()))\n",
        "make_dot(y, params=dict(model.named_parameters()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n_e0zYqaogbf"
      },
      "source": [
        "## Cross entropy loss calculation\n",
        "\n",
        "Cross entropy loss: uses cross entropy as a loss\n",
        "\n",
        "Cross entropy:\n",
        "$ H(p, q) = \\frac{1}{N} \\sum_{i=1}^N \\sum_{j=1}^C -p_{i,j} \\log (q_{i,j})$.\n",
        "\n",
        "Example:\n",
        "\n",
        "Consider two three-dimensional vectors \n",
        "(e.g. hidden layer outputs of a neural network for two input samples) \n",
        "$\n",
        "Z = \n",
        "\\left[\\begin{array}{c}\n",
        "Z_1 \\\\\n",
        "Z_2 \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{ccc}\n",
        "z_{1,1} & z_{1,2} & z_{1,3} \\\\\n",
        "z_{2,1} & z_{2,2} & z_{3,3} \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{ccc}\n",
        "1 & -1 & 2 \\\\\n",
        "-1 & 0 & 1 \\\\\n",
        "\\end{array}\\right]$.\n",
        "\n",
        "By applying softmax, we have a categorical distribution $P_i$ for each $Z_i, i \\in \\left\\{1,2\\right\\}$, where the number of categoreis is three. \n",
        "\n",
        "$Q = \n",
        "\\left[\\begin{array}{c}\n",
        "Q_1 \\\\\n",
        "Q_2 \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{ccc}\n",
        "q_{1,1} & q_{1,2} & q_{1,3} \\\\\n",
        "q_{2,1} & q_{2,2} & q_{3,3} \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\mathrm{softmax}(Z)\n",
        "=\n",
        "\\left[\\begin{array}{ccc}\n",
        "0.2594965 &  0.035119 &  0.7053845 \\\\\n",
        "0.0900306 &  0.2447285 &  0.665241 \\\\\n",
        "\\end{array}\\right]\n",
        "$.\n",
        "\n",
        "Assume that the reference distribution is $P$.\n",
        "\n",
        "$P = \n",
        "\\left[\\begin{array}{c}\n",
        "P_1 \\\\\n",
        "P_2 \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{ccc}\n",
        "p_{1,1} & p_{1,2} & p_{1,3} \\\\\n",
        "p_{2,1} & p_{2,2} & p_{3,3} \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{ccc}\n",
        "0.0 & 0.0 & 1.0 \\\\\n",
        "0.0 & 1.0 & 0.0 \\\\\n",
        "\\end{array}\\right]\n",
        "$.\n",
        "\n",
        "Then corss-entropy loss $L$ becomes:\n",
        "\n",
        "$L=\\frac{1}{2}\n",
        "  \\sum_{i=1}^2 \\sum_{j=1}^3 -p_{i,j} \\log (q_{i,j}) = \\frac{1}{2}\\left\\{-\\log (q_{1,3}) - \\log(q_{2,2})\\right\\} = \\frac{1}{2} \\left\\{ -\\log (0.7053845) - \\log(0.2447285) \\right\\}\n",
        "  = 0.878309\n",
        "$.\n",
        "\n",
        "When the reference is given by indexes of correct catetories\n",
        "$C=\n",
        "\\left[\\begin{array}{c}\n",
        "c_1 \\\\\n",
        "c_2 \\\\\n",
        "\\end{array}\\right]\n",
        "=\n",
        "\\left[\\begin{array}{c}\n",
        "3 \\\\\n",
        "2 \\\\\n",
        "\\end{array}\\right]$,\n",
        "cross-entropy is obtained by:\n",
        "$L=\\frac{1}{2}\n",
        "  \\sum_{i=1}^2 -p_{i,c_i} \\log (q_{i,c_i}) = \\frac{1}{2}\\left\\{-\\log (q_{1,3}) - \\log(q_{2,2})\\right\\} = 0.878309\n",
        "$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WsexB-pCoe3S"
      },
      "source": [
        "z = torch.tensor([[1, -1, 2], [-1, 0, 1]], dtype=torch.float32)\n",
        "print(z)\n",
        "q = nn.Softmax(1)(z)\n",
        "print(q)\n",
        "c = torch.tensor([2, 1]) # index starts from 0. therefore, the second entroy is 1 and the third entory is 2.\n",
        "print(c)\n",
        "loss = nn.NLLLoss()\n",
        "loss(torch.log(q),c)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6kmq2FiEhh80"
      },
      "source": [
        "########## Implementation details : nn.CrossEntropyLoss #############\n",
        "# Pytorch's nn.CrossEntropyLosss applies softmax internally.\n",
        "# Therefore, if you use nn.CrossEntropyLoss to obtain cross entropy loss, \n",
        "# you have to fed the output of your neural network before applying softmax.\n",
        "loss = nn.CrossEntropyLoss()\n",
        "loss(z, c)\n",
        "#####################################################################"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XBTl9_JCNcoR"
      },
      "source": [
        "## Back-propagation\n",
        "\n",
        "Obtain gradients of parameters a neural network to minimize cross entropy loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YeNcfWfdsz9G"
      },
      "source": [
        "# an example input (two samples of three dimensinal vectors)\n",
        "x = torch.randn(2, 3)\n",
        "## if you want gradients of the input in addition to the gradient of network parameters:\n",
        "## x = torch.randn(2, 3, requires_grad=True)\n",
        "\n",
        "# an example reference\n",
        "c = torch.tensor([1, 0])\n",
        "\n",
        "# my defined neural network\n",
        "model = myNN()\n",
        "\n",
        "# initialize gradient\n",
        "model.zero_grad()\n",
        "\n",
        "# obtain cross entropy loss\n",
        "loss = nn.NLLLoss()\n",
        "celoss = loss(torch.log(model(x)), c)\n",
        "\n",
        "# apply back-propagation\n",
        "celoss.backward() \n",
        "\n",
        "# obtained gradients\n",
        "print(model.layer1.weight.grad)\n",
        "print(model.layer1.bias.grad)\n",
        "print(x.grad)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}