2-activation-functions.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2-activation-functions.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP/UKtiFfCCTZmFHuusnkoO",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ymoslem/PyTorchNLP/blob/main/2-activation-functions.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SseRC94irgME"
      },
      "source": [
        "# Sigmoid activation function\n",
        "\n",
        "## $f{(x)}=\\frac1{1+e^{-x}}$\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q4NeQCae6gEZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "d0d82d66-2514-4409-e9d1-e2caaa024248"
      },
      "source": [
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "x = torch.arange(-5.0, 5.0, 0.1)\n",
        "y = torch.sigmoid(x)\n",
        "\n",
        "plt.plot(x.numpy(), y.numpy())\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfzklEQVR4nO3deXydZZ338c8v+54uSbolaVqaLilla2hTGBZlKwVhXAYolEWW6ktRfAZBFh/GQccRcUQeB8VSlL3IIkzFSgVECtJCU7pvaZouSbdszb6e5Hr+SGBCbclpe5L7LN/36+UrOefczfkek3y5cp3rvm5zziEiIqEvyusAIiISGCp0EZEwoUIXEQkTKnQRkTChQhcRCRMxXj1xRkaGy8vL8+rpRURC0qpVq6qdc5mHe8yzQs/Ly6O4uNirpxcRCUlmtutIj2nKRUQkTKjQRUTChApdRCRMqNBFRMJEv4VuZr81s0oz23CEx83M/p+ZlZrZOjM7LfAxRUSkP/6M0J8AZn/G4xcD+b3/mw/8+vhjiYjI0eq30J1zy4DazzjkcuAp12MFMMTMRgUqoIiI+CcQ69DHAOV9blf03rfv0APNbD49o3hyc3MD8NQiIsGju9vR2O6jobWTxjYfTe0+Gts6aWrv+by53UdTexfnTc7i5JwhAX/+QT2xyDm3AFgAUFhYqI3YRSRoOedoaPVR1dROdVM7NU0d1DT3fDzY0sHBlk7qWno+r2/tpK6lp7j9ucREVmp80Bb6HiCnz+3s3vtERIKSc47qpg721LWy52Ar++pb2Vffxv76NvY3tHGgoY3KxnY6fN2H/ffpibEMTYplaHIcmSnx5Gelkp4YS1piLGkJMZ98TE2IJSU+huT4GFITej4mxUYTFWUD8roCUeiLgVvN7HlgJlDvnPuH6RYRkcHU3e3YW9/KjupmdlY3U1bdzO6aFnbXtlB+sIW2zk+XdWJsNKOGJDAiNYHCsUMZkZZAZmo8manxZKTEMzwljuHJ8QxNiiUmOjhXfPdb6Ga2CDgXyDCzCuDfgFgA59yjwBJgDlAKtABfHaiwIiKHcs5R2djO5n0NbNnfSMn+RrZVNlFa2URrZ9cnxyXFRZM7LIlxGcmcMzGT7KGJjBmaxJghiYwZkkhaYgxmAzNyHiz9Frpzbm4/jzvgmwFLJCLyGSob2lhdXsfa8jo27G1g0956qps6Pnl8VHoCE7JSuGpGDhOyUjghM4VxGclkpcaHfGH3x7PdFkVE+tPd7diyv5HiXbWs3HmQVTtr2VvfBkBMlJE/IpXPTcpi6ug0poxKY/LINNKTYj1O7R0VuogEDeccO6qbeXdbNcu317BiRw11LZ0AjExLoDBvKDflDuWUnHSmjk4nITba48TBRYUuIp5q6+xi+fYa/rqlkr+VVFJe2wrAmCGJXDBlBLNOGM7pecPIHpoY9lMmx0uFLiKDrqndx1ubD/D6hv28U1JFS0cXSXHRnHFCBvPPPoGz8zMYOzzZ65ghR4UuIoOirbOLv26pZPGavby9tZJ2XzdZqfH886ljuKBgBGecMJz4GE2hHA8VuogMGOcca8rreHFVBa+t3UtDm4/M1HjmzsjlkpNGMT136ICdZBOJVOgiEnCNbZ28umYvz32wm837GkiMjWb2iSP50mljOOOEDKJV4gNChS4iAbOrppkn3t/Ji8UVNLX7mDo6jf/44olcdvJoUhMidznhYFGhi8hxW1Nex6/eLuWNzQeIiTIuPWk0180ayyk5Q7QyZRCp0EXkmC3fXsMjb5fyXmk16YmxfPPcCVw7aywj0hK8jhaRVOgictQ+2n2Qny3dyvvba8hMjeeeOZO5euZYUuJVKV7S//si4rdtBxr5yZ+38NaWSjJS4rjv0gKunpmrMzaDhApdRPpV09TOL97cxnMf7iYpLpo7LprEDWfkkawReVDRd0NEjqir2/HMil387C9baeno4pqZudx2Xj7DU+K9jiaHoUIXkcNaU17H919dz4Y9DfzThAz+7QsF5I9I9TqWfAYVuoh8SkuHjweXbuWJ93eSmRLPL+eeyqUnjdLywxCgQheRTyzfXsP3Xl7H7toW5hXl8r3Zk3VCUAhRoYsI7b4uHnx9Kwvf20HusCQW3VLErBOGex1LjpIKXSTCbTvQyLcWrWbL/kauLRrL3XMmkxSnaghF+q6JRCjnHL9fWc6/Ld5ISnwMj19fyHlTRngdS46DCl0kArV2dPH9Vzfw8kcV/NOEDH5+5clkpep0/VCnQheJMDuqm/n606soqWzktvPy+fZ5+drONkyo0EUiyLvbqvjmsx8RHWU88dUZnDMx0+tIEkAqdJEI4Jzjifd38qM/bWZCZgoLry8kZ1iS17EkwFToImHO19XND/64kWdW7OaCghE8dOUp2hUxTOm7KhLGWjp8fHvRat7cXMnXzhnP9y6arGt4hjEVukiYqm5q56Yni1lfUccPL5/KtbPyvI4kA0yFLhKG9ta1Mm/hB+ytb+XRedO5cOpIryPJIFChi4SZHdXNzFv4AQ2tnTx900xOzxvmdSQZJCp0kTCyZX8D8xZ+SLdzLJpfxIlj0r2OJINIhS4SJrbsb+Dqxz4gNtp4/uYiJmRp7/JIE+XPQWY228y2mlmpmd11mMdzzextM1ttZuvMbE7go4rIkXxc5nHRUfx+/iyVeYTqt9DNLBp4BLgYKADmmlnBIYd9H3jBOXcqcBXwq0AHFZHD27q/8ZOR+aL5ReRlJHsdSTzizwh9BlDqnCtzznUAzwOXH3KMA9J6P08H9gYuoogcyY7qZq5Z2DvNMn8W41TmEc2fQh8DlPe5XdF7X18/AOaZWQWwBPjW4b6Qmc03s2IzK66qqjqGuCLysY+XJnY7x7M3F6nMxb85dD/MBZ5wzmUDc4CnzewfvrZzboFzrtA5V5iZqU2BRI5VdVM78x7vWZr41I0zmJCV4nUkCQL+FPoeIKfP7eze+/q6CXgBwDm3HEgAMgIRUEQ+randxw2/+5C9da08fsPpWpoon/Cn0FcC+WY2zszi6HnTc/Ehx+wGzgMwsyn0FLrmVEQCrLOrm288+xGb9zXyq2tOY8Y4nTQk/6vfQnfO+YBbgaXAZnpWs2w0s/vN7LLew24HbjGztcAi4AbnnBuo0CKRyDnHPX9Yz7KSKn78xRP5/GRdLk4+za8Ti5xzS+h5s7Pvfff1+XwTcGZgo4lIX794cxsvrqrgtvPyufL0XK/jSBAK1JuiIjKAXl29h4ff2sYVhdl85/x8r+NIkFKhiwS5VbsOcufL6ygaP4wf/fM0zLSfuRyeCl0kiFUcbOFrTxczOj2BX18znbgY/crKkWlzLpEg1dzu4+Yni2n3dfP8/NMZmhzndSQJcvrPvUgQcs5x50vrKDnQyCNXn6YTh8QvKnSRIPSbZWX8af0+vjd7MmdP1FnV4h8VukiQWVZSxU9f38KlJ41i/tnjvY4jIUSFLhJEymtb+Nai1UwckcpPv3KSVrTIUVGhiwSJdl8Xtz73Ed3djkfnTScpTmsW5OjoJ0YkSPz4T5tZW1HPo/Om6yIVckw0QhcJAn9cu5cnl+/i5n8ax+wTR3odR0KUCl3EYzuqm7nr5XVMHzuU71082es4EsJU6CIeavd18a1FHxEbE8Uv555KbLR+JeXYaQ5dxEM/fX0rG/Y0sODa6Ywekuh1HAlxGg6IeOSvWw7w+Hs7uH7WWC6cqnlzOX4qdBEPVDa08d0X1zFlVBp3z5nidRwJEyp0kUHmnOO7L62jpcPHL+eeQkJstNeRJEyo0EUG2VPLd7GspIp7LylgQlaq13EkjKjQRQbRtgON/HjJZj43KZN5M3UZOQksFbrIIOnwdXPb82tIiY/hp185Wfu0SMBp2aLIIHn4rRI27Wtg4XWFZKbGex1HwpBG6CKDYPXug/z6b9u5ojCb8wtGeB1HwpQKXWSAtXZ0cfsLaxmVnsj/vbTA6zgSxjTlIjLAfrp0C2XVzTx380xSE2K9jiNhTCN0kQG0oqyG3/19JzeckccZEzK8jiNhToUuMkBaOnzc+dI6xg5P4s7Zk7yOIxFAUy4iA+TBpVvZXdvC8/OLdPUhGRQaoYsMgOKdtTzx/k6unzWWovHDvY4jEUKFLhJgbZ1d3PHSOrKHJnLnbF2wQgaP/g4UCbCH3ihhR++qluR4/YrJ4NEIXSSA1lfU89i7ZVx1eo5Wtcig86vQzWy2mW01s1Izu+sIx1xhZpvMbKOZPRfYmCLBr7OrmztfXkdGSrz2OBdP9Pv3oJlFA48AFwAVwEozW+yc29TnmHzgbuBM59xBM8saqMAiwWrBsjI272vgN9dOJz1RJxDJ4PNnhD4DKHXOlTnnOoDngcsPOeYW4BHn3EEA51xlYGOKBLftVU08/NY25kwbyUW6nJx4xJ9CHwOU97ld0XtfXxOBiWb2dzNbYWazD/eFzGy+mRWbWXFVVdWxJRYJMt3djrv/sJ6EmCh+cNlUr+NIBAvUm6IxQD5wLjAXeMzMhhx6kHNugXOu0DlXmJmZGaCnFvHWi6vK+XBHLffMmUJWaoLXcSSC+VPoe4CcPreze+/rqwJY7JzrdM7tAEroKXiRsFbV2M5//GkzM8YN44rCnP7/gcgA8qfQVwL5ZjbOzOKAq4DFhxzzKj2jc8wsg54pmLIA5hQJSve/tom2zm5+/MVpREXpCkTirX4L3TnnA24FlgKbgReccxvN7H4zu6z3sKVAjZltAt4G7nDO1QxUaJFg8Letlfxx7V6++bkJTMhK8TqOCOac8+SJCwsLXXFxsSfPLXK8Wju6uOChd4iPiWLJbWcRHxPtdSSJEGa2yjlXeLjHdF6yyDF4+K1tVBxs5ffzi1TmEjR06r/IUdqyv4GF75ZxRWE2M7WTogQRFbrIUejudtzzh/WkJcZy98U6vV+Ciwpd5CgsWrmbj3bXce+cKQxNjvM6jsinqNBF/FTV2M4Df97CrPHD+dJph54sLeI9FbqIn368ZDNtnd386IsnYqY15xJ8VOgifnh/ezWvrN7D188ZzwmZWnMuwUmFLtKPdl8X3391A7nDkvjG5yZ4HUfkiLQOXaQfC94po6yqmSe+ejoJsVpzLsFLI3SRz7Crpplfvl3KJdNGce4kXbdFgpsKXeQInHPc9z8biYuO4r4vFHgdR6RfKnSRI1iyfj/vlFTxrxdMZESa9jmX4KdCFzmMxrZO7n9tIwWj0rhu1liv44j4RW+KihzGQ29so7KxnUfnTScmWuMeCQ36SRU5xIY99Tzx/g7mzsjl1NyhXscR8ZsKXaSP7m7H91/dwNCkOL530WSv44gcFRW6SB/PryxnTXkd914yhfSkWK/jiBwVFbpIr+qmdh54fQtF44fxxVO1+ZaEHhW6SK//XLKFlg4fP/pnbb4loUmFLgIs317Dyx9VcMtZ45mQlep1HJFjokKXiNfh6+b7r64nZ1gi3/p8vtdxRI6Z1qFLxHvs3TK2VzXzuxtOJzFOm29J6NIIXSLa7poW/t9b25gzbSSfm6zNtyS0qdAlYjnnuG/xBmKijPsunep1HJHjpkKXiLVk/X7+trWKf71wEiPTtfmWhD4VukSkhrZOfvDHjZw4Jo3rtfmWhAm9KSoR6cHXt1LT1M7j1xdq8y0JG/pJloizevdBnvlgF9fNyuOk7CFexxEJGBW6RJTOrm7ueWUDWanx3H7hRK/jiASUplwkojz+3g4272vg19ecRmqCNt+S8KIRukSM3TUt/OLNEi4oGMHsE0d6HUck4PwqdDObbWZbzazUzO76jOO+bGbOzAoDF1Hk+DnnuPfV9USbcf/lU7X5loSlfgvdzKKBR4CLgQJgrpn9wyXQzSwVuA34INAhRY7X/6zZy7vbqrlz9mRGpSd6HUdkQPgzQp8BlDrnypxzHcDzwOWHOe6HwANAWwDziRy32uYOfvjaJk7JGcK8Iq05l/DlT6GPAcr73K7ove8TZnYakOOc+9NnfSEzm29mxWZWXFVVddRhRY7FD1/bRH1rJz/58jSiozTVIuHruN8UNbMo4OfA7f0d65xb4JwrdM4VZmZmHu9Ti/Trb1sreWX1Hr5x7glMHpnmdRyRAeVPoe8Bcvrczu6972OpwInA38xsJ1AELNYbo+K1pnYf976ygQlZKXzz8xO8jiMy4Pwp9JVAvpmNM7M44Cpg8ccPOufqnXMZzrk851wesAK4zDlXPCCJRfz0s6Vb2VvfygNfnkZ8jPY5l/DXb6E753zArcBSYDPwgnNuo5ndb2aXDXRAkWNRvLOWJ5fv5LqisUwfO8zrOCKDwq8zRZ1zS4Alh9x33xGOPff4Y4kcu7bOLu58aR2j0xO5c/Zkr+OIDBqd+i9h5+dvlFBW3cyzN88kOV4/4hI5dOq/hJWPdh9k4btlzJ2Ry5kTMryOIzKoVOgSNj6eahmZlsA9czTVIpFHf49K2HjojRJKK5t48sYZ2klRIpJG6BIWinfWsqB3quWciTppTSKTCl1CXkuHj9tfXEv20ETuvWSK13FEPKMpFwl5P/nzFnbXtrDoliJStKpFIphG6BLSlpVU8dTyXdx45jiKxg/3Oo6Ip1ToErIONnfw3RfXkp+Vwh0XTfI6jojn9PephCTnHHf/YT0HWzr43VdPJyFWe7WIaIQuIenFVRW8vnE/371wElNHp3sdRyQoqNAl5OyqaebfF2+kaPwwbj5rvNdxRIKGCl1CSoevm28vWk10lPFfV5yiKxCJ9KE5dAkp//WXraytqOfX15zGmCG62LNIXxqhS8h4p6SK3ywr4+qZuVw8bZTXcUSCjgpdQkJlYxu3v7CGSSNSue/SAq/jiAQlTblI0Ovqdty2aA1N7T6eu6VISxRFjkCFLkHvoTdKWF5Ww4NfOYmJI1K9jiMStDTlIkHt7a2V/PfbpVxRmM2/FOZ4HUckqKnQJWjtqWvl//x+DZNHpnL/5Sd6HUck6KnQJSi1dXbx9adX4ety/HredM2bi/hBc+gSdJxz3PvKBtbvqeex6woZl5HsdSSRkKARugSdp5bv4uWPKvjO+flcUDDC6zgiIUOFLkHlg7IafvjaJs6fMoJvfz7f6zgiIUWFLkFjV00zX39mFbnDk/j5lScTpX1aRI6KCl2CQkNbJzc9WUy3g8evP520hFivI4mEHBW6eM7X1c2tz61mZ3Uzj86brjdBRY6RVrmIp5xz3P/aJpaVVPGTL01j1gm6LqjIsdIIXTz16DtlPLV8F/PPHs9VM3K9jiMS0lTo4plXV+/hgde38IWTR3PX7MlexxEJeSp08cTfS6u546W1FI0fxs/+5SStaBEJAL8K3cxmm9lWMys1s7sO8/i/mtkmM1tnZm+Z2djAR5Vwsaa8jvlPFTM+I4XfXFtIfIxO6xcJhH4L3cyigUeAi4ECYK6ZHXqFgdVAoXPuJOAl4KeBDirhYev+Rm743YcMT4nnqZtmkJ6o5YkigeLPCH0GUOqcK3POdQDPA5f3PcA597ZzrqX35gogO7AxJRzsqmlm3uMfEBcdxbM3z2REWoLXkUTCij+FPgYo73O7ove+I7kJ+PPhHjCz+WZWbGbFVVVV/qeUkFde28LVj31AZ1c3z9w8k5xhSV5HEgk7AX1T1MzmAYXAg4d73Dm3wDlX6JwrzMzMDORTSxCrONjC3MdW0NjWydM3ztRVh0QGiD8nFu0B+l4qJrv3vk8xs/OBe4FznHPtgYknoa7iYAtXLVhBQ2snz95cxLTsdK8jiYQtf0boK4F8MxtnZnHAVcDivgeY2anAb4DLnHOVgY8poWhXTfMnZf7MzTNV5iIDrN8RunPOZ2a3AkuBaOC3zrmNZnY/UOycW0zPFEsK8KKZAex2zl02gLklyG3d38i1j/fMmWtkLjI4/NrLxTm3BFhyyH339fn8/ADnkhC2tryO63/3IfExUbzwtVnka85cZFBocy4JqHdKqvjGM6sYlhLHszcVkTtcq1lEBotO/ZeAeWFlOTc+sZLc4cm89PUzVOYig0wjdDluzjkefmsbv3hzG2flZ/Cra04jVReoEBl0KnQ5Lq0dXdzx0lpeW7ePr0zP5j+/NI3YaP3hJ+IFFbocs711rcx/upiNexu46+LJfO3s8fSuchIRD6jQ5ZisKKvh1udW09bZxcLrCjlvygivI4lEPBW6HJXubsdvlpXx4NIt5A1P5rlbdCq/SLBQoYvfDjZ3cMdLa3lzcyWXnDSKB758Einx+hESCRb6bRS/vLetmttfXENtcwc/+EIB15+Rp/lykSCjQpfP1NbZxc+WbmXhezuYkJXCb284namjdRq/SDBSocsRrdp1kDtfWsv2qmauLRrLPXOmkBiny8WJBCsVuvyDlg4fP/9LCY//fQej0xN56sYZnD1R+9eLBDsVunzKXzbu59//uIk9da1cMzOXuy6erLM+RUKECl2Anr3Lf/jaJt7cXMmkEam88LVZzBg3zOtYInIUVOgRrr6lk1/+dRtPLt9JbHQU986Zwg1n5un0fZEQpEKPUG2dXTyzYhePvF1KXWsnV0zP4fYLJ5KVluB1NBE5Rir0CNPh6+aF4nJ++ddtHGho56z8DO6+eAoFo9O8jiYix0mFHiFaO7p4fuVuFiwrY199G4Vjh/LwVadSNH6419FEJEBU6GGupqmdZz/YzZPv76SmuYMZecP4zy9N45yJmTrTUyTMqNDD1Ka9DTz5/k5eWbOHDl83507K5BvnTtDKFZEwpkIPI60dXfxx3V6e+2A3a8rrSIiN4orCbG44YxwTslK8jiciA0yFHuK6ux0f7qzl5VUV/HnDfprafUzISuG+Swv40mljGJIU53VEERkkKvQQ5JxjbUU9f1q3lyXr97OnrpXkuGjmTBvFV6ZnM2PcMM2Pi0QgFXqI6Ozq5sMdtbyx6QBvbDrAnrpWYqONs/MzueOiSVw0daQ2zhKJcCr0ILa3rpVlJVW8U1LFe6XVNLb5iI+J4qz8DG47P5+LCkaSnqR9VkSkhwo9iOyvb2PlzlqWl9WwfHsNO6qbARiVnsCcE0fx+SlZnJWfQVKcvm0i8o/UDB7p8HWzZX8Da8rrWL27juJdtZTXtgKQGh/DjHHDuGZmLmdPzCQ/K0Vz4iLSLxX6IGhq97F1fyNb9jewYU8DG/fWs2VfIx1d3QBkpMRTOHYo18/K4/S8YUwdnUaMNscSkaOkQg8Q5xy1zR3sqG6mrKqZ0qomSiub2FbZ+MnIGyA9MZapo9O44cw8Ts4ewsk56YwZkqgRuIgcNxX6UWhu97G3rpWKulb2HGyl4mAr5bUt7K5tYVdNMw1tvk+OjYuOYnxmMidnD+HKwhwmj0xj0shUsoeqvEVkYER8oXd3O+pbO6lp7qCmqZ3qpg6qGtuoamrnQEM7BxraONDQxr76Nhr7FDZAbLSRMzSJnGFJnJIzhLyMZMZnJJOXkUzO0ERNm4jIoPKr0M1sNvAwEA0sdM795JDH44GngOlADXClc25nYKMennOOdl83Te0+mtt9NLb5aGr30dTmo6Gtk8Y2Hw2tndS3dlL38ceWDg62/O/Hrm73D183OsrISo0nKzWescOTmTV+OCPTExk9JIExQxIZMzSRrNQEoqM02haR4NBvoZtZNPAIcAFQAaw0s8XOuU19DrsJOOicm2BmVwEPAFcOROAXVpbz6LLttLR30dzho6Wj67CFfKikuGjSE2NJT4xlSFIs+VkpDEmKY3hyHMOS4xieEsfw5HgyUuPISIlnWFIcUSprEQkh/ozQZwClzrkyADN7Hrgc6FvolwM/6P38JeC/zcycc/037VEamhxHwag0kuKiSYqLISkumuT4GFLiY0iOjyE1IYbU+BhSEmJIS4glLTGWlPgY4mI0/SEi4c2fQh8DlPe5XQHMPNIxzjmfmdUDw4HqvgeZ2XxgPkBubu4xBb6gYAQXFIw4pn8rIhLOBnXY6pxb4JwrdM4VZmZmDuZTi4iEPX8KfQ+Q0+d2du99hz3GzGKAdHreHBURkUHiT6GvBPLNbJyZxQFXAYsPOWYxcH3v518B/joQ8+ciInJk/c6h986J3wospWfZ4m+dcxvN7H6g2Dm3GHgceNrMSoFaekpfREQGkV/r0J1zS4Alh9x3X5/P24B/CWw0ERE5GlrLJyISJlToIiJhQoUuIhImzKvFKGZWBezy5MmPTwaHnDAVISLxdes1R45Qet1jnXOHPZHHs0IPVWZW7Jwr9DrHYIvE163XHDnC5XVrykVEJEyo0EVEwoQK/egt8DqARyLxdes1R46weN2aQxcRCRMaoYuIhAkVuohImFChHwczu93MnJlleJ1loJnZg2a2xczWmdkrZjbE60wDycxmm9lWMys1s7u8zjPQzCzHzN42s01mttHMbvM602Axs2gzW21mr3md5Xip0I+RmeUAFwK7vc4ySN4ATnTOnQSUAHd7nGfA9LmO7sVAATDXzAq8TTXgfMDtzrkCoAj4ZgS85o/dBmz2OkQgqNCP3UPAnUBEvKvsnPuLc87Xe3MFPRc6CVefXEfXOdcBfHwd3bDlnNvnnPuo9/NGegpujLepBp6ZZQOXAAu9zhIIKvRjYGaXA3ucc2u9zuKRG4E/ex1iAB3uOrphX24fM7M84FTgA2+TDIpf0DMw6/Y6SCD4tR96JDKzN4GRh3noXuAeeqZbwspnvWbn3P/0HnMvPX+ePzuY2WRwmFkK8DLwHedcg9d5BpKZXQpUOudWmdm5XucJBBX6ETjnzj/c/WY2DRgHrDUz6Jl6+MjMZjjn9g9ixIA70mv+mJndAFwKnBfmlxj05zq6YcfMYukp82edc3/wOs8gOBO4zMzmAAlAmpk945yb53GuY6YTi46Tme0ECp1zobJT2zExs9nAz4FznHNVXucZSL0XOi8BzqOnyFcCVzvnNnoabABZz+jkSaDWOfcdr/MMtt4R+nedc5d6neV4aA5d/PXfQCrwhpmtMbNHvQ40UHrf/P34OrqbgRfCucx7nQlcC3y+9/u7pnfkKiFEI3QRkTChEbqISJhQoYuIhAkVuohImFChi4iECRW6iEiYUKGLiIQJFbqISJj4/0tmGO21nq61AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZriAARjoxIJP"
      },
      "source": [
        "# Tanh activation function\n",
        "\n",
        "## $f{(x)=\\text{tanh}\\text{ }x}=\\frac{e^x-e^{-x}}{e^x+e^{-x}}$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "9NB8sA3jxMpV",
        "outputId": "dd5dfada-da5c-467e-95b0-e92597a27c8f"
      },
      "source": [
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "x = torch.arange(-5.0, 5.0, 0.1)\n",
        "y = torch.tanh(x)\n",
        "\n",
        "plt.plot(x.numpy(), y.numpy())\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZRcdZ338fe3qrcknb07a3c2EglhC9CGTYYtgSgzhBlB0RGDwmT0yDiOisI4j/M8OHrw8Yz4eIZRchCJsglRTJRg2FUGCOlASNKBkE7I0p3uTmfvpPeq7/NH3cai6U5CqrpvV9XndU5R9/7u71Z9iyT1qbv+zN0REZHcFQm7ABERCZeCQEQkxykIRERynIJARCTHKQhERHJcXtgFnIiSkhKfMmVK2GWIiGSUNWvW7HH30u7tGRkEU6ZMobKyMuwyREQyiplt76ldu4ZERHKcgkBEJMcpCEREcpyCQEQkxykIRERyXFqCwMzuM7PdZrahl+VmZj82s2ozW2dmZyctW2hmm4PHwnTUIyIixy9dWwT3A/OPsvyjwIzgsQj4CYCZjQL+HTgXmAP8u5mNTFNNIiJyHNJyHYG7/8nMphylywLgF5645/UrZjbCzMYDlwBPu/s+ADN7mkSgPJyOukQk+3XG4rR1xmnvTDx3xOK0xxLPHZ1ORzxOZ8zpjMXpjDux4NEZd+KemO56jsUdd4i7Ew+ePZh+95nEtDs4Xc+8Z75LVz9I9CHo9+5yPKlvcnuSbkMFLLxgCqOLC9Pwf+4v+uuCsonAzqT5mqCtt/b3MbNFJLYmmDRpUt9UKSL9KhZ39h5po7Gpjf1HOtjX3M7+I+0cbOngUEsHB1s6ONzW+e6juS1Gc0cnLe0xWjvitHbE6Ixn/5gqZn+Zvnr2xIwNgpS5+2JgMUBFRUX2/8mLZIFY3Nm5r5lte4+wfW8z2/c2s+tAC3UHW9h1sJW9h9vo7Xt8cEGUYUX5FBflUVyYeJQWFzK4IMqggjwG5Ucpyo9QlB+lIC9CYV6Ewrwo+VGjIC9CQTRCfjRCXtTIi3Q9G9HuDzMiwXM0kpiOGBhGJAIRM4zgOWi3CBhgwbJ324Mv7K75v0x3tQdtSZ8z+Uvekmf6UX8FQS1QnjRfFrTVktg9lNz+Qj/VJCJp1NzeyYbaQ6yrOUDVrkNsqm+iuvEw7Z3xd/sMyo8yceQgxg8vYua4YYwdVkjp0EJKigsZXVzIqCH5jBhcwPBB+eRHdVJjf+mvIFgO3GJmj5A4MHzQ3evMbCXwvaQDxFcAt/dTTSKSgsNtnazaupdXtu7lla37qNp18N1f9+OGFTFz/FA+MqOE6WOKmVYyhEmjB1NaXBjar17pXVqCwMweJvHLvsTMakicCZQP4O4/BVYAHwOqgWbgc8GyfWb2HWB18FJ3dB04FpGBp+FQKyur6nl6YwOrtu6jPRanIC/C2ZNGcMul0zmzfARnlI2gdGh692FL37JMHLy+oqLCdfdRkf7R0h5jZVU9v36thher9+AOU0uGMPeUMVw6cwxnTxpJUX407DLlOJjZGnev6N6eMQeLRaR/1R1sYclL23n41R0cbOmgbOQg/unS6Vw9ewLTxwwNuzxJIwWBiLzHzn3N3PXM2yxfu4u4O/NPG8dnz5/CnCmjiES0fz8bKQhEBIDGpjb+67nNPPTqDiJmfPb8KXzuwimUjxocdmnSxxQEIjkuHnceenUH33/yLZo7Ynzyw+V8+bIZjBteFHZp0k8UBCI5rHp3E7f9ej2V2/dz4fTRfGfBaUwrLQ67LOlnCgKRHOTu/Gr1Tr69vIrBBVF+cO0ZXHtOmc7xz1EKApEc09zeyb/9dgO/ea2Wj0wv4a5PztZ5/zlOQSCSQ3YdaOFzP1/N27ub+MrcGfzTZTOI6kygnKcgEMkRbzc08dmfvcqRtk6WfG4Of/Wh0rBLkgFCQSCSA1Zv28dN96+mKD/Ko184n1PGDwu7JBlAFAQiWe6VrXtZeN+rTBw5iF98fg5lI3VdgLyXgkAki72x8wA33b+aSaMG88ii89I+oIlkB93wWyRLvd3QxMKfv8qo4gJ+edO5CgHplYJAJAvtOtDCZ+5dRUE0woM3naerhOWotGtIJMu0dsT4wgNraG6PsfSL5zNptI4JyNEpCESyiLvzb7/dwLqagyy+4RxmjtPZQXJsadk1ZGbzzWyTmVWb2W09LL/LzNYGj7fN7EDSsljSsuXpqEckV/3i5e0sXVPDly+fwRWnjgu7HMkQKW8RmFkUuBuYB9QAq81subtv7Orj7v+S1P+fgLOSXqLF3WenWodIrnt9x36+8/uNXD5zDF+5fEbY5UgGSccWwRyg2t23uns78Aiw4Cj9PwU8nIb3FZFAS3uMrz76BmOHFfHDT87WADLygaQjCCYCO5Pma4K29zGzycBU4Lmk5iIzqzSzV8zsmt7exMwWBf0qGxsb01C2SPa488k3eWfPEX5w3RkMH5QfdjmSYfr79NHrgaXuHktqmxwMpvxp4EdmdlJPK7r7YnevcPeK0lLdI0Wky4ub97Dk5e18/sKpXHBSSdjlSAZKRxDUAuVJ82VBW0+up9tuIXevDZ63Ai/w3uMHInIUB1s6uHXpG5xUOoRvzD857HIkQ6UjCFYDM8xsqpkVkPiyf9/ZP2Y2ExgJvJzUNtLMCoPpEuBCYGP3dUWkZ//51CYaDrXyw0/Mpig/GnY5kqFSPmvI3TvN7BZgJRAF7nP3KjO7A6h0965QuB54xN09afVTgHvMLE4ilO5MPttIRHpXtesgD7yynRvOm8yZ5SPCLkcymL33ezkzVFRUeGVlZdhliITG3bnupy/zzp4jPPf1S3SAWI6Lma0Jjsm+h+41JJKBHn+9lsrt+/nm/JkKAUmZgkAkwzS1dvC9FW9xZvkIrj2nLOxyJAvoXkMiGeanf9zCnsNt/GxhhS4ck7TQFoFIBmlsauO+F7fxN2dO0AFiSRsFgUgGufv5atpjcf5lru4lJOmjIBDJELUHWnho1Q6uPbuMaaXFYZcjWURBIJIhfvzMZgC+rK0BSTMFgUgGeGfPEZa+VsOnz53ExBGDwi5HsoyCQCQD3P18NflR40uXTg+7FMlCCgKRAa7uYAvL1tbyyYpySocWhl2OZCEFgcgAd9+L7xB3uPmiaWGXIllKQSAygB1s6eChVTu46vTxlI8aHHY5kqUUBCID2IOrtnOkPcY/XqytAek7CgKRAaq1I8Z9L27johklnDpheNjlSBZTEIgMUI+/Xsuew2184eIeR28VSZu0BIGZzTezTWZWbWa39bD8RjNrNLO1wePmpGULzWxz8FiYjnpEMp27s+SlbcwaP4wLThoddjmS5VK++6iZRYG7gXlADbDazJb3MNLYr9z9lm7rjgL+HagAHFgTrLs/1bpEMlnl9v28Vd/EnX93Oma6w6j0rXRsEcwBqt19q7u3A48AC45z3SuBp919X/Dl/zQwPw01iWS0X768naFFeVw9e0LYpUgOSEcQTAR2Js3XBG3dfdzM1pnZUjMr/4DriuSMxqY2ntxQx7XnlDG4QEOGSN/rr4PFvwOmuPsZJH71L/mgL2Bmi8ys0swqGxsb016gyEDxaOVOOmLOZ86bHHYpkiPSEQS1QHnSfFnQ9i533+vubcHsvcA5x7tu0mssdvcKd68oLS1NQ9kiA08s7jz4ynYunD6ak3Sraekn6QiC1cAMM5tqZgXA9cDy5A5mNj5p9mrgzWB6JXCFmY00s5HAFUGbSE567q3d7DrYyg3nTQm7FMkhKe+AdPdOM7uFxBd4FLjP3avM7A6g0t2XA182s6uBTmAfcGOw7j4z+w6JMAG4w933pVqTSKZ6aNV2xg4rZO4pY8IuRXJIWo5EufsKYEW3tm8nTd8O3N7LuvcB96WjDpFM1nColT++3cgXLzmJvKiu9ZT+o79tIgPEb16rJe5w7Tnlx+4skkYKApEBwN15bM1OPjxlJFNLhoRdjuQYBYHIAPDajgNsbTzCteeUhV2K5CAFgcgAsHTNTgblR7nqDF1JLP1PQSASspb2GL97o46Pnj6O4kJdSSz9T0EgErI/VNVxuK2T63SQWEKiIBAJ2W9eq6V81CDOnToq7FIkRykIREK0u6mV/6newzWzJxKJ6HbTEg4FgUiInlhXR9xhgW43LSFSEIiEaNnaXcwaP4zpY4aGXYrkMAWBSEi27z3C2p0HtDUgoVMQiIRk+dpdAPzNmQoCCZeCQCQE7s6yN3YxZ8ooJowYFHY5kuMUBCIheLOuierdhzUmsQwICgKRECx7o5a8iPGx08cfu7NIH1MQiPQzd+f3b9Rx0YwSRg0pCLsckfQEgZnNN7NNZlZtZrf1sPyrZrbRzNaZ2bNmNjlpWczM1gaP5d3XFck262oOUnugRTeYkwEj5TtcmVkUuBuYB9QAq81subtvTOr2OlDh7s1m9kXg/wKfDJa1uPvsVOsQyRQr1teRHzXmnTI27FJEgPRsEcwBqt19q7u3A48AC5I7uPvz7t4czL4C6KbrkpPcnSfW13Hh9BKGD84PuxwRID1BMBHYmTRfE7T15ibgyaT5IjOrNLNXzOya3lYys0VBv8rGxsbUKhYJyfrag9Tsb9FBYhlQ+vXm52b2GaACuDipebK715rZNOA5M1vv7lu6r+vui4HFABUVFd4vBYuk2RPr68iLGFfM0m4hGTjSsUVQCyTfSL0saHsPM5sLfAu42t3butrdvTZ43gq8AJyVhppEBhx358n19VwwvYQRg3W2kAwc6QiC1cAMM5tqZgXA9cB7zv4xs7OAe0iEwO6k9pFmVhhMlwAXAskHmUWyRtWuQ+zY18xVp48LuxSR90h515C7d5rZLcBKIArc5+5VZnYHUOnuy4EfAMXAY2YGsMPdrwZOAe4xsziJULqz29lGIlnjifV1RCPGFbMUBDKwpOUYgbuvAFZ0a/t20vTcXtZ7CTg9HTWIDGSJ3UJ1XHDSaEbqIjIZYHRlsUg/2NTQxLa9zcw/TVsDMvAoCET6wcoNDZjBPJ0tJAOQgkCkH/yhqp5zJo1kzNCisEsReR8FgUgf27G3mTfrDmm3kAxYCgKRPrayqh6AK09VEMjApCAQ6WMrq+qZNX4Y5aMGh12KSI8UBCJ9aHdTK2t27NfWgAxoCgKRPvT0xgbc0fEBGdAUBCJ96A8b6plaMoQPjS0OuxSRXikIRPrIwZYOXt6ylytmjSW4tYrIgKQgEOkjL2zaTWfcuULHB2SAUxCI9JGnNjZQUlzIWeUjwi5F5KgUBCJ9oK0zxgtv7WberLFEItotJAObgkCkD7y0ZS9H2mNccaruLSQDn4JApA88VdXAkIIoF5w0OuxSRI5JQSCSZvG488ybDVwycwyFedGwyxE5prQEgZnNN7NNZlZtZrf1sLzQzH4VLF9lZlOSlt0etG8ysyvTUY9ImNbWHKCxqU0D1EvGSDkIzCwK3A18FJgFfMrMZnXrdhOw392nA3cB3w/WnUVijONTgfnAfwevJ5KxnqpqID9qXDpzTNiliByXdGwRzAGq3X2ru7cDjwALuvVZACwJppcCl1viCpsFwCPu3ubu7wDVweuJZKynNtZz3rTRDCvKD7sUkeOSjiCYCOxMmq8J2nrs4+6dwEFg9HGuC4CZLTKzSjOrbGxsTEPZIulXvfswWxuPaCQyySgZc7DY3Re7e4W7V5SWloZdjkiPntqYGHtAQSCZJB1BUAuUJ82XBW099jGzPGA4sPc41xXJGE9VNXBm2XDGDx8Udikixy0dQbAamGFmU82sgMTB3+Xd+iwHFgbT1wLPubsH7dcHZxVNBWYAr6ahJpF+13ColbU7D+jeQpJx8lJ9AXfvNLNbgJVAFLjP3avM7A6g0t2XAz8Dfmlm1cA+EmFB0O9RYCPQCXzJ3WOp1iQShqc3NgDotFHJOCkHAYC7rwBWdGv7dtJ0K3BdL+t+F/huOuoQCdPTGxuYWjKE6WM09oBklow5WCwykDW1dvDSlj3M09gDkoEUBCJp8MKmRjpirt1CkpEUBCJpkBh7oICzJo0MuxSRD0xBIJKi9s44L7y1m7mnjCWqsQckAykIRFL00pY9NLV1auwByVgKApEUrayqp7gwjwtOKgm7FJEToiAQSUEs7jxV1cAlJ5dSlK8b50pmUhCIpGDN9v3sPdLO/NN0NbFkLgWBSAr+sKGegrwIl5yssQckcykIRE6Qu7Oyqp6PTC+huDAtF+mLhEJBIHKCqnYdovZAC/N1kznJcAoCkRO0sqqeiMHlp2i3kGQ2BYHICfrDhnrmTB3F6OLCsEsRSYmCQOQEbGk8zObdh7lSu4UkCygIRE7AinV1ADptVLKCgkDkBDyxvo6zJ43QkJSSFVIKAjMbZWZPm9nm4Pl9t140s9lm9rKZVZnZOjP7ZNKy+83sHTNbGzxmp1KPSH/Y2niYt+qb+Njp48MuRSQtUt0iuA141t1nAM8G8901A59191OB+cCPzGxE0vJb3X128FibYj0ife7JDfUACgLJGqkGwQJgSTC9BLimewd3f9vdNwfTu4DdQGmK7ysSmifW1XHWpBFMGKHdQpIdUg2Cse5eF0zXA0e9D6+ZzQEKgC1Jzd8NdhndZWa9nodnZovMrNLMKhsbG1MsW+TEbNtzhI11h7hKWwOSRY4ZBGb2jJlt6OGxILmfuzvgR3md8cAvgc+5ezxovh2YCXwYGAV8s7f13X2xu1e4e0VpqTYoJBwrNuhsIck+x7xBirvP7W2ZmTWY2Xh3rwu+6Hf30m8Y8ATwLXd/Jem1u7Ym2szs58DXP1D1Iv1sxfo6ziwfQdnIwWGXIpI2qe4aWg4sDKYXAsu6dzCzAuBx4BfuvrTbsvHBs5E4vrAhxXpE+syOvc1sqD3EVadra0CyS6pBcCcwz8w2A3ODecyswszuDfp8Avgr4MYeThN90MzWA+uBEuA/UqxHpM/8bt0uQGcLSfZJ6d657r4XuLyH9krg5mD6AeCBXta/LJX3F+kv7s5vX6/lw1NGareQZB1dWSxyHN6qb2Lz7sNcPXti2KWIpJ2CQOQ4LFu7i7yI6bRRyUoKApFjiMed372xi4tmlDBqSEHY5YiknYJA5Bhe27Gf2gMtLNBuIclSCgKRY1i2dhdF+RHmzTrqhfMiGUtBIHIUHbE4T6yvY+4pYxmiAeolSykIRI7ixc172HeknavPnBB2KSJ9RkEgchSPrdnJqCEFXHKyBqiX7KUgEOnF/iPtPLNxN9fMnkhBnv6pSPbS326RXixbW0t7LM51FWVhlyLSpxQEIr14bE0Np00cxinjh4VdikifUhCI9GDjrkNU7TrEdeeUh12KSJ9TEIj04LE1OymIRnS2kOQEBYFIN+2dcZat3cXcWWMYqVtKSA5QEIh08+ybDew70q7dQpIzUgoCMxtlZk+b2ebgeWQv/WJJg9IsT2qfamarzKzazH4VjGYmEqoHVm1nwvAiLppREnYpIv0i1S2C24Bn3X0G8Gww35MWd58dPK5Oav8+cJe7Twf2AzelWI9ISqp3H+Z/qvfy9+dNJi+qDWbJDan+TV8ALAmml5AYd/i4BOMUXwZ0jWP8gdYX6QsPrtpOftT4RIV2C0nuSDUIxrp7XTBdD/R2e8YiM6s0s1fMrOvLfjRwwN07g/kaQPf5ldA0t3eydE0NHz1tPKVDC8MuR6TfHPN2imb2DDCuh0XfSp5xdzcz7+VlJrt7rZlNA54LBqw/+EEKNbNFwCKASZMmfZBVRY7LsrW7aGrt5LPnTw67FJF+dcwgcPe5vS0zswYzG+/udWY2Htjdy2vUBs9bzewF4Czg18AIM8sLtgrKgNqj1LEYWAxQUVHRW+CInBB355cvb2fmuKGcM7nHcx5Eslaqu4aWAwuD6YXAsu4dzGykmRUG0yXAhcBGd3fgeeDao60v0h9e23GAjXWHuOH8ySQOX4nkjlSD4E5gnpltBuYG85hZhZndG/Q5Bag0szdIfPHf6e4bg2XfBL5qZtUkjhn8LMV6RE7Iz17cytCiPK7RcJSSg1Iacsnd9wKX99BeCdwcTL8EnN7L+luBOanUIJKqbXuO8OSGer548UkahUxykk6Ulpy3+M9byY9GuPHCKWGXIhIKBYHktMamNpauqeHjZ5cxZmhR2OWIhEJBIDltyUvb6IjF+YeLpoZdikhoFASSsw63dfKLl7dx5axxTCstDrsckdAoCCRnPbxqB4daO/nHi6eFXYpIqBQEkpMOt3Xykz9u4SPTSzhrki4gk9ymIJCc9PMX32HfkXa+fuXJYZciEjoFgeScA83tLP7zVubNGsvs8hFhlyMSOgWB5Jx7/rSVw22dfO2KD4VdisiAoCCQnLK7qZX7/2cbf3PGBGaOGxZ2OSIDgoJAcsqPn91MeyzOv8zT1oBIFwWB5IyqXQd5aNUO/v7cSUwtGRJ2OSIDhoJAcoK78+1lVYwcXMDX5ulMIZFkCgLJCY+/Xsua7fv55vyZDB+cH3Y5IgOKgkCy3qHWDr634i1ml4/g2nPKwi5HZMDRzdcl6/3wqbfZe6SN+26sIBLR6GMi3aW0RWBmo8zsaTPbHDy/71p9M7vUzNYmPVrN7Jpg2f1m9k7Sstmp1CPS3Utb9nD/S9u44bzJnFGmi8dEepLqrqHbgGfdfQbwbDD/Hu7+vLvPdvfZwGVAM/BUUpdbu5a7+9oU6xF5V1NrB7c+to6pJUO47aMzwy5HZMBKNQgWAEuC6SXANcfofy3wpLs3p/i+Isd0x+82Unewhf/8xJkMLtBeUJHepBoEY929LpiuB8Yeo//1wMPd2r5rZuvM7C4zK+xtRTNbZGaVZlbZ2NiYQsmSC56qquexNTV88ZKTOFt3FxU5qmMGgZk9Y2YbengsSO7n7g74UV5nPIlB7FcmNd8OzAQ+DIwCvtnb+u6+2N0r3L2itLT0WGVLDtuxt5lbl65j1vhh/PPluoJY5FiOub3s7nN7W2ZmDWY23t3rgi/63Ud5qU8Aj7t7R9Jrd21NtJnZz4GvH2fdIj1qbu9k0S8rcXd+8pmzKcjTGdIix5Lqv5LlwMJgeiGw7Ch9P0W33UJBeGBmRuL4woYU65Ec5u58Y+k6NjU08eNPncXk0bqNhMjxSDUI7gTmmdlmYG4wj5lVmNm9XZ3MbApQDvyx2/oPmtl6YD1QAvxHivVIDrvnT1v5/bo6br3yZC45eUzY5YhkjJROpXD3vcDlPbRXAjcnzW8DJvbQ77JU3l+ky9I1Ndz55FtcdcZ4vnjxSWGXI5JRtANVMt4fNtTxjaVvcNGMEn74iTNJ7GkUkeOlIJCM9ufNjXz54bXMLh/BPTecQ2FeNOySRDKOgkAy1lNV9dy8pJJppUP4+Y1zdNGYyAlSEEhGevjVHXzhgTXMHD+Mh/7hPN1aWiQF+gklGSUed3783GZ+9MxmLjm5lP/++7O1JSCSIv0LkoxxoLmdrz36Bs++tZuPn13GnR8/nfyoNmpFUqUgkIywducBvvTga+xuauX/XH0qnz1/ss4OEkkTBYEMaK0dMX787GYW/2krY4cVsfQLF3BmucYVEEknBYEMWC9t2cO//mY92/Y2c+05Zfyvq2bpoLBIH1AQyICzqb6JH6zcxDNvNjB59GAevPlcLpxeEnZZIllLQSADxqb6Ju754xYeX1tLcUEet155Mp+/cCqDCnSRmEhfUhBIqGJx50+bG7nvxXf48+Y9FOVHWHTRNL5w8UmMHFIQdnkiOUFBIKHY3NDEb16v5bev11J3sJUxQwu59cqT+fScSQoAkX6mIJB+0RGLs3bnAZ55s4FnNjawpfEI0Yhx8YdK+dePncKVp47TIDIiIVEQSJ840tbJhtqDvLbjAK9s3Uvltn0caY+RFzHOmzaaG86bzFVnTKB0aK/DVItIP1EQSEo6YnFq9rewZfdhNjU0sbmhiY11h6jefZh4MIL19DHF/O3ZEzl/WgkXfaiEYUU6BVRkIEkpCMzsOuB/A6cAc4IBaXrqNx/4f0AUuNfdu0Yymwo8AowG1gA3uHt7KjVJerg7LR0x9h1pZ8/hdhqb2mhsaqP+UCu7DrRQd7CFnftaqD3QQqzrGx+YMLyIk8cNZf5p4zmzbDhnlo+gpFi/+kUGslS3CDYAfwfc01sHM4sCdwPzgBpgtZktd/eNwPeBu9z9ETP7KXAT8JMUa8o67k7cE2fYxOJOzJ1YzOmMx+mMOx2xOJ3BfHtnYr4jFqetM057Z+K5rTNGa0eM1o44ze0xWto7aemIcbgtxuG2To60dXKopYNDrR0caulkf3M7bZ3x99ViBmOHFjF+RBFnlA1nwewJTB49hKklg5kxdqh+7YtkoFSHqnwTONY9X+YA1e6+Nej7CLDAzN4ELgM+HfRbQmLros+C4FuPr2fVO/venXf3Hvt5LzOetE5yn66XcRz3pHn3YJ2kZV3z7y5LfMnHPbE87h48EnfajAXtfWFwQZQhhXkUF+YxpDDKsKJ8ppUUM7Qoj1FDChg5pICRg/MpKS6kdGjiUVJcqBu9iWSZ/jhGMBHYmTRfA5xLYnfQAXfvTGp/37jGXcxsEbAIYNKkSSdUyIQRgzh57NBuL9zL+733vd/T3jXbUx8L/mMYZn/p/+580CFif2mPRuzd6YgRzL93OmpGNALRSIRoBPIiEfKiiXXzg+m8aISCqJEfjZAfjVCQF6EwL/FclB9NPPIiDC7Ioyg/opu2iQhwHEFgZs8A43pY9C13X5b+knrm7ouBxQAVFRUn9Bv5S5dOT2tNIiLZ4JhB4O5zU3yPWqA8ab4saNsLjDCzvGCroKtdRET6UX/s7F0NzDCzqWZWAFwPLPfEzvbngWuDfguBftvCEBGRhJSCwMz+1sxqgPOBJ8xsZdA+wcxWAAS/9m8BVgJvAo+6e1XwEt8Evmpm1SSOGfwslXpEROSDs97OnBnIKioqvLKyx0sWRESkF2a2xt0rurfrPEARkRynIBARyXEKAhGRHKcgEBHJcRl5sNjMGoHtYddxAkqAPWEX0c9y8TNDbn7uXPzMkFmfe7K7l3ZvzMggyFRmVtnTEftsloufGXLzc+fiZ4bs+NzaNSQikuMUBCIiOU5B0L8Wh11ACHLxM0Nufu5c/MyQBZ9bxwhERHKctghERHKcgtD/Ak4AAAJPSURBVEBEJMcpCEJiZl8zMzezkrBr6Wtm9gMze8vM1pnZ42Y2Iuya+oqZzTezTWZWbWa3hV1PfzCzcjN73sw2mlmVmf1z2DX1FzOLmtnrZvb7sGtJhYIgBGZWDlwB7Ai7ln7yNHCau58BvA3cHnI9fcLMosDdwEeBWcCnzGxWuFX1i07ga+4+CzgP+FKOfG6AfyZxe/2MpiAIx13AN0iMZZ/13P2ppLGpXyExGl02mgNUu/tWd28HHgEWhFxTn3P3Ond/LZhuIvHF2Ov449nCzMqAq4B7w64lVQqCfmZmC4Bad38j7FpC8nngybCL6CMTgZ1J8zXkwBdiMjObApwFrAq3kn7xIxI/6OJhF5KqY45ZLB+cmT0DjOth0beAfyWxWyirHO0zu/uyoM+3SOxGeLA/a5P+YWbFwK+Br7j7obDr6Utm9tfAbndfY2aXhF1PqhQEfcDd5/bUbmanA1OBN8wMErtIXjOzOe5e348lpl1vn7mLmd0I/DVwuWfvxSu1QHnSfFnQlvXMLJ9ECDzo7r8Ju55+cCFwtZl9DCgChpnZA+7+mZDrOiG6oCxEZrYNqHD3TLlz4Qkxs/nAD4GL3b0x7Hr6ipnlkTgYfjmJAFgNfDppjO6sZIlfNUuAfe7+lbDr6W/BFsHX3f2vw67lROkYgfSH/wKGAk+b2Voz+2nYBfWF4ID4LcBKEgdMH832EAhcCNwAXBb8+a4NfilLhtAWgYhIjtMWgYhIjlMQiIjkOAWBiEiOUxCIiOQ4BYGISI5TEIiI5DgFgYhIjvv/cBVaE3QwZacAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x1mfSRt2xyHv"
      },
      "source": [
        "# ReLU activation function\n",
        "\n",
        "## $f{(x)=\\text{max}{(0,x)}}$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "0RPiSTI4ybQO",
        "outputId": "d162f45d-5952-4cce-fb8f-964e7e8f0f77"
      },
      "source": [
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "relu = torch.nn.ReLU()\n",
        "x = torch.arange(-5.0, 5.0, 0.1)\n",
        "y = relu(x)\n",
        "\n",
        "plt.plot(x.numpy(), y.numpy())\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAWgklEQVR4nO3deXiU9dXG8fvIIrKJSkBkMbghiAIhAnVrXaq417Yqi/Z1qbQsFq3V2mr31rbWtYpaWmttCaAWt7prxap1qVkA2QUEAcGEPSwh23n/SEZRg5lk5pnnmZnv57q4DGScOSN4z4+TZ3KbuwsAEF17hD0AAOCLEdQAEHEENQBEHEENABFHUANAxLUM4k47d+7subm5Qdw1AGSkoqKide6e09DnAgnq3NxcFRYWBnHXAJCRzGzF7j7H6gMAIi6uE7WZLZdULqlGUrW75wc5FADgE01ZfZzo7usCmwQA0CBWHwAQcfEGtUt6wcyKzGxMQzcwszFmVmhmhWVlZcmbEACyXLxBfZy750k6XdJ4Mzvhszdw98nunu/u+Tk5DV5hAgBohriC2t1X1/+zVNJjkoYEORQA4BONBrWZtTOzDrGPJZ0qaW7QgwFAOvnf+xt0/+vvK4hvHR3PVR9dJT1mZrHbT3X355I+CQCkqbLynZowtVjt9mypkUN6qm3r5L6XsNF7c/dlkgYk9VEBIEPU1LomTi/R5h1VevCyIUkPaSmgt5ADQLa446XFemPpet38zaPUt1vHQB6D66gBoJleWVSqu15eovMH99AF+T0DexyCGgCa4cNNO3T1Q7N0+P4d9Mtz+wf6WAQ1ADRRZXWtxk8tVlWN657RedqrdYtAH48dNQA00e+fW6iSDzZp0qg8HZTTPvDH40QNAE3w3Nw1uv/193XJMbk686huKXlMghoA4rR83TZd+8gcDejZST8+o2/KHpegBoA4VFTVaFxBsfbYwzRp1CC1bpm6+GRHDQBx+PmT8zR/zRY9cMnR6rFP25Q+NidqAGjEjKJVmv7OSo0/8WCdeHiXlD8+QQ0AX2DR2nLd8Pi7GnbQvrr6lMNCmYGgBoDd2LqzWmMLitShTSv9ceQgtWwRTmQS1ADQAHfX9TPmaPm6bfrjiEHq0qFNaLMQ1ADQgH+8tUJPzVmja07toy8dvF+osxDUAPAZs1du0q+emq8T++Ro7JcPDnscghoAdrVpe6XGFRSrS4c2uu2CgdpjDwt7JK6jBoCY2lrXNQ/PVml5hR757jHap13rsEeSxIkaAD72p1eX6d8LS3XDGX01sGensMf5GEENAJLeXrZet7ywSGce2U3/d0xu2ON8CkENIOuVlldowrQS9dq3rX73jSNVX+YdGeyoAWS1mlrXxGmztGVHlf5+2RB1aNMq7JE+h6AGkNVuf3Gx3ly2Xn8IsJw2Uaw+AGStmYtKdffMJbogv4fOD7CcNlEENYCstDqF5bSJIqgBZJ3K6lqNLyhWdY3r3osGq02rYMtpE8WOGkDW+e2zCzRr5SbdMzpPvTu3C3ucRnGiBpBVnn13jR7473JdckyuzjgyNeW0iSKoAWSN99dt07X/nKOBKS6nTRRBDSArVFTVaOyUIrVsYZo0Oi+l5bSJYkcNICv87Il5Wri2XA9cerS6d9or7HGaJH1eUgCgmf5ZtEoPFdaX0/ZJfTltouIOajNrYWYlZvZUkAMBQDItXLtFN4ZcTpuoppyoJ0paENQgAJBs5RVVGjelOPRy2kTFNbWZ9ZB0pqS/BDsOACSHu+v6R9/V8vXbdNfIcMtpExXvy8sdkq6TVBvgLACQNH9/c4WenrNGPzitj4YdFG45baIaDWozO0tSqbsXNXK7MWZWaGaFZWVlSRsQAJpq1spN+vXT83XS4V303RPCL6dNVDwn6mMlnWNmyyVNl3SSmU357I3cfbK757t7fk5OTpLHBID4bNpeqfEfl9MOiEQ5baIaDWp3/5G793D3XEkjJL3s7hcFPhkANFFtrev79eW0k0bnqVPbaJTTJio9vwQKAA2479WlenlhqW48s1+kymkT1aR3Jrr7K5JeCWQSAEjAW8vW65bnF+nMo7rpW186MOxxkooTNYC0V1peoSunlSi3czv9/htHRa6cNlF8rw8AaS1WTlteUaUplw9V+z0zL9Yy7xkByCqxctpbzh+gPvt3CHucQLD6AJC2Zi6sK6e9ML+nvjm4R9jjBIagBpCWVm/aoasfnqW+3TrqF+ceEfY4gSKoAaSdXctp7xmdF/ly2kSxowaQdm56pq6c9t40KadNFCdqAGnl6Tlr9Lc3luvSY3N1epqU0yaKoAaQNpaVbdUPZ8zRoF6d9KPT06ecNlEENYC0UFFVo3EFxWrVwjRpVHqV0yaKHTWAtPDTJ+Zq0UfleuCSo3VAmpXTJip7XpIApK1HClfq4cJVmnDiIfpKGpbTJoqgBhBpC9du0U+emKtjDt5PV6VpOW2iCGoAkVVeUaWxU4rVsU0r3TlikFpkQAlAc7CjBhBJsXLaDzZs19RvD1VOhz3DHik0nKgBRNKDbyyvK6c9tY+Gpnk5baIIagCRU/LBRv3mmQU6+fAu+s4JB4U9TugIagCRsnFbpSZMLVGXDm10a4aU0yaKHTWAyKgrp52lsvKdeuS7X8qYctpEcaIGEBn3/mepZi4q041n9dWADCqnTRRBDSAS3ly6Xre+sEhnDzhAFw/LrHLaRBHUAEK3azntb79+ZMaV0yaKHTWAUFXX1Op700q0dWeVCr6dmeW0ieK/CIBQ3f7SYr21bINuzeBy2kSx+gAQmpkLSzVp5lKNOLqnvpHB5bSJIqgBhCJWTtuvW0f9/JzMLqdNFEENIOUqq2s1rqBYNVlSTpsodtQAUu6mZxZo9spNuu+iPOVmQTltojhRA0ipWDntZcf21vD+2VFOmyiCGkDK7FpOe/3ph4c9TtogqAGkxI7K7C2nTRQ7agApkc3ltIlq9CXNzNqY2f/MbLaZzTOzX6RiMACZ4+HClXqkaJWuzNJy2kTFc6LeKekkd99qZq0kvW5mz7r7WwHPBiADLFizRT95fK6OPWQ/TczSctpENRrU7u6Sttb/tFX9Dw9yKACZobyiSuMKirX3Xq10x4XZW06bqLi2+WbWwsxmSSqV9KK7v93AbcaYWaGZFZaVlSV7TgBpxt11/Yy6ctq7Rg7K6nLaRMUV1O5e4+4DJfWQNMTM+jdwm8nunu/u+Tk5OcmeE0CaefCN5Xr63TW69jTKaRPVpOtj3H2TpJmShgczDoBMECunPaVvF405nnLaRMVz1UeOmXWq/3gvSV+VtDDowQCkp1g5bdeObXTr+QMpp02CeK766CbpQTNrobpgf9jdnwp2LADpqLbWdXV9Oe0/x35Je7dtFfZIGSGeqz7mSBqUglkApLl7/7NUrywq06++1l9H9aCcNll4DyeApHhj6Trd+sIinTPgAF00tFfY42QUghpAwkq3VOh702apN+W0geB7fQBISHVNra6cVqJtO6s19Yqhakc5bdLxXxRAQm57cbHefn+DbrtggA7rSjltEFh9AGi2lxd+pHteWaqRQ3rq63mU0waFoAbQLKs2btfVD81Wv24d9bOzKacNEkENoMl2VtdofEGxamspp00FdtQAmuympxdo9qrNlNOmCCdqAE3yr9kf6sE3V+jy4yinTRWCGkDclpZt1fUz5iiPctqUIqgBxGVHZY3GTSlW65Z76O5ReWrVgvhIFXbUABrl7rrx8blaXFquv106hHLaFOMlEUCjHi5cqRnFdeW0Xz6MYpBUI6gBfKH5H27RT5+YRzltiAhqALu1paJK4wqK1KltK905gnLasLCjBtAgd9cP/zlHKzfu0PQxw9S5PeW0YeFEDaBBD/x3uZ6du1bXndZHR+fuG/Y4WY2gBvA5xR9s1E3PLNApfbtqzAmU04aNoAbwKRu3VWpCQbH237uNbj1/ACUAEcCOGsDHamtdVz00S+u2VmrG2GMop40ITtQAPjZp5hL9Z3GZfnp2Px3ZY++wx0E9ghqAJOmNJet0+0uLde7AAzSactpIIagB6KMtFfre9BL17txON51HOW3UsKMGstwn5bQ1mnrFMMppI4jfESDL3fLCYv2PctpIY/UBZLF/L/hI9/1nqUYO6UU5bYQR1ECWWrlhu77/8GwdcUBH/ezsfmGPgy9AUANZaGd1jcZPLVatU06bDthRA1noN08v0JxVm3XfRYN14H6U00YdJ2ogy/xr9of6+5srdMXxvTW8//5hj4M4ENRAFomV0w4+cB9dN5xy2nTRaFCbWU8zm2lm881snplNTMVgAJJre2W1xk4p0p6tWujuUYMop00j8eyoqyVd4+7FZtZBUpGZveju8wOeDUCSxMpp3yvdqgcvHaJue1NOm04afUl19zXuXlz/cbmkBZK6Bz0YgOR56J2VerR4ta486VCdQDlt2mnS333MLFfSIElvN/C5MWZWaGaFZWVlyZkOQMLmfbhZP31yno47pLMmnnxo2OOgGeIOajNrL2mGpKvcfctnP+/uk909393zc3J4xQaioK6ctlj7tG2lO0YMpJw2TcV1HbWZtVJdSBe4+6PBjgQgGdxd1z0yR6sop0178Vz1YZLul7TA3W8LfiQAyfDX/y7Xc/PW6ofDKadNd/GsPo6VdLGkk8xsVv2PMwKeC0ACilZs1G+fWaCv9uuqK46nnDbdNbr6cPfXJbHYAtLEhm2VmjC1WN06tdEtlNNmBL7XB5BBYuW062PltHtRTpsJeGsSkEEmzVyiVymnzTgENZAhKKfNXAQ1kAFi5bQH5bSnnDYDsaMG0tyu5bTTrsijnDYD8TsKpLlYOe3tFw7QoZTTZiRWH0Aae2n+J+W05w2inDZTEdRAmqorp52l/t0pp810BDWQhmLltC7pnlGDKafNcOyogTT066fqymknXzxYvfZrG/Y4CBgnaiDNPDn7Q/3jrRUac8JBOvUIymmzAUENpJElpXXltEfn7qNrT+sT9jhIEYIaSBPbK6s1rqBIe7VqobtG5lFOm0XYUQNpYNdy2n9cNlT7790m7JGQQrwkA2kgVk478eRDddyhncMeBylGUAMRFyunPf7QzrryJMppsxFBDURYrJx237atdceFlNNmK3bUQETtWk770Jhh2o9y2qzFiRqIqPtff1/PzVur64cfrnzKabMaQQ1EUNGKDfrdswt1ar+u+vbxvcMeByEjqIGIqSunLdEBnfbSHyinhdhRA5HycTnttko9Sjkt6nGiBiLk7vpy2p+ffYT6d6ecFnUIaiAi/ltfTnveoO4aOaRn2OMgQghqIAI+2lKhidNLdEhOe/3mvP7spfEp7KiBkFXV1GrC1GJtr6zR9DF5atua/y3xafyJAEJ2y/OL9M7yjbpzxEAd0oVyWnweqw8gRC/O/0h/enWZRg/tpXMHdg97HEQUQQ2EZOWG7bqmvpz2J2dRTovdI6iBEFRU1WhcAeW0iA87aiAEv356vt5dTTkt4tPoidrM/mpmpWY2NxUDAZnuiVmrNeWtDyinRdziWX38TdLwgOcAssKS0nL96NF3KadFkzQa1O7+qqQNKZgFyGjbK6s1dkox5bRoMnbUQAq4u258bK6WlFFOi6ZL2ku6mY0xs0IzKywrK0vW3QIZYfo7K/VoCeW0aJ6kBbW7T3b3fHfPz8nJSdbdAmlv7urN+ll9Oe33KKdFM7AkAwK0paJK46d+Uk67B+W0aIZ4Ls+bJulNSX3MbJWZXR78WED6i5XTrt64Q5NGD6KcFs3W6BcT3X1kKgYBMk2snPbGM/tq8IGU06L5WH0AAYiV0552RFddfhzltEgMQQ0k2fqtOzW+oK6c9uZvUk6LxHEdNZBENfXltBu2U06L5OFEDSTR3S8v0WvvraOcFklFUANJ8vp763THvxfr65TTIskIaiAJ1m6uK6c9tEt7/ZpyWiQZQQ0kqKqmVldOK9aOqhrdM5pyWiQff6KABFFOi6BxogYSECunvWgY5bQIDkENNFOsnPbI7ntTTotAEdRAM8TKaSXpntF52rMl5bQIDjtqoBli5bR//la+eu5LOS2CxYkaaKJYOe13TjhIX+3XNexxkAUIaqAJdi2n/QHltEgRghqIU6yctm3rFrp7FOW0SB121EAc3F031JfTTrl8qLp2pJwWqcORAIjDtP+t1GMlq3X1KYfp2EMop0VqEdRAI+au3qyf/2ueTjgsRxNOPCTscZCFCGrgC2zeUaVxBcXarx3ltAgPO2pgN9xd1z4yWx9u2qGHvjNM+7ZrHfZIyFKcqIHduP/19/XC/I90/emHU06LUBHUQANi5bTDj9ifclqEjqAGPiNWTtt9n7108/lHUQKA0LGjBnbx2XLajm0op0X4OFEDu7jr5ff02nvr9ItzKKdFdBDUQL3X3ivTnf9+T1/P664RR1NOi+ggqAFJazbv0FXTZ9WV036NclpEC0GNrFdVU6sJU0vqy2kHU06LyOFPJLLezc8tVNGKjfrjyEE6pEv7sMcBPocTNbLa8/PW6s+vva+Lhx2ocwYcEPY4QIMIamStD9Zv1w8ema2jeuytG8/qG/Y4wG4R1MhKFVU1GltQJJM0aRTltIi2uILazIab2SIzW2Jm1wc9FBC0Xz41X/M+3KLbLhhIOS0ir9GgNrMWkiZJOl1SP0kjzaxf0IMBQXm8ZLWmvv2BvvPlg3QK5bRIA/Fc9TFE0hJ3XyZJZjZd0rmS5id7mLPvel0VVTXJvlvgU1Zs2K4hufvq2lMpp0V6iCeou0taucvPV0ka+tkbmdkYSWMkqVevXs0a5uCcdqqsqW3WvwvEK6/XPrrm1MPUknJapImkXUft7pMlTZak/Px8b8593DFiULLGAYCMEc+RYrWkXb/xQY/6XwMApEA8Qf2OpEPNrLeZtZY0QtKTwY4FAIhpdPXh7tVmNkHS85JaSPqru88LfDIAgKQ4d9Tu/oykZwKeBQDQAL7sDQARR1ADQMQR1AAQcQQ1AEScuTfrvSlffKdmZZJWJP2Og9dZ0rqwh0ixbHzOUnY+b55ztB3o7jkNfSKQoE5XZlbo7vlhz5FK2ficpex83jzn9MXqAwAijqAGgIgjqD9tctgDhCAbn7OUnc+b55ym2FEDQMRxogaAiCOoASDiCOoGmNk1ZuZm1jnsWVLBzP5gZgvNbI6ZPWZmncKeKSjZWNRsZj3NbKaZzTezeWY2MeyZUsXMWphZiZk9FfYsiSCoP8PMeko6VdIHYc+SQi9K6u/uR0laLOlHIc8TiCwuaq6WdI2795M0TNL4LHnekjRR0oKwh0gUQf15t0u6TlLWfJXV3V9w9+r6n76luhafTPRxUbO7V0qKFTVnNHdf4+7F9R+Xqy64uoc7VfDMrIekMyX9JexZEkVQ78LMzpW02t1nhz1LiC6T9GzYQwSkoaLmjA+sXZlZrqRBkt4Od5KUuEN1h660b8xOWrltujCzlyTt38CnbpD0Y9WtPTLOFz1vd3+i/jY3qO6vyQWpnA2pYWbtJc2QdJW7bwl7niCZ2VmSSt29yMy+EvY8icq6oHb3Uxr6dTM7UlJvSbPNTKr763+xmQ1x97UpHDEQu3veMWZ2iaSzJJ3smXtxfdYWNZtZK9WFdIG7Pxr2PClwrKRzzOwMSW0kdTSzKe5+UchzNQtveNkNM1suKd/d0+U7bzWbmQ2XdJukL7t7WdjzBMXMWqrui6Unqy6g35E0KtM7QK3u5PGgpA3uflXY86Ra/Yn6B+5+VtizNBc7akjS3ZI6SHrRzGaZ2X1hDxSE+i+YxoqaF0h6ONNDut6xki6WdFL97++s+pMm0gQnagCIOE7UABBxBDUARBxBDQARR1ADQMQR1AAQcQQ1AEQcQQ0AEff/SNc74YYwGoYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zyy_yB7rzytX"
      },
      "source": [
        "# PReLU activation function\n",
        "\n",
        "## $f{(x)=\\text{max}{(x,ax)}}$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "agEJu2Bnz5KQ",
        "outputId": "23e40027-8d23-41e8-93b4-97c7cddeb433"
      },
      "source": [
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "prelu = torch.nn.PReLU(num_parameters=1)\n",
        "x = torch.arange(-5.0, 5.0, 0.1)\n",
        "y = prelu(x)\n",
        "\n",
        "plt.plot(x.detach().numpy(), y.detach().numpy())\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeIElEQVR4nO3de1RW54Hv8e8jFxHkIhcTFBC5dIgmahABJZc22k6MmbjOmDSptxgvOG1zjmZ1Thrbk3XSZnVWelk1J9N0TZSQFC81UzNtJzbNTM10msQICl5oDCYCkYhXLiIqIvC+z/kD6kpS4wX2+27el99nLZdc3vXs30b4udn72fsx1lpERCRwDXM7gIiIDIyKXEQkwKnIRUQCnIpcRCTAqchFRAJcqBsbTUxMtOnp6W5sWkQkYFVVVTVba5M++3FXijw9PZ3Kyko3Ni0iErCMMQ2X+7hOrYiIBDgVuYhIgHPk1Iox5jBwFvAAPdbaPCfGFRGRq3PyHPmXrLXNDo4nIiLXQKdWREQCnFNFboH/NMZUGWOKL/cCY0yxMabSGFPZ1NTk0GZFRMSpIr/NWpsLzAa+aYy547MvsNaus9bmWWvzkpL+ahqkiIj0kyNFbq092vf3KeDXQL4T44qIBIuOrh6e+vcDnOnodnzsARe5MSbKGBP9l7eBrwDvDXRcEZFg0dntYeWGKsp2HmbPx6cdH9+JWSs3AL82xvxlvM3W2jccGFdEJOB1e7w8unkPbx9q5icPTOZLOaMd38aAi9xaWw9MdiCLiEhQ8Xgtj72yj+01p3h67kTun5rik+1o+qGIiA94vZZvv1rNturjrJmdw6Lp6T7blopcRMRh1lr+778fYGtVI6tmZrPyzkyfbk9FLiLiIGstz/z+IBvKGyi+I4PVs7J9vk0VuYiIg/7fm4d44a16FhWOY83sHPomgviUilxExCHr3qrj2e2HmJebwvfum+iXEgcVuYiIIzbsPMw/vX6QOZOS+dH9kxg2zD8lDipyEZEB21rVyJO/PcDMnNGs/eoUQvxY4qAiFxEZkG3Vx3h8636KshJ4fkEu4aH+r1UVuYhIP21//ySrt+xj6rhRrF+cR0RYiCs5VOQiIv3wzqFmvrF5DxPGxFC6ZBqR4a6sZQ+oyEVErtvuw62sKKskIzGKsqX5REeEuZpHRS4ich2qG9t45KXdJMdFsGFZAXGR4W5HUpGLiFyrgyfaWVy6i1FRYWxaXkBS9HC3IwEqchGRa1LXdI6FJRVEhIaweXkhybEj3I50iYpcROQqjrR2sGB9BdbCxuUFpMZHuh3pU9y7zCoiEgBOnOlkfkk5F7o9bCkuJGv0SLcj/RUdkYuIfI7mcxeZX1LO6fPdlC3N56bkGLcjXZaKXETkMto6ulhYUsGxtguULpnG5NQ4tyN9LhW5iMhnnO3s5uHSXdQ3nWf94jzyx8e7HemKVOQiIp/Q0dXDspcrOXCsnZ8vyOX27CS3I12VilxEpE9nt4eVG6qobGjl2YemMGvCDW5HuiaatSIiAnR7vDy6eS9vH2rmJw9M5t5JY9yOdM10RC4iQ57Ha3nslX1srznJ03Mncv/UFLcjXRfHitwYE2KM2WuM2ebUmCIivub1Wr79ajXbqo/znXtyWDQ93e1I183JI/JVQI2D44mI+JS1lqdeO8DWqkZWzcym+I5MtyP1iyNFboxJAeYAJU6MJyLia9ZannnjIGU7Gyi+I4PVs7LdjtRvTh2RPws8Dng/7wXGmGJjTKUxprKpqcmhzYqI9M9zb9bywp/qWViYxprZOX5b8d4XBlzkxph7gVPW2qorvc5au85am2etzUtKGvzzMkUkeK17q4612z9kXm4K37/v5oAucXDmiLwIuM8YcxjYAtxljNnowLgiIo7bsPMw//T6QeZMSuZH909imJ9XvPeFARe5tXaNtTbFWpsOPAT8l7V24YCTiYg4bGtVI0/+9gCzbhrNsw9OISQIShw0j1xEhoht1cd4fOt+bstK5GfzcwkLCZ76c/TOTmvtfwP/7eSYIiID9WbNSVZv2cfUcaNYt3gqEWEhbkdyVPD8lyQichnvHGrm65v2MHFMDKVLphEZHnxPJlGRi0jQ2n24lRVllWQkRvGLpflER4S5HcknVOQiEpSqG9t45KXdJMdFsGFZAXGR4W5H8hkVuYgEnYMn2llcuotRUWFsWl5AUvRwtyP5lIpcRIJKXdM5FpZUEBEawublhSTHjnA7ks+pyEUkaBxp7WDB+goANq0oIDU+0uVE/hF8l29FZEg6caaT+SXlXOj2sKW4kMykkW5H8hsdkYtIwGs6e5H5JeWcPt9N2dJ8bkqOcTuSX6nIRSSgtXV0sejFCo61XaB0yTQmp8a5HcnvVOQiErDOdnbzcOku6pvOs35xHvnj492O5AoVuYgEpI6uHpa9XMmBY+08vyCX27OH7uOxVeQiEnA6uz2s3FBFZUMrzz40hS9PuMHtSK7SrBURCSjdHi+Pbt7L24ea+fH9k7h30hi3I7lOR+QiEjA8Xstjr+xje81Jnp47kQfyUt2ONCioyEUkIHi9lm+/Ws226uOsmZ3DounpbkcaNFTkIjLoWWt56rUDbK1qZNXMbFbemel2pEFFRS4ig5q1lmfeOEjZzgZW3pHB6lnZbkcadFTkIjKoPfdmLS/8qZ5FheN4YnZOwK947wsqchEZtNa/Vc/a7R9y/9QUvnffRJX451CRi8igtLG8gR+8XsOcScn8cN4khgXJive+oCIXkUHn1apG/s9v3mPWTaN59sEphKjEr0hFLiKDyu+qj/O/t+7n9uxEfjY/l7AQ1dTV6CskIoPGmzUnWbVlL1PHjeKFRVOJCAtxO1JAGHCRG2MijDG7jDH7jTEHjDHfcyKYiAwt7xxq5uub9jBhTAylS6YRGa4niFwrJ75SF4G7rLXnjDFhwDvGmN9ba8sdGFtEhoDdh1tZUVZJRmIUZUvziY4IcztSQBlwkVtrLXCu792wvj92oOOKyNBQ3djGIy/tJjkugg3LCoiLDHc7UsBx5By5MSbEGLMPOAX8wVpbcZnXFBtjKo0xlU1NTU5sVkQC3MET7Swu3UVcZBiblheQFD3c7UgByZEit9Z6rLVTgBQg3xhz82Ves85am2etzUtKGroPgBeRXnVN51hYUkFEaAi/XFFIcuwItyMFLEdnrVhr24A/Anc7Oa6IBJcjrR0sWF+BtbBxeQGp8ZFuRwpoTsxaSTLGxPW9PQL4MnBwoOOKSHA6caaT+SXlXOj2sHF5AVmjR7odKeA5MWslGfiFMSaE3v8Y/tVau82BcUUkyDSfu8iCknJOn+9m0/ICbkqOcTtSUHBi1ko1cKsDWUQkiLV1dLGwpIKjbRcoW1rA5NQ4tyMFDd3ZKSI+d7azm4dLd1HfdJ71i/PIHx/vdqSgoiIXEZ/q6Oph2cuVHDjWzvMLcrk9W7PWnKYiFxGf6ez2sHJDFZUNrax9cApfnnCD25GCkh5mICI+0e3x8ujmvbx9qJmfPDCZv5s8xu1IQUtH5CLiOI/X8tgr+9hec5Kn507k/qkpbkcKaipyEXGU12v59qvVbKs+znfuyWHR9HS3IwU9FbmIOMZay1OvHWBrVSOrZmZTfEem25GGBBW5iDjCWsszvz9I2c4Giu/IYPWsbLcjDRkqchFxxHNv1vLCW/UsKhzHmtk5WvHej1TkIjJg69+qZ+32D7l/agrfu2+iStzPVOQiMiAbyhv4wes1zJmUzA/nTWKYVrz3OxW5iPTb1qpGnvzNe8zMGc3ar04hRCXuChW5iPTLtupjPL51P7dlJfL8glzCQ1UnbtFXXkSu2/b3T7J6yz6mjhvFusVTiQgLcTvSkKYiF5Hr8s6hZr6xaQ8TxsTw4pJpRIbrSR9uU5GLyDXbfbiVFWWVZCRFUbY0n5iIMLcjCSpyEblG+4+08chLu0mOjWDDsgLiIsPdjiR9VOQiclU1x9tZXLqLuMgwNi4vICl6uNuR5BNU5CJyRXVN51j0YgUjwkLYvLyQMXEj3I4kn6EiF5HPdaS1gwXrK7AWNi4vIC0h0u1Ichm63Cwil3X8zAXml5RzodvDL1cUkjV6pNuR5HPoiFxE/krT2YssKKng9PluypbmM2FMjNuR5ApU5CLyKW0dXSx6sYJjbRcoXTKNyalxbkeSqxhwkRtjUo0xfzTGvG+MOWCMWeVEMBHxv7Od3Txcuov6pvOsX5xH/vh4tyPJNXDiHHkP8C1r7R5jTDRQZYz5g7X2fQfGFhE/6ejqYdnLlRw41s6/LJzK7dlJbkeSazTgI3Jr7XFr7Z6+t88CNcDYgY4rIv7T2e1h5YYqKhtaWfvgFGZNuMHtSHIdHD1HboxJB24FKi7zuWJjTKUxprKpqcnJzYrIAHR7vDy6eQ9vH2rmmXmT+LvJY9yOJNfJsSI3xowEXgVWW2vbP/t5a+06a22etTYvKUm/sokMBh6v5bFX9rG95hRPz53IV/NS3Y4k/eBIkRtjwugt8U3W2n9zYkwR8S2v1/LEq9Vsqz7Omtk5LJqe7nYk6ScnZq0Y4EWgxlr704FHEhFfs9by1GsH+FVVI6tmZrPyzky3I8kAOHFEXgQsAu4yxuzr+3OPA+OKiA9Ya3nmjYOU7Wxgxe3jWT0r2+1IMkADnn5orX0H0EJ9IgHiuTdreeFP9SwsTOM799ykFe+DgO7sFBlC1r9Vz9rtHzIvN4Xv33ezSjxIqMhFhogN5Q384PUa5tySzA/n3cIwrXgfNFTkIkPA1qpGnvzNe8zMGc3aB6cQGqIf/WCif02RIPe76uM8vnU/RVkJPL8gl/BQ/dgHG/2LigSx7e+fZNWWveSmjWL94jwiwkLcjiQ+oCIXCVLvHGrmG5v3MGFMDKWPTCMyXOvIBCsVuUgQqjzcyoqySsYnRPGLR/KJiQhzO5L4kIpcJMhUN7ax5KXdJMdGsHF5AaOiwt2OJD6mIhcJIgdPtLO4dBdxkWFsWlFAUvRwtyOJH6jIRYJEXdM5FpZUEBEawublhSTHjnA7kviJilwkCBxp7WBhSe8yAJtWFJCWEOlyIvEnXcYWCXAnznQyv6Scji4PW4oLyUwa6XYk8TMdkYsEsOZzF1lQUs7p892ULc3npuQYtyOJC1TkIgGqraOLhSUVHG27wEuPTGNyapzbkcQlKnKRAHS2s5uHS3dR33Se9YvzmJYe73YkcZGKXCTAXOjysOzlSg4ca+fnC3K5PVtr4A51KnKRAHKxx0PxhkoqG1p59qEpzJpwg9uRZBDQrBWRANHt8fLNTXt5+1AzP75/EvdOGuN2JBkkdEQuEgA8Xstjr+xje81Jnp47kQfyUt2OJIOIilxkkPN6LU+8Ws226uOsmZ3DounpbkeSQUZFLjKIWWt56rUD/KqqkVUzs1l5Z6bbkWQQUpGLDFLWWp554yBlOxsoviOD1bOy3Y4kg5SKXGSQeu7NWl74Uz2LCsexZnaOVryXz+VIkRtjSo0xp4wx7zkxnshQt/6tetZu/5B5uSl8776JKnG5IqeOyF8G7nZoLJEhbUN5Az94vYY5tyTzw3m3MGyYSlyuzJEit9a+BbQ6MZbIULa1qpEnf/MeM3NGs/bBKYSG6OynXJ2+S0QGid9VH+fxrfu5LSuR5xfkEh6qH0+5Nn77TjHGFBtjKo0xlU1NTf7arEhAeLPmJKu27GXquFGsWzyViLAQtyNJAPFbkVtr11lr86y1eUlJesiPyF+8c6iZr2/aw4QxMZQumUZkuJ6cIddHv7uJuKjycCsryirJSIyibGk+0RFhbkeSAOTU9MNfAjuBvzHGNBpjljkxrkgwq25s45GXdpMcG8GGZQXERYa7HUkClCO/w1lrv+bEOCJDxcET7Swu3UVcVBibVhSQFD3c7UgSwHRqRcTP6pvOsbBkFxGhIWxeXkhy7Ai3I0mAU5GL+NGR1g4WlFRgrWXj8gJS4yPdjiRBQJfHRfzkxJlO5peU09HlYUtxIVmjR7odSYKEjshF/KD53EUWlJRz+nw3ZUvzuSk5xu1IEkRU5CI+1tbRxaIXd3G07QKlS6YxOTXO7UgSZFTkIj50trObh1/aTd2pc6xfnEf++Hi3I0kQUpGL+MiFLg/LXq7kwNEz/HxBLrdn645m8Q0VuYgPXOzxULyhksqGVp59aAqzJtzgdiQJYpq1IuKwbo+Xb27ay9uHmvnx/ZO4d9IYtyNJkNMRuYiDPF7LY6/sY3vNSZ6eO5EH8lLdjiRDgIpcxCFer+WJV6vZVn2cNbNzWDQ93e1IMkSoyEUcYK3lqdcO8KuqRv7XzGxW3pnpdiQZQlTkIgNkreWZNw5StrOBFbeP57FZ2W5HkiFGRS4yQP/8X7W88Kd6Fham8Z17btKK9+J3KnKRAVj/Vj0//cOH/H3uWL5/380qcXGFilyknzaUN/CD12uYc0syP5o3iWHDVOLiDhW5SD9srWrkyd+8x8yc0ax9cAqhIfpREvfou0/kOm2rPsbjW/dzW1Yizy/IJTxUP0biLn0HilyHN2tOsnrLPqaOG8W6xVOJCAtxO5KIilzkWu2obebrm/YwYUwMpUumERmuJ1zI4KAiF7kGuw+3svwXlWQkRlG2NJ/oiDC3I4lcoiIXuYrqxjYeeWk3ybERbFhWQFxkuNuRRD5FRS5yBQdPtLO4dBdxkWFsWlFAUvRwtyOJ/BUVucjnqG86x8KSXUSEhrB5eSHJsSPcjiRyWY4UuTHmbmPMB8aYWmPME06MKeKmI60dLCipwFrLxuUFpCVEuh1J5HMNuMiNMSHA88BsYALwNWPMhIGOK+KWE2c6mV9STkeXh43LC8gaPdLtSCJX5MT8qXyg1lpbD2CM2QLMBd53YGwRn7PWUtd0nnfrmtlR28y7dS1YC5uWF3BTcozb8USuyokiHwsc+cT7jUDBZ19kjCkGigHS0tIc2KxI/x0/c4EdtS2821fcJ9o7ARgbN4K7J97I4unp3JIS63JKkWvjtzsarLXrgHUAeXl51l/bFQFo6+hiZ10LO+qaebe2hfrm8wDER4UzPTOBosxEZmQmMC4hUk8wlIDjRJEfBT65MGFK38dEXNPR1cOuj1p5t66FHbXNvH+8HWshKjyE/PHxzC9IY0ZmIjk3RuuphRLwnCjy3UC2MWY8vQX+EDDfgXFFrlm3x8u+I22957hrW9h75DTdHkt4yDCmpMWxeuYXKMpKYHJqHGF6UqEEmQEXubW2xxjzKPAfQAhQaq09MOBkIlfg9VreP97Ou3W957h3fdRKR5cHY+CWsbEsvW08RZmJTEuPZ0S4Hmwlwc2Rc+TW2teB150YS+RyrLUcbunom1XSzM66Fk53dAOQmRTFvNwUirISKMxI0C30MuTo8W0yaJ1q72RHXTM7alvYWdfC0bYLANwYE8GXckZTlJlIUVYiN8ZGuJxUxF0qchk0zlzopry+d0rgjroWak+dAyAuMozpGQn8wxczKcpMYHxilGaWiHyCilxc09ntYffh1t753HXNvHf0DF4LI8JCmDY+ngemplCUlciE5BjNLBG5AhW5+E2Px8v+xjN9R9zN7Gloo8vjJXSY4da0OP7nXdnMyEzg1rRRWj5N5DqoyMVnvF7LByfPsqO29+JkxUetnLvYA8CE5BgenjGOGVmJ5KfHEzVc34oi/aWfHnHUkdbemSU76lrYWddM87kuANITIrlvyhiKMhOZnplAfJRmlog4RUUuA9J09mLvXO7a3tvfG0/3ziwZHT2c27ISmZHVO7NkbJye5S3iKypyuS5nO7upqG+99MySD06eBSAmIpTCjARW3J5BUVYCmUkjNbNExE9U5HJFnd0e9nx8+tIRd3XjGTxey/DQYUxLj2furb2nS24eG0uIZpaIuEJFLp/i8Vr+fPTMpQuUuw+3crHHS8gww+SUWL7xxUxmZCZya1ocEWG69V1kMFCRD3HWWmpPnbt0gbK8voWznb0zS3JujGZBwTiKshLIHx9PdESYy2lF5HJU5ENQ4+mOS6dK3q1roensRQBS40cw55ZkZmT1Pps7caRWjBcJBCryIaDl3EV21rdcuoOyoaUDgMSR4UzPTKQoM4GirERS47XAsEggUpEHofMXexdV+Mvpkprj7QCMHB5KYUY8D09PpygrkS/coJklIsFARR4Eunq87P34NDvqeh84te9IGz1eS3joMKamjeIfv/IFZmQlMmlsLKFaVEEk6KjIA5DHa3n/WHvfI16bqTx8mgvdHoYZuCUljuI7MijKSmTquFGaWSIyBKjIA4C1lrqm85fuoNxZ38KZC72LKmSPHsmD01KZkZlAQUYCsSM0s0RkqFGRD1LHz1zovThZ2zuz5ER7JwBj40bwtxNvoCir95klo6O1qILIUKciHyTaOrrYWddy6db3+ubzAMRHhTM9M6FvNZwE0uIjdYFSRD5FRe6Sjq4edh8+fenZ3AeOtWMtRIWHUJCRwPyCNGZkJpJzY7QWVRCRK1KR+0lXj5f9jW29iwfXtrD3yGm6PZawEENu2igem/UFirISmJQSR5hmlojIdVCR+4jXa6k50X7pDspdH7XS0eXBGLh5TCxLbxtPUWYi09LjGRGumSUi0n8qcodYa2lo6bh0jntnfQut53sXVchIimJebgpFWQkUZiQQF6lFFUTEOQMqcmPMA8BTwE1AvrW20olQgeJUe+el4n63roWjbb2LKiTHRvClvxnNjMwEZmQlkByrRRVExHcGekT+HvD3wAsOZBn0znR0U/5RS98FyhZqT50DIHZEGDMyE/iHL2ZSlJnA+MQozSwREb8ZUJFba2uAoC2tzm4Puw+3XnrY1HtHz+C1MCIshGnj43lgagpFWYlMSI7RzBIRcY3fzpEbY4qBYoC0tDR/bfa69Hi87G88c+kmnKqPT9PV4yV0mOHWtDgevSuboswEbk0bRXioZpaIyOBw1SI3xmwHbrzMp75rrf3ttW7IWrsOWAeQl5dnrzmhD1lr+eDk2Ut3UFZ81Mq5i72LKkxIjuHh6eOYkZVIfno8UcN1XVhEBqertpO1dpY/gvjLxy0dvFvXe457Z10zzed6Z5akJ0Ry35Te9SenZyYQH6WZJSISGIL+MLPp7MVLD5vaUddM4+nemSWjo4dzW1YiM7ISKcpKZGycZpaISGAa6PTD/wH8M5AE/M4Ys89a+7eOJOuns53dVNS3XpoW+MHJswBER4QyPSOBFbdnUJSVQGaSFlUQkeAw0FkrvwZ+7VCWfuns9rCn4XTfs7lb+PPRM3i8luGhw5iWHs/cW3tPl9w8NpYQzSwRkSAUcKdWPF7Ln4+euXS6ZPfhVi72eAkZZpicEss3vpjJ9MwEctO0qIKIDA0BVeTPvXmI9W/Xc7azd2ZJzo3RLCgYR1FWAvnj44mO0KIKIjL0BFSR3xgbwZxbkpmRlciMzAQSRw53O5KIiOsCqsi/mpfKV/NS3Y4hIjKo6PZEEZEApyIXEQlwKnIRkQCnIhcRCXAqchGRAKciFxEJcCpyEZEApyIXEQlwxlr/r/FgjGkCGvy+4YFLBJrdDuFnQ3GfYWju91DcZwis/R5nrU367AddKfJAZYyptNbmuZ3Dn4biPsPQ3O+huM8QHPutUysiIgFORS4iEuBU5NdnndsBXDAU9xmG5n4PxX2GINhvnSMXEQlwOiIXEQlwKnIRkQCnIu8nY8y3jDHWGJPodhZfM8b82Bhz0BhTbYz5tTEmzu1MvmKMudsY84ExptYY84TbefzBGJNqjPmjMeZ9Y8wBY8wqtzP5izEmxBiz1xizze0sA6Ei7wdjTCrwFeBjt7P4yR+Am621k4APgTUu5/EJY0wI8DwwG5gAfM0YM8HdVH7RA3zLWjsBKAS+OUT2G2AVUON2iIFSkffPWuBxYEhcKbbW/qe1tqfv3XIgxc08PpQP1Fpr6621XcAWYK7LmXzOWnvcWrun7+2z9BbbWHdT+Z4xJgWYA5S4nWWgVOTXyRgzFzhqrd3vdhaXLAV+73YIHxkLHPnE+40MgUL7JGNMOnArUOFuEr94lt4DMq/bQQYqoBZf9hdjzHbgxst86rvAd+g9rRJUrrTP1trf9r3mu/T+Gr7Jn9nEP4wxI4FXgdXW2na38/iSMeZe4JS1tsoY80W38wyUivwyrLWzLvdxY8wtwHhgvzEGek8x7DHG5FtrT/gxouM+b5//whizBLgXmGmD9+aDo0DqJ95P6ftY0DPGhNFb4pustf/mdh4/KALuM8bcA0QAMcaYjdbahS7n6hfdEDQAxpjDQJ61NlCenNYvxpi7gZ8Cd1prm9zO4yvGmFB6L+bOpLfAdwPzrbUHXA3mY6b3qOQXQKu1drXbefyt74j8H62197qdpb90jlyuxc+AaOAPxph9xph/cTuQL/Rd0H0U+A96L/j9a7CXeJ8iYBFwV9+/776+I1UJEDoiFxEJcDoiFxEJcCpyEZEApyIXEQlwKnIRkQCnIhcRCXAqchGRAKciFxEJcP8fti961TSFRVAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2NDsvOBc1hgl"
      },
      "source": [
        "# Softmax activation function\n",
        "\n",
        "## $\\textit{softmax}{(x_i)}=\\frac{e^{x_i}}{\\overset k{\\underset{j=1}{\\mathrm\\Sigma}}e^{x_{\\hspace{-0.166667em}j}}}$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MiyPLYtV1rEH",
        "outputId": "fb46792d-84c3-4e7d-e549-6cd1dbca91c1"
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch\n",
        "\n",
        "softmax = nn.Softmax(dim=1)\n",
        "x_input = torch.randn(1, 3)\n",
        "y_input = softmax(x_input)\n",
        "\n",
        "print(x_input)\n",
        "print(y_input)\n",
        "print(torch.sum(y_input, dim=1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-2.2136, -0.6236,  0.7784]])\n",
            "tensor([[0.0387, 0.1899, 0.7714]])\n",
            "tensor([1.])\n"
          ]
        }
      ]
    }
  ]
}