Micrograd-revisited.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "21410a53-082f-42d5-bf17-7261d9e453c5",
   "metadata": {},
   "source": [
    "### Revisiting Micrograd just to have a better understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "486a9d20-4e79-4df9-b973-6fa916b745fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class Value:\n",
    "    \n",
    "    def __init__(self, data, _children=(), _op=''):\n",
    "        self.data = data\n",
    "        self.grad = 0.0\n",
    "        self._backward = lambda : None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f'Value(data = {self.data})'\n",
    "\n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data + other.data, (self, other), '+')\n",
    "\n",
    "        def _backward():\n",
    "            # by chain rule, self.grad = local_derivative * output.grad\n",
    "            self.grad += 1 * out.grad\n",
    "            other.grad += 1 * out.grad\n",
    "\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "\n",
    "    def __radd__(self, other):\n",
    "        return self+other\n",
    "\n",
    "    def __neg__(self):\n",
    "        return self * (-1)\n",
    "\n",
    "    def __sub__(self, other):\n",
    "        return self + (-other)\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data * other.data, (self, other), '*')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += other.data * out.grad\n",
    "            other.grad += self.data * out.grad\n",
    "\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "\n",
    "    def __rmul__(self, other):\n",
    "        return self * other\n",
    "\n",
    "    def __pow__(self, other):\n",
    "        assert isinstance(other, (int, float))\n",
    "        out = Value(self.data**other, (self, ), f'**{other}')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad = (other * self.data**(other-1) ) * out.grad\n",
    "    \n",
    "        out._backward = _backward\n",
    "    \n",
    "        return out\n",
    "    \n",
    "\n",
    "    def __truediv__(self, other):\n",
    "        return self * other**-1\n",
    "        \n",
    "\n",
    "    def tanh(self):\n",
    "        n = self.data\n",
    "        t = (math.exp(2*n) - 1) / (math.exp(2*n) + 1 )\n",
    "        out = Value(t, (self,), 'tanh')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += (1 - t**2) * out.grad\n",
    "\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "\n",
    "    def exp(self):\n",
    "        exp = math.exp(self.data)\n",
    "        out = Value(exp, (self,), 'exp')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += out.data * out.grad\n",
    "\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "    \n",
    "    def backward(self):\n",
    "        # do topological sorting and run back propagation\n",
    "\n",
    "        def dfs(val, visited, topo_sorted) -> None:\n",
    "            if val in visited : return\n",
    "        \n",
    "            visited.add(val)\n",
    "            # print(val)\n",
    "        \n",
    "            for k in val._prev:\n",
    "                dfs(k, visited, topo_sorted)\n",
    "        \n",
    "            topo_sorted.append(val)\n",
    "        \n",
    "        def build_topo(out) -> list:\n",
    "            topo_sorted = []\n",
    "            visited = set()\n",
    "            dfs(out, visited, topo_sorted)\n",
    "            topo_sorted.reverse()\n",
    "            return topo_sorted\n",
    "            \n",
    "        \n",
    "        topo = build_topo(self)\n",
    "        \n",
    "        self.grad = 1\n",
    "        \n",
    "        for op in topo:\n",
    "            op._backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb172297-d8d1-43b6-a55d-6e753b3dd287",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{Value(data = 4.0), Value(data = 6.0)}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(2.0)\n",
    "b = Value(3.0)\n",
    "c = Value(4.0)\n",
    "d = a*b + c\n",
    "d._prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f3c9b2bb-bb2b-4ec1-b772-3d1b393482a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'+'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d._op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "faac230d-ab96-4e39-8441-32867abe2062",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value(data = 10.0) {Value(data = 4.0), Value(data = 6.0)} +\n",
      "Value(data = 6.0) {Value(data = 3.0), Value(data = 2.0)} *\n"
     ]
    }
   ],
   "source": [
    "def print_graph(val):\n",
    "    if val._prev:\n",
    "        print(val, val._prev, val._op)\n",
    "        for child in val._prev:\n",
    "            print_graph(child)\n",
    "\n",
    "print_graph(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "e32fea44-dea8-4479-ac9d-8e7338a2847f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# neuron\n",
    "import math\n",
    "\n",
    "# inputs\n",
    "x1 = Value(2.0)\n",
    "x2 = Value(0.0)\n",
    "# weights\n",
    "w1 = Value(-3.0)\n",
    "w2 = Value(1.0)\n",
    "# bias of the neuron\n",
    "b = Value(0.881)\n",
    "# x1w1 + x2w2 + b\n",
    "x1w1 = x1*w1\n",
    "x2w2 = x2*w2\n",
    "x1w1x2w2 = x1w1 + x2w2 \n",
    "n = x1w1x2w2 + b\n",
    "# o = n.tanh()\n",
    "e = (2*n).exp()\n",
    "o = (e-1) / (e+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fcc9e21-ab44-453d-a72d-e1e8ac5be2b3",
   "metadata": {},
   "source": [
    "#### Manual Back Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "97ef4024-141b-4922-a8f4-5c3e35f2f51b",
   "metadata": {},
   "outputs": [],
   "source": [
    "o.grad = 1\n",
    "o._backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9d835f04-37c7-4bfc-9072-7bc0a49aa03d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00014312714365083412"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "28ac75d8-897f-4b3c-bfa9-447264ffb761",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00014312714365083412"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n._backward()\n",
    "x1w1x2w2.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "20679039-a0e0-4f58-a7f9-b78ce6121202",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00014312714365083412"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1w1x2w2._backward()\n",
    "x1w1.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e479b82e-a9f3-4dba-acc1-dd849e473e27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1w1._backward()\n",
    "w1.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7df717a5-1d03-4670-be67-83e53265aa4a",
   "metadata": {},
   "source": [
    "#### Auto Back Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "e222b624-815a-4d2c-b381-013ae653c3b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00028625428730212794"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "cffd201f-e90c-43ff-bae8-8c91adca872c",
   "metadata": {},
   "outputs": [],
   "source": [
    "o.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "006cd6e9-0ba8-4c1d-a56d-f654286ee548",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value(data = 4.0)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2*a"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bbbab64-6341-42d7-847a-6b851e0a25e1",
   "metadata": {},
   "source": [
    "### Building Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "abf3438a-021c-4a2a-836f-1566cfe965a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value(data = 0.7522054968986874)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "class Neuron:\n",
    "    def __init__(self, nin):\n",
    "        self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "        self.b = Value(random.uniform(-1,1))\n",
    "\n",
    "    def __call__(self, x):\n",
    "        act = sum(wi*xi for wi, xi in zip(self.w, x)) + self.b\n",
    "        out = act.tanh()\n",
    "        return out\n",
    "\n",
    "    def parameters(self):\n",
    "        return self.w + [self.b]\n",
    "\n",
    "\n",
    "class Layer:\n",
    "    def __init__(self, nin, nout):\n",
    "        self.neurons = [Neuron(nin) for _ in range(nout)] \n",
    "\n",
    "    def __call__(self, x):\n",
    "        outs = [n(x) for n in self.neurons]\n",
    "        return outs[0] if len(outs) == 1 else outs\n",
    "\n",
    "    def parameters(self):\n",
    "        return [p for neuron in self.neurons for p in neuron.parameters()]\n",
    "\n",
    "\n",
    "class MLP:\n",
    "    def __init__(self, nin, nout):\n",
    "        sz = [nin] + nout\n",
    "        self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nout))]\n",
    "\n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def parameters(self):\n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n",
    "        \n",
    "        \n",
    "\n",
    "x = [2.0, 3.0, -1.0]\n",
    "mlp = MLP(3, [4, 4, 1])\n",
    "mlp(x)\n",
    "\n",
    "# mlp.parameters()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "f3956c7d-94f8-4b61-9426-04bb8d7b807a",
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [\n",
    "    [2.0, 3.0, -1.0],\n",
    "    [3.0, -1.0, 0.5],\n",
    "    [0.5, 1.0, 1.0],\n",
    "    [1.0, 1.0, -1.0]\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "a1efae11-a0e0-4a29-9955-65f2df84211d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss on step 0 is 2.6541134189807596\n",
      "loss on step 1 is 0.9795115724683165\n",
      "loss on step 2 is 0.6512208412708855\n",
      "loss on step 3 is 0.47795152765428855\n",
      "loss on step 4 is 0.37178866211972605\n",
      "loss on step 5 is 0.30122078969614474\n",
      "loss on step 6 is 0.25151957480179454\n",
      "loss on step 7 is 0.21490871409074142\n",
      "loss on step 8 is 0.1869696140437594\n",
      "loss on step 9 is 0.16503374747862634\n",
      "loss on step 10 is 0.1474062731871071\n",
      "loss on step 11 is 0.13296472852090085\n",
      "loss on step 12 is 0.12093910782359811\n",
      "loss on step 13 is 0.11078516667123652\n",
      "loss on step 14 is 0.10210823618864609\n",
      "loss on step 15 is 0.09461568296049513\n",
      "loss on step 16 is 0.08808627306139548\n",
      "loss on step 17 is 0.08234986434643338\n",
      "loss on step 18 is 0.07727360546090878\n",
      "loss on step 19 is 0.07275234683557791\n"
     ]
    }
   ],
   "source": [
    "for k in range(20):\n",
    "    # forward pass\n",
    "    ypred = [mlp(x) for x in xs]\n",
    "    loss = sum((yout-ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "    print(f'loss on step {k} is {loss.data}')\n",
    "\n",
    "    # backward pass\n",
    "    for p in mlp.parameters():\n",
    "        p.grad = 0.0\n",
    "    loss.backward()\n",
    "\n",
    "    # update\n",
    "    for p in mlp.parameters():\n",
    "        p.data += -0.05 * p.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "4bcdb6ae-4af3-4079-8f5f-558c958f7841",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Value(data = 1.0), Value(data = -1.0), Value(data = -1.0), Value(data = 1.0)]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f6e6e78-97f6-448d-a60f-3ce19c38a354",
   "metadata": {},
   "source": [
    "### Finally completed the video & the implementation.\n",
    "Next steps:\n",
    "1. complete the assignment\n",
    "2. Check the source code & especially the demo part where there's some implementation of different type of data set with decision boundary.\n",
    "3. Move confidently to 'MakeMore'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fc71009-e9d4-4380-b291-456cdbeb36c4",
   "metadata": {},
   "source": [
    "### Assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "a3e7dc69-23e5-437b-b6de-46fa6a786f0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK for dim 0: expected -12.353553390593273, yours returns -12.353559348809995\n",
      "OK for dim 1: expected 10.25699027111255, yours returns 10.256991666679482\n",
      "OK for dim 2: expected 0.0625, yours returns 0.062499984743169534\n"
     ]
    }
   ],
   "source": [
    "# here is a mathematical expression that takes 3 inputs and produces one output\n",
    "from math import sin, cos\n",
    "\n",
    "def f(a, b, c):\n",
    "  return -a**3 + sin(3*b) - 1.0/c + b**2.5 - a**0.5\n",
    "    \n",
    "# expected answer is the list of \n",
    "ans = [-12.353553390593273, 10.25699027111255, 0.0625]\n",
    "\n",
    "\n",
    "# now estimate the gradient numerically without any calculus, using# the approximation we used in the video.\n",
    "# you should not call the function df from the last cell# -----------\n",
    "def get_numerical_grad(a, b, c):\n",
    "    h = 0.000001\n",
    "    df_da = (f(a+h, b, c) - f(a, b, c)) / h\n",
    "    df_db = (f(a, b+h, c) - f(a, b, c)) / h\n",
    "    df_dc = (f(a, b, c+h) - f(a, b, c)) / h\n",
    "    return [df_da, df_db, df_dc]\n",
    "    \n",
    "numerical_grad = get_numerical_grad(2, 3, 4) # TODO# -----------\n",
    "\n",
    "for dim in range(3):  \n",
    "    ok = 'OK' if abs(numerical_grad[dim] - ans[dim]) < 1e-5 else 'WRONG!'  \n",
    "    print(f\"{ok} for dim {dim}: expected {ans[dim]}, yours returns {numerical_grad[dim]}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "896c1dd0-3bb5-4bac-9fe4-7b73d9f44a9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK for dim 0: expected -12.353553390593273, yours returns -12.353554401639766\n",
      "OK for dim 1: expected 10.25699027111255, yours returns 10.256994551617105\n",
      "OK for dim 2: expected 0.0625, yours returns 0.06250000390650712\n"
     ]
    }
   ],
   "source": [
    "# there is an alternative formula that provides a much better numerical \n",
    "# approximation to the derivative of a function.\n",
    "# learn about it here: https://en.wikipedia.org/wiki/Symmetric_derivative\n",
    "# implement it. confirm that for the same step size h this version gives a\n",
    "# better approximation.\n",
    "\n",
    "# -----------\n",
    "def get_numerical_grad2(a, b, c):\n",
    "    h = 0.001\n",
    "    df_da = (f(a+h, b, c) - f(a-h, b, c)) / (2*h)\n",
    "    df_db = (f(a, b+h, c) - f(a, b-h, c)) / (2*h)\n",
    "    df_dc = (f(a, b, c+h) - f(a, b, c-h)) / (2*h)\n",
    "    return [df_da, df_db, df_dc]\n",
    "    \n",
    "\n",
    "numerical_grad2 = get_numerical_grad2(2, 3, 4) # TODO\n",
    "# -----------\n",
    "\n",
    "for dim in range(3):\n",
    "  ok = 'OK' if abs(numerical_grad2[dim] - ans[dim]) < 1e-5 else 'WRONG!'\n",
    "  print(f\"{ok} for dim {dim}: expected {ans[dim]}, yours returns {numerical_grad2[dim]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab7b4c15-cd41-4fa1-b58d-74ee5b70eaed",
   "metadata": {},
   "source": [
    "#### Support for Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "35df2353-6ff8-4ee8-b921-630b97974856",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Value class starter code, with many functions taken out\n",
    "from math import exp, log\n",
    "\n",
    "class Value:\n",
    "  \n",
    "  def __init__(self, data, _children=(), _op='', label=''):\n",
    "    self.data = data\n",
    "    self.grad = 0.0\n",
    "    self._backward = lambda: None\n",
    "    self._prev = set(_children)\n",
    "    self._op = _op\n",
    "    self.label = label\n",
    "\n",
    "  def __repr__(self):\n",
    "    return f\"Value(data={self.data})\"\n",
    "      \n",
    "  \n",
    "  def __add__(self, other): # exactly as in the video\n",
    "    other = other if isinstance(other, Value) else Value(other)\n",
    "    out = Value(self.data + other.data, (self, other), '+')\n",
    "    \n",
    "    def _backward():\n",
    "      self.grad += 1.0 * out.grad\n",
    "      other.grad += 1.0 * out.grad\n",
    "        \n",
    "    out._backward = _backward\n",
    "    \n",
    "    return out\n",
    "\n",
    "  def __radd__(self, other):\n",
    "      return self+other\n",
    "\n",
    "  # def __sub__(self, other):\n",
    "  #     other = other if isinstance(other, Value) else Value(other)\n",
    "  #     data = self.data-other.data\n",
    "  #     out = Value(data, (self, other), '-')\n",
    "\n",
    "  #     def _backward():\n",
    "  #         self.grad += 1.0 * out.grad\n",
    "  #         other.grad += -1.0 * out.grad\n",
    "\n",
    "  #     out._backward = _backward\n",
    "\n",
    "  #     return out\n",
    "\n",
    "\n",
    "  def __mul__(self, other):\n",
    "      other = other if isinstance(other, Value) else Value(other)\n",
    "      data = self.data * other.data\n",
    "      out = Value(data, (self, other), '*')\n",
    "\n",
    "      def _backward():\n",
    "          self.grad += other.data * out.grad\n",
    "          other.grad += self.data * out.grad\n",
    "\n",
    "      out._backward = _backward\n",
    "      \n",
    "      return out \n",
    "\n",
    "  def __rmul__(self, other):\n",
    "      return self*other\n",
    "\n",
    "  def __neg__(self):\n",
    "      return self*(-1)\n",
    "\n",
    "    \n",
    "  def __pow__(self, other):\n",
    "      assert isinstance(other, (int, float))\n",
    "      data = self.data**other\n",
    "      out = Value(data, (self, ), '**')\n",
    "\n",
    "      def _backward():\n",
    "          self.grad += other * self.data**(other-1) * out.grad\n",
    "\n",
    "      out._backward = _backward\n",
    "      \n",
    "      return out\n",
    "      \n",
    "    \n",
    "  def __truediv__(self, other):\n",
    "      return self * (other**-1)\n",
    "\n",
    "  def __rtruediv__(self, other):\n",
    "      return other * (self**-1)\n",
    "    \n",
    "\n",
    "  def exp(self):\n",
    "      data = math.exp(self.data)\n",
    "      out = Value(data, (self,), 'exp')\n",
    "    \n",
    "      def _backward():\n",
    "        self.grad += out.data * out.grad\n",
    "    \n",
    "      out._backward = _backward\n",
    "    \n",
    "      return out\n",
    "\n",
    "  def log(self):\n",
    "      data = math.log(self.data)\n",
    "      out = Value(data, (self,), 'log')\n",
    "\n",
    "      def _backward():\n",
    "          self.grad += (1/self.data) * out.grad \n",
    "\n",
    "      out._backward = _backward\n",
    "      \n",
    "      return out\n",
    "\n",
    "      \n",
    "  # ------\n",
    "  # re-implement all the other functions needed for the exercises below\n",
    "  # your code here\n",
    "  # TODO\n",
    "  # ------\n",
    "\n",
    "  def backward(self): # exactly as in video  \n",
    "    topo = []\n",
    "    visited = set()\n",
    "    def build_topo(v):\n",
    "      if v not in visited:\n",
    "        visited.add(v)\n",
    "        for child in v._prev:\n",
    "          build_topo(child)\n",
    "        topo.append(v)\n",
    "    build_topo(self)\n",
    "    \n",
    "    self.grad = 1.0\n",
    "    for node in reversed(topo):\n",
    "      node._backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "9c57fd57-42aa-4a73-90f3-6f51854efa25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1755153626167147\n",
      "OK for dim 0: expected 0.041772570515350445, yours returns 0.041772570515350445\n",
      "OK for dim 1: expected 0.8390245074625319, yours returns 0.8390245074625319\n",
      "OK for dim 2: expected 0.005653302662216329, yours returns 0.005653302662216329\n",
      "OK for dim 3: expected -0.8864503806400986, yours returns -0.8864503806400986\n"
     ]
    }
   ],
   "source": [
    "# without referencing our code/video __too__ much, make this cell work\n",
    "# you'll have to implement (in some cases re-implemented) a number of functions\n",
    "# of the Value object, similar to what we've seen in the video.\n",
    "# instead of the squared error loss this implements the negative log likelihood\n",
    "# loss, which is very often used in classification.\n",
    "\n",
    "# this is the softmax function\n",
    "# https://en.wikipedia.org/wiki/Softmax_function\n",
    "def softmax(logits):\n",
    "  counts = [logit.exp() for logit in logits]\n",
    "  denominator = sum(counts)\n",
    "  out = [c / denominator for c in counts]\n",
    "  return out\n",
    "\n",
    "# this is the negative log likelihood loss function, pervasive in classification\n",
    "logits = [Value(0.0), Value(3.0), Value(-2.0), Value(1.0)]\n",
    "probs = softmax(logits)\n",
    "loss = -probs[3].log() # dim 3 acts as the label for this input example\n",
    "loss.backward()\n",
    "print(loss.data)\n",
    "\n",
    "ans = [0.041772570515350445, 0.8390245074625319, 0.005653302662216329, -0.8864503806400986]\n",
    "for dim in range(4):\n",
    "  ok = 'OK' if abs(logits[dim].grad - ans[dim]) < 1e-5 else 'WRONG!'\n",
    "  print(f\"{ok} for dim {dim}: expected {ans[dim]}, yours returns {logits[dim].grad}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "57f75d34-1800-4c6b-add0-6c69a163ae60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.1755, dtype=torch.float64)\n",
      "OK for dim 0: expected 0.041772570515350445, yours returns 0.041772570515350445\n",
      "OK for dim 1: expected 0.8390245074625319, yours returns 0.8390245074625319\n",
      "OK for dim 2: expected 0.005653302662216329, yours returns 0.005653302662216329\n",
      "OK for dim 3: expected -0.8864503806400986, yours returns -0.8864503806400988\n"
     ]
    }
   ],
   "source": [
    "# verify the gradient using the torch library\n",
    "# torch should give you the exact same gradient\n",
    "import torch\n",
    "\n",
    "logits = [torch.tensor(0.0).double(), torch.tensor(3.0).double(), torch.tensor(-2.0).double(), torch.tensor(1.0).double()]\n",
    "\n",
    "for tensor in logits:\n",
    "    tensor.requires_grad = True \n",
    "\n",
    "probs = softmax(logits)\n",
    "loss = -probs[3].log() # dim 3 acts as the label for this input example\n",
    "loss.backward()\n",
    "print(loss.data)\n",
    "\n",
    "ans = [0.041772570515350445, 0.8390245074625319, 0.005653302662216329, -0.8864503806400986]\n",
    "for dim in range(4):\n",
    "  ok = 'OK' if abs(logits[dim].grad - ans[dim]) < 1e-5 else 'WRONG!'\n",
    "  print(f\"{ok} for dim {dim}: expected {ans[dim]}, yours returns {logits[dim].grad}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f62380ab-ff67-464e-a888-b06d227498f6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}