L2-Regularization on MNIST Database.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# L2-Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import struct as st\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "%matplotlib inline\n",
    "np.set_printoptions(threshold=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train,y_train), (x_test,y_test) = tf.keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0xb3cad5978>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAN+UlEQVR4nO3dX4hcdZrG8edRZ4jGkeimjUFlMxtzI6tR6QRBGV0kQ/SmlagY4p8F2Ygm4GBAxYXohRdBdAaFVYz/JiOzjiOjmAvdnayOyIiMdkI0UXE10qIx6XSIMAmCWZN3L/pk6Ildv2qrTv1J3u8Hmqo6b506L4c8OVXnV6d+jggBOPod0+sGAHQHYQeSIOxAEoQdSIKwA0kc182NzZw5M+bMmdPNTQKpjIyMaPfu3Z6s1lbYbS+W9LCkYyU9GRFrSs+fM2eOhoeH29kkgILBwcGGtZbfxts+VtJ/SLpc0tmSlto+u9XXA9BZ7XxmXyjp04j4LCL2S/qdpKF62gJQt3bCfrqkLyY8/rJa9ndsL7c9bHt4bGysjc0BaEfHz8ZHxNqIGIyIwYGBgU5vDkAD7YR9u6QzJzw+o1oGoA+1E/Z3Jc2z/VPbP5Z0naT19bQFoG4tD71FxHe2V0r6b40PvT0dER/U1hmAWrU1zh4Rr0h6paZeAHQQX5cFkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgibZmcQV6af/+/cX64sWLG9a2bdtWXPe9994r1mfMmFGs96O2wm57RNJeSQckfRcRg3U0BaB+dRzZ/yUidtfwOgA6iM/sQBLthj0k/dH2RtvLJ3uC7eW2h20Pj42Ntbk5AK1qN+wXR8QFki6XtML2zw5/QkSsjYjBiBgcGBhoc3MAWtVW2CNie3W7S9JLkhbW0RSA+rUcdtvTbf/k0H1JP5e0ta7GANSrnbPxsyS9ZPvQ6/xnRPxXLV3hiLF379626iXTp08v1jdu3Fisv/HGGw1r8+fPL657/PHHF+tHopbDHhGfSSrvMQB9g6E3IAnCDiRB2IEkCDuQBGEHkuAS16PAjh07GtYeeeSR4rojIyNtbbvZ8FezS0lLHnrooWJ9y5YtxXpENKzNmzevuO7BgweL9SMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9qPAW2+91bD2wAMPdHTb06ZNK9Zvv/32hrUXX3yxuO6qVata6umQ6vLrSa1YsaK47tF4iStHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2I8Cjjz5arN95550tv/Ydd9xRrM+aNatYv+2224r1E044oWGt2Tj6ggULivXR0dFi/bTTTmtYu+iii4rrHo04sgNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoyzHwH27dtXrH/zzTcNa2eddVZx3XvvvbdYP/HEE4v1Zvbs2dOwdv/99xfX3blzZ7HebErnxx57rGHtuOPy/dNvemS3/bTtXba3Tlh2iu0Ntj+pbk/ubJsA2jWVt/G/lrT4sGV3S3otIuZJeq16DKCPNQ17RLwp6fD3YkOS1lX310m6sua+ANSs1RN0syLi0ARjOyU1/AK17eW2h20Pj42Ntbg5AO1q+2x8jM+e13AGvYhYGxGDETE4MDDQ7uYAtKjVsI/ani1J1e2u+loC0Amthn29pJuq+zdJermedgB0StPBRtvPSbpU0kzbX0q6V9IaSb+3fbOkzyVd28kms7v22vLufeGFFxrWNm3aVFx39erVxfqaNWuK9W+//bZYL10v/+yzzxbXbfax7+GHHy7Wh4aGivVsmoY9IpY2KF1Wcy8AOoivywJJEHYgCcIOJEHYgSQIO5BEvuv8jkBnnHFGsX7ZZY0HRpoNvTWbNnnp0kaDMeOWLVtWrG/btq1YL2n2E9pLlixp+bUz4sgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzn4EaPazxzNmzGj5tb/44oti/cILLyzWx3+oqDHbDWvNpppetGhRsY4fhiM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBOPtRoNm0zL10/fXXN6ytWrWquO5JJ51UdzupcWQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZz8CHDx4sFjfsGFDw1qz683bdcMNNxTr69at6+j2MXVNj+y2n7a9y/bWCcvus73d9ubq74rOtgmgXVN5G/9rSYsnWf6riDiv+nul3rYA1K1p2CPiTUl7utALgA5q5wTdStvvV2/zT270JNvLbQ/bHh4bG2tjcwDa0WrYH5M0V9J5knZIeqjREyNibUQMRsTgwMBAi5sD0K6Wwh4RoxFxICIOSnpC0sJ62wJQt5bCbnv2hIdXSdra6LkA+kPTcXbbz0m6VNJM219KulfSpbbPkxSSRiTd0sEe07v11luL9SeffLJhrfS77XXo9OujPk3DHhFLJ1n8VAd6AdBBfF0WSIKwA0kQdiAJwg4kQdiBJLjEtQv27t1brD///PPF+hNPPFGsl4a/LrnkkuK6CxYsKNYffPDBYv2rr74q1tE/OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs3fBxo0bi/VbbmnvCuHSOPyyZcuK67799tvFerNx9vnz5xfr6B8c2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZa/Dxxx8X60uWLGnr9ZuN059zzjkNa/v27Suuu2LFipZ6OmTu3LltrY/u4cgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzl6DV199tVj/+uuvi/WrrrqqWD///POL9QMHDjSsvf7668V19+zZU6xHRLE+e/bsYh39o+mR3faZtv9k+0PbH9i+vVp+iu0Ntj+pbk/ufLsAWjWVt/HfSVoVEWdLulDSCttnS7pb0msRMU/Sa9VjAH2qadgjYkdEbKru75X0kaTTJQ1JWlc9bZ2kKzvVJID2/aATdLbnSDpf0l8kzYqIHVVpp6RZDdZZbnvY9vDY2FgbrQJox5TDbvtESX+Q9IuI+OvEWoyfxZn0TE5ErI2IwYgYHBgYaKtZAK2bUtht/0jjQf9tRLxYLR61Pbuqz5a0qzMtAqhD06E3j88H/JSkjyLilxNK6yXdJGlNdftyRzo8AhxzTPn/zNKUylOpl4bWJOmdd95pWLvmmmuK686cObNYv+uuu4r1oaGhYh39Yyrj7BdJukHSFtubq2X3aDzkv7d9s6TPJV3bmRYB1KFp2CPiz5IaHXouq7cdAJ3C12WBJAg7kARhB5Ig7EAShB1IgktcazA6OtrW+qeeemqxfvXVVxfr69evb3nbzS7PveCCC1p+bfQXjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7DU499xz21r/8ccfL9ab/Zxz6ReAVq9eXVy3NN0zji4c2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZa9Dst9OfeeaZYn3lypXF+qJFi4r10m/DX3fddcV1kQdHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IYirzs58p6TeSZkkKSWsj4mHb90n6N0lj1VPviYhXOtVoP5s2bVqxfuONN7ZVB+owlS/VfCdpVURssv0TSRttb6hqv4qIBzvXHoC6TGV+9h2SdlT399r+SNLpnW4MQL1+0Gd223MknS/pL9Wilbbft/207ZMbrLPc9rDt4bGxscmeAqALphx22ydK+oOkX0TEXyU9JmmupPM0fuR/aLL1ImJtRAxGxGDpt9IAdNaUwm77RxoP+m8j4kVJiojRiDgQEQclPSFpYefaBNCupmG3bUlPSfooIn45YfnsCU+7StLW+tsDUJepnI2/SNINkrbY3lwtu0fSUtvnaXw4bkTSLR3pEEAtpnI2/s+SPEkp5Zg6cKTiG3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkHBHd25g9JunzCYtmStrdtQZ+mH7trV/7kuitVXX29o8RMenvv3U17N/buD0cEYM9a6CgX3vr174kemtVt3rjbTyQBGEHkuh12Nf2ePsl/dpbv/Yl0VurutJbTz+zA+ieXh/ZAXQJYQeS6EnYbS+2/bHtT23f3YseGrE9YnuL7c22h3vcy9O2d9neOmHZKbY32P6kup10jr0e9Xaf7e3Vvtts+4oe9Xam7T/Z/tD2B7Zvr5b3dN8V+urKfuv6Z3bbx0r6X0mLJH0p6V1JSyPiw6420oDtEUmDEdHzL2DY/pmkfZJ+ExH/XC17QNKeiFhT/Ud5ckTc1Se93SdpX6+n8a5mK5o9cZpxSVdK+lf1cN8V+rpWXdhvvTiyL5T0aUR8FhH7Jf1O0lAP+uh7EfGmpD2HLR6StK66v07j/1i6rkFvfSEidkTEpur+XkmHphnv6b4r9NUVvQj76ZK+mPD4S/XXfO8h6Y+2N9pe3utmJjErInZU93dKmtXLZibRdBrvbjpsmvG+2XetTH/eLk7Qfd/FEXGBpMslrajervalGP8M1k9jp1OaxrtbJplm/G96ue9anf68Xb0I+3ZJZ054fEa1rC9ExPbqdpekl9R/U1GPHppBt7rd1eN+/qafpvGebJpx9cG+6+X0570I+7uS5tn+qe0fS7pO0voe9PE9tqdXJ05ke7qkn6v/pqJeL+mm6v5Nkl7uYS9/p1+m8W40zbh6vO96Pv15RHT9T9IVGj8jv03Sv/eihwZ9/ZOk96q/D3rdm6TnNP627v80fm7jZkn/IOk1SZ9I+h9Jp/RRb89K2iLpfY0Ha3aPertY42/R35e0ufq7otf7rtBXV/YbX5cFkuAEHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4k8f8/3S/CFebGuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "image_index = 17\n",
    "print(y_train[image_index])\n",
    "plt.imshow(x_train[image_index],cmap = 'Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_flattened = np.reshape(x_train,(x_train.shape[0],-1))\n",
    "x_test_flattened = np.reshape(x_test,(x_test.shape[0],-1))\n",
    "y_train = y_train.reshape(y_train.shape[0],1)\n",
    "y_test = y_test.reshape(y_test.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_flattened = x_train_flattened.astype('float32')\n",
    "x_test_flattened = x_test_flattened.astype('float32')\n",
    "y_train = y_train.astype('float32')\n",
    "y_test = y_test.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_flattened = x_train_flattened/255\n",
    "x_test_flattened = x_test_flattened/255\n",
    "X_train = x_train_flattened.T\n",
    "X_test = x_test_flattened.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_labels = 10\n",
    "Y_train = np.zeros((n_labels,y_train.shape[0]))\n",
    "Y_test = np.zeros((n_labels,y_test.shape[0]))\n",
    "\n",
    "for i in range(len(y_train)):\n",
    "    Y_train[int(y_train[i])][i] = 1\n",
    "    \n",
    "for i in range(len(y_test)):\n",
    "    Y_test[int(y_test[i])][i] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(z):\n",
    "    return np.maximum(z,0)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1.0/(1.0+np.exp(-z))\n",
    "\n",
    "def tanh(z):\n",
    "    return np.tanh(z)\n",
    "\n",
    "def atan(z):\n",
    "    return np.arctan(z)\n",
    "\n",
    "def softmax(z):\n",
    "    return np.divide(np.exp(z-np.max(z,axis=0)),np.sum(np.exp(z-np.max(z,axis=0)),axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activation_function(z,activation):\n",
    "    if activation == 'relu':\n",
    "        return relu(z)\n",
    "    elif activation == 'tanh':\n",
    "        return tanh(z)\n",
    "    elif activation == 'atan':\n",
    "        return atan(z)\n",
    "    elif activation == 'sigmoid':\n",
    "        return sigmoid(z)\n",
    "    elif activation == 'softmax':\n",
    "        return softmax(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters(X,nn_architecture,nn_activations,learning_rate,lam):\n",
    "    cache = {}\n",
    "    h_params = {'nn_arch': nn_architecture,\n",
    "               'lr': learning_rate,\n",
    "               'nn_acti': nn_activations,\n",
    "               'lam': lam}\n",
    "    W = np.random.rand(nn_architecture[0],X.shape[0]) * 0.01\n",
    "    B = np.random.rand(nn_architecture[0],1)\n",
    "    cache.update({'W'+str(0): W, 'B'+str(0):B})\n",
    "    for i in range(1,len(nn_architecture)):\n",
    "        W = np.random.rand(nn_architecture[i],nn_architecture[i-1]) * 0.01\n",
    "        B = np.random.rand(nn_architecture[i],1)\n",
    "        cache.update({'W'+str(i): W, 'B'+str(i):B})\n",
    "        \n",
    "    return cache,h_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_nn_network(X,cache,h_params):\n",
    "    A = X\n",
    "    for i in range(len(h_params['nn_arch'])):\n",
    "        X = A\n",
    "        Z = np.matmul(cache['W'+str(i)],X) + cache['B'+str(i)]\n",
    "        A = activation_function(Z,h_params['nn_acti'][i])\n",
    "        cache.update({'A'+str(i): A,\n",
    "                     'Z'+str(i): Z})\n",
    "    return A,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_function(Y_pred,Y_true,cost):\n",
    "    if cost == 'mae':\n",
    "        return 1.0/Y_pred.shape[1] * np.sum(np.abs(Y_pred-Y_true),axis = 1,keepdims = True)\n",
    "    elif cost == 'mse':\n",
    "        return 0.5/Y_pred.shape[1] * np.sum((Y_pred-Y_true)**2,axis = 1,keepdims = True)\n",
    "    elif cost == 'cross-entropy':\n",
    "        if Y_pred.shape[0] == 2:\n",
    "            return -1.0/Y_pred.shape[1] * np.sum(np.multiply(Y_true,np.log(Y_pred))+np.multiply((1.0-Y_true),np.log(1.0-Y_pred)),axis = 1, keepdims = True)\n",
    "        else:\n",
    "            return -1.0/Y_pred.shape[1] * np.sum(np.multiply(Y_true,np.log(Y_pred)),axis = 1,keepdims = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_function_derivative(Y_pred,Y_train,cost):\n",
    "    if cost == 'mae':\n",
    "        return 1.0/Y_pred.shape[1]\n",
    "    elif cost == 'mse':\n",
    "        return 1.0/Y_pred.shape[1] * (Y_pred - Y_train)\n",
    "    elif cost == 'cross-entropy':\n",
    "        if Y_pred.shape[0] == 2:\n",
    "            return 1.0/Y_pred.shape[1] * np.divide((Y_pred-Y_train),((1-Y_pred)*(Y_pred)))\n",
    "        else:\n",
    "            return 1.0/Y_pred.shape[1] * np.divide(Y_train,Y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activation_derivative(Z,activation):\n",
    "    if activation == 'relu':\n",
    "        return (lambda X: X>=0)(Z) * 1\n",
    "    elif activation == 'atan':\n",
    "        return 1.0/(Z**2 +1.0)\n",
    "    elif activation == 'tanh':\n",
    "        return 1.0-(tanh(Z))**2\n",
    "    elif activation == 'sigmoid':\n",
    "        return np.multiply(sigmoid(Z),1.0-sigmoid(Z))\n",
    "    elif activation == 'softmax':\n",
    "        return np.multiply(softmax(Z),np.eye(Z.shape[0])-softmax(Z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params):\n",
    "    \n",
    "    # Obtain the pre-multiplier (also dB)\n",
    "    if h_params[\"nn_acti\"][-1] == 'softmax' and cost == 'cross-entropy':\n",
    "        pre_mul = Y_pred-Y_true\n",
    "    else:\n",
    "        # First find the derivatives for the cost function\n",
    "        dJ = cost_function_derivative(Y_pred,Y_true,cost)\n",
    "        # Then find the activation derivative for the last layer\n",
    "        dA_dZ = activation_derivative(cache['A'+str(len(h_params['nn_arch'])-1)],h_params['nn_acti'][len(h_params['nn_arch'])-1])\n",
    "        pre_mul = np.matmul(dJ.T,dA_dZ)\n",
    "        \n",
    "    # Obtain the dW and dB\n",
    "    dW = np.matmul(pre_mul,cache['A'+str(len(h_params['nn_arch'])-2)].T)\n",
    "    dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "    # Store it in a cache\n",
    "    cache.update({'dW'+str(len(h_params['nn_arch'])-1): dW,\n",
    "                 'dB'+str(len(h_params['nn_arch'])-1): dB})\n",
    "\n",
    "    # Similarly for the middle layers\n",
    "    for i in range(len(h_params['nn_arch'])-2,0,-1):\n",
    "        pre_mul = np.matmul(cache['W'+str(i+1)].T,pre_mul)\n",
    "        dA_dZ = activation_derivative(cache['A'+str(i)],h_params['nn_acti'][i])\n",
    "        pre_mul = np.multiply(pre_mul,dA_dZ)\n",
    "        dW = np.matmul(pre_mul,cache['A'+str(i-1)].T)\n",
    "        dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "        cache.update({'dW'+str(i): dW,\n",
    "                     'dB'+str(i): dB})\n",
    "\n",
    "    # Similarly for the first layer\n",
    "    pre_mul = np.matmul(cache['W'+str(1)].T,pre_mul)\n",
    "    dA_dZ = activation_derivative(cache['A'+str(0)],h_params['nn_acti'][0])\n",
    "    pre_mul = np.multiply(pre_mul,dA_dZ)\n",
    "    dW = np.matmul(pre_mul,X.T)\n",
    "    dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "    cache.update({'dW'+str(0): dW,\n",
    "                 'dB'+str(0): dB})\n",
    "    \n",
    "    return cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_weights(cache,h_params,regularization):\n",
    "    if regularization == None:\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            cache['W'+str(i)] -= cache['dW'+str(i)] * h_params['lr']\n",
    "            cache['B'+str(i)] -= cache['dB'+str(i)] * h_params['lr']\n",
    "    elif regularization == 'L2':\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            cache['W'+str(i)] -= (cache['dW'+str(i)] + h_params['lam'] * cache['W'+str(i)]/cache['A0'].shape[1]) * h_params['lr']\n",
    "            cache['B'+str(i)] -= cache['dB'+str(i)] * h_params['lr']\n",
    "    return cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_network(X,Y_true,nn_architecture,nn_activations,cost,learning_rate,n_epoch,regularization,lam):\n",
    "    cache,h_params = initialize_parameters(X,nn_architecture,nn_activations,learning_rate,lam)\n",
    "    cost = []\n",
    "    if regularization == None:\n",
    "        for i in range(n_epoch):\n",
    "            Y_pred,cache = forward_nn_network(X,cache,h_params)\n",
    "            loss = cost_function(Y_pred,Y_true,cost)\n",
    "            print(np.sum(loss))\n",
    "            cache = backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params)\n",
    "            cache = update_weights(cache,h_params)\n",
    "            cost.append(np.sum(loss))\n",
    "    elif regularization == 'L2':\n",
    "        for i in range(n_epoch):\n",
    "            Y_pred,cache = forward_nn_network(X,cache,h_params)\n",
    "            loss = cost_function(Y_pred,Y_true,cost)\n",
    "            for j in range(len(h_params['nn_arch'])):\n",
    "                loss += 0.5 * h_params['lam'] * np.sum(np.square(cache['W'+str(j)])) / Y_pred.shape[1]\n",
    "            print(np.sum(loss))\n",
    "            cache = backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params)\n",
    "            cache = update_weights(cache,h_params,regularization)\n",
    "            cost.append(np.sum(loss))\n",
    "    return cache,h_params,cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "cache,h_params = initialize_parameters(X_train,[25,10],['sigmoid','softmax'],learning_rate =1e-4,lam = 1e-5)\n",
    "cost = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.677745853907209\n",
      "0.6770066041064273\n",
      "0.6762967755365556\n",
      "0.6756661194348292\n",
      "0.6750520226313065\n",
      "0.6745580408219105\n",
      "0.6740489250927671\n",
      "0.6737193724075915\n",
      "0.6732805384060687\n",
      "0.6731131422580012\n",
      "0.6727058085615558\n",
      "0.6727199729551256\n",
      "0.6723428585574773\n",
      "0.6725690021537989\n",
      "0.672219131342709\n",
      "0.672656890789042\n",
      "0.6722840562974883\n",
      "0.672958135152436\n",
      "0.6724841420771577\n",
      "0.6734623669689785\n",
      "0.6728052261409445\n",
      "0.6740816936042413\n",
      "0.6731821712461691\n",
      "0.6746651659498434\n",
      "0.6734553302091385\n",
      "0.6750619866869378\n",
      "0.6734877828934778\n",
      "0.675189700980012\n",
      "0.6732467040369869\n",
      "0.6750161684279695\n",
      "0.6727715444925194\n",
      "0.6745072467460999\n",
      "0.672098353946401\n",
      "0.6737125669209616\n",
      "0.6712891491750326\n",
      "0.6727477620014635\n",
      "0.670412279879706\n",
      "0.6717236849592059\n",
      "0.6695029962317095\n",
      "0.6706773458449884\n",
      "0.6685433971382975\n",
      "0.6696644284269053\n",
      "0.6676065994589907\n",
      "0.6687041938550802\n",
      "0.666734195767638\n",
      "0.6677628265393816\n",
      "0.6658776367972262\n",
      "0.666856735128601\n",
      "0.6650002854047016\n",
      "0.6659986226149676\n",
      "0.6641210308351851\n",
      "0.6651147812265502\n",
      "0.6632683044019523\n",
      "0.6642445143014837\n",
      "0.6624498023346873\n",
      "0.6634287369121746\n",
      "0.6616603499211086\n",
      "0.6626018416643294\n",
      "0.6608541414064042\n",
      "0.6618088646599296\n",
      "0.660043573993464\n",
      "0.6610744110950626\n",
      "0.6592480588123044\n",
      "0.6603583056124955\n",
      "0.6584730058656381\n",
      "0.6596769525021233\n",
      "0.65779691670569\n",
      "0.6590487779936449\n",
      "0.6572490131445541\n",
      "0.6583969399031671\n",
      "0.6567370156647754\n",
      "0.657727642241075\n",
      "0.656190886167845\n",
      "0.6570499788096853\n",
      "0.6555677899474615\n",
      "0.6563284400425363\n",
      "0.6548488658233298\n",
      "0.6555608969563305\n",
      "0.6541075920729635\n",
      "0.6548536859181686\n",
      "0.6534462031638169\n",
      "0.6542779048487397\n",
      "0.6528928139769096\n",
      "0.6537810800259946\n",
      "0.6523943158761775\n",
      "0.6533006817245441\n",
      "0.6518885364567492\n",
      "0.6528623258072263\n",
      "0.6514046958644469\n",
      "0.6524350255823335\n",
      "0.6509448486197514\n",
      "0.6518726212355264\n",
      "0.6503737736600511\n",
      "0.6511588750040943\n",
      "0.6496416104190496\n",
      "0.6504239864309098\n",
      "0.6488653368052548\n",
      "0.6497037257645841\n",
      "0.6481366634348633\n",
      "0.6489258027133661\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    Y_pred,cache = forward_nn_network(X_train,cache,h_params)\n",
    "    loss = cost_function(Y_pred,Y_train,'cross-entropy')\n",
    "    for j in range(len(h_params['nn_arch'])):\n",
    "        loss += 0.5 * h_params['lam'] * np.sum(np.square(cache['W'+str(j)])) / Y_train.shape[1]\n",
    "    print(np.sum(loss))\n",
    "    cache = backward_nn_network(X_train,Y_pred,Y_train,'cross-entropy',cache,h_params)\n",
    "    cache = update_weights(cache,h_params,'L2')\n",
    "    cost.append(np.sum(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1076ceef0>]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhc9X3v8fdXM9olS7Yl7zbC2GCMg21wjQlZWIMhBNICt0CzkAvXzS1paZInPJDc0pKkTZu0IaG0CQTSkIQkJEASym5C2EpikI1twMbGBoNlY1tetO/S9/4xR/JIGlljeUajM/68nmceneU3Z77HiM8c/c45v2PujoiIhF9OpgsQEZHUUKCLiGQJBbqISJZQoIuIZAkFuohIlohm6oMrKiq8qqoqUx8vIhJKq1ev3uvulYnWZSzQq6qqqK6uztTHi4iEkpm9M9Q6dbmIiGQJBbqISJZQoIuIZAkFuohIllCgi4hkCQW6iEiWUKCLiGSJUAb6y9v2s2lXY6bLEBEZU5IOdDOLmNkrZvZwgnX5ZnafmW0xs1VmVpXKIge6/Pt/4PzvPJfOjxARCZ3DOUK/Htg4xLprgAPuPge4FfiXIy1MREQOT1KBbmYzgI8Cdw3R5BLgnmD6fuAcM7MjL09ERJKV7BH6d4AbgJ4h1k8HtgO4exdQD0w84upERCRpwwa6mV0E7HH31Uf6YWa2wsyqzay6trb2SDcnIiJxkjlCPwO42My2Ab8Azjaznw5oswOYCWBmUaAM2DdwQ+5+p7svcfcllZUJR38UEZERGjbQ3f0md5/h7lXAFcDT7v6JAc0eAj4dTF8WtPGUVioiIoc04vHQzeyrQLW7PwTcDfzEzLYA+4kFv4iIjKLDCnR3fwZ4Jpi+OW55G3B5KgsTEZHDE8o7RUVEZDAFuohIllCgi4hkCQW6iEiWUKCLiGQJBbqISJZQoIuIZAkFuohIllCgi4hkCQW6iEiWUKCLiGQJBbqISJZQoIuIZAkFuohIllCgi4hkCQW6iEiWUKCLiGQJBbqISJZQoIuIZAkFuohIlhg20M2swMxeMrN1Zva6md2SoM3VZlZrZmuD17XpKVdERIYSTaJNO3C2uzeZWS7wgpk95u5/HNDuPnf/XOpLFBGRZAwb6O7uQFMwmxu8PJ1FiYjI4UuqD93MIma2FtgDrHT3VQmaXWpm683sfjObmdIqRURkWEkFurt3u/siYAaw1MwWDGjy30CVu58MrATuSbQdM1thZtVmVl1bW3skdYuIyACHdZWLu9cBvweWD1i+z93bg9m7gFOHeP+d7r7E3ZdUVlaOpF5iPUAiIjJQMle5VJpZeTBdCJwHvDGgzdS42YuBjaksMp7yXEQksWSucpkK3GNmEWJfAL9094fN7KtAtbs/BPyNmV0MdAH7gavTVbDyXEQksWSuclkPLE6w/Oa46ZuAm1JbWmI9OkQXEUkodHeKKs9FRBILX6Cr00VEJKHwBbryXEQkIQW6iEiWCF+gq8tFRCSh8AW68lxEJKHQBbouWxQRSSx0ga44FxFJLHyBrkQXEUkohIGuRBcRSSSEgZ7pCkRExqbwBXqmCxARGaNCF+jxV7mo+0VE5KDQBXp8hivPRUQOCl+gx3W66Jp0EZGDQhfo8Z3oinMRkYNCF+g9Hj+tSBcR6RW6QI/vclGei4gcFL5A10lREZGEQhfo8d0s6nIRETlo2EA3swIze8nM1pnZ62Z2S4I2+WZ2n5ltMbNVZlaVjmKh/1G5Al1E5KBkjtDbgbPdfSGwCFhuZssGtLkGOODuc4BbgX9JbZmJKc5FRA4aNtA9pimYzQ1eA7P0EuCeYPp+4Bwzs5RVGaffnaI96fgEEZFwSqoP3cwiZrYW2AOsdPdVA5pMB7YDuHsXUA9MTGWhvdTlIiKSWFKB7u7d7r4ImAEsNbMFI/kwM1thZtVmVl1bWzuSTfT700BxLiJy0GFd5eLudcDvgeUDVu0AZgKYWRQoA/YleP+d7r7E3ZdUVlaOqGBd5SIiklgyV7lUmll5MF0InAe8MaDZQ8Cng+nLgKc9TUMhqstFRCSxaBJtpgL3mFmE2BfAL939YTP7KlDt7g8BdwM/MbMtwH7girRVrMFcREQSGjbQ3X09sDjB8pvjptuAy1Nb2lD1HJzuUaCLiPQJ4Z2i8dNKdBGRXqEL9H6Dc2WwDhGRsSZ8gR5/hK4+FxGRPqEL9P7PFM1gISIiY0zoAr3f8LnqdBER6RO6QI+nHhcRkYNCF+i6U1REJLHQBbqeWCQiklj4Aj1+WokuItIndIHev8slg4WIiIwxoQt0XeUiIpJY6AI9vtOls0uBLiLSK3SBHn+E3tzRlblCRETGmNAFeny/eYsCXUSkT+gCPf7Klqb27gxWIiIytoQv0OOmm9vTd4T+ak09tY3tadu+iEiqhS7Q4y9bTGegf+z2F7j49hfStn0RkVQLXaDHH6I3p6nLpa0ztt336tvSsn0RkXQIXaDHd7mk66SoulpEJIxCF+g9/U6KpifQ9zYNHeg3PfgqP/7DtrR8rojIkRj2IdFmNhP4MTCZ2AHyne7+3QFtzgR+C7wdLHrQ3b+a2lJjvN9li+npctnb1DFo2Zp3D/DOvmZ+/tK7AHzq9Kq0fLaIyEgNG+hAF/BFd19jZqXAajNb6e4bBrR73t0vSn2J/cV3uTS2pecIfV+CI/Q/+88X+6ZL8pP5ZxMRGV3DJpO7vwe8F0w3mtlGYDowMNBHRW+XS2FuhLqWwUfSqRB/MvSMf36aixZO7be+q6cnLZ8rInIkDqsP3cyqgMXAqgSrTzezdWb2mJmdlILaEgsO0SeW5LG/OT2BXnOgtW96R10rdzz7Vr/1bZ09fVfCiIiMFUkHupmVAA8Af+vuDQNWrwGOcfeFwL8DvxliGyvMrNrMqmtra0dUcO8IixOL89ifpiP0HXUtlBYc+o8XXQkjImNNUoFuZrnEwvxed39w4Hp3b3D3pmD6USDXzCoStLvT3Ze4+5LKysoRFex9R+j51LV00tWd2u6Pl7ftZ+32Os6eN+mQ7fYEgb5tbzOX3P4Cuxt0zbqIZNawgW5mBtwNbHT3bw/RZkrQDjNbGmx3XyoL7dU7OFdFSR4AB1o6U7Ztd+fy7/+Bts4eppQV9H1GIrWN7Wza1ciZ//oM62rqeXLD7pTVISIyEskcoZ8BfBI428zWBq8LzeyzZvbZoM1lwGtmtg64DbjC0/R8uN7NTizJB0hpP/qeuG6UCUV5/OgzS/vmS/OjXHXaLL5w3vEAvL6znuc2H+w2yrGUlSEiMiLJXOXyAnDIuHL324HbU1XUIT8r+Dm1rACA9+pbOWFK6RFvt6fHOe2fftc3P74ojwXTy7j0lBk8sKaGZcdN5J/+9H109zjfeWoz//70ln7vV5+6iGRa6O4U7T1Cnzm+CIhdhZIKewYEcnlRLgBFeREAZlcWAxDJMcqL+nfFmCnQRSTzQhjosZ+TxuUTzbF+lxgeie0HWvrN94Z2R1fspOuUcQVxNfTvTTq2onjQF4KIyGgLX6AHP6M5OUwrL0xdoO/vH+jjgyP03vFiJhQfPCq/Yfm8vun8aA5VE4tZuWE3N9y/LiW1iIiMROgCvfdOUTOYO6mE13fWp2S7W2ub+s2XFsQCvTEI9HHBPMCVS2dx6SkzgnZRTpwa68P/ZXUNze1d1KfwyhsRkWSFLtDnTSnlS+efQEVJPkuPncBbtc0puQb8hS0Hr7I898RJTCqNXUXTG9YzJxT2az9rQqwPvyA30tefD3DyLU/yyR8mupFWRCS9QjfK1JxJpcyZFAvZc06czLee2MQVd/6R2RXFRCOGYVhwTY4ZGAYG5YW5XLl0Fgumlw3aZneP82pNXd/8rX++iJzgOsQvnncCHzt5Wt9n9hpXePCf7tJTZ/DlX79Kj8e2tb6mnrbObgpyI6nefRGRIYUu0OPNmVTCv/2vhfz0j++wu7GNzi7vGxrAPdbf3nsCc3dDO79aXcMdnziVswbcBXqgpYMeh+vOOo65k0r7ulsA8qI5Cb8Eetu4Q24kh7/88HF875mtfev/Z8teTpw6jmnlhYPeKyKSDqEOdIBLFk3nkkXTh213oLmDT/3wJVb8pJrbrzqF80+a0m8dwLwp4/jYwmlJfW5hcPS9aGY5wKCxX665pxqAbf/80aS2JyJypELXhz5S44vz+Om1p3HStDKuu3cNv127o2/dviDQ469kGc6ZJ1Ty5Qvn8a+XLwRid5L2+ujJU4d6m4hI2hw1gQ5QVpjLT65ZyqnHjOf6X6zlrudjw+L2HqGPL0o+0Ivzo6z40HEUBjcexXfTzJ1UksKqRUSSc1QFOsSC957/vZTlJ03h649s5BuPbew7Qp94iMG4hhP/FKP4m5BufGA953772ZEXLCKSpND3oY9EQW6E//iLU7j5t6/1e3hF7+3+IzGuMPbe2ZXFfdMAv3h5+8gLFRE5DEdloENsTJavf3wB86aU8s3HN3H5kpnkR0d+meGimeXcfNF8Lj1lBsX5EcqLcqmLu8Ho92/sYeOuBv7qzDmpKF9EZBBL0yi3w1qyZIlXV1dn5LMH6u5xIike//aXL2/nhgfWD1quq15E5EiY2Wp3X5Jo3VHXh55IqsMcoChfNxWJyOhSoKdJcV7i3qzunsz8RSQi2U+BnibF+YkDvaWja5QrEZGjhQI9TXofjDHQ5+9bS9WNj4xyNSJyNFCgp8nMCUXMnVTC7Vct5vkbzupb/tTGPQB0dvdQ36phdkUkdRToaVJWmMvKL3yYi06exswJRXzjz97Xb/2191Sz8JYnM1SdiGSjYQPdzGaa2e/NbIOZvW5m1ydoY2Z2m5ltMbP1ZnZKesoNr+kDRl18dnMtoJOkIpI6yRyhdwFfdPf5wDLgOjObP6DNBcDc4LUC+F5Kq8wCQ/WpN+skqYikyLCB7u7vufuaYLoR2AgMHK/2EuDHHvNHoNzMNORgnMIhAr2lvXuUKxGRbHVYfehmVgUsBgY+Y206ED9oSQ2DQ/+oNtR16W/uaeShdTtHuRoRyUZJj+ViZiXAA8DfunvDSD7MzFYQ65Jh1qxZI9lEaI0vzsMs9oSjeJ+8+yUALlgwhdyIzlGLyMgllSBmlksszO919wcTNNkBzIybnxEs68fd73T3Je6+pLKyciT1hlZZYS7P33AWKz//oYTrG1o7dRmjiByRZK5yMeBuYKO7f3uIZg8BnwqudlkG1Lv7eymsMyvMGF/E1OBql4EPwfjS/etZeMuTtHepT11ERiaZLpczgE8Cr5rZ2mDZl4FZAO7+feBR4EJgC9ACfCb1pWaHkvwo/3Pj2TS2dbL8O8/3LX/6jdgNR83t3Uc0jK+IHL2GDXR3fwE45HCEHhuD97pUFZXtppcXsqMu8brm9q7DerapiEgvnYXLkOIhLmPcWtvEz1a9O8rViEg2OGqfWJRpJflRKkry2NvU0W/51f/1MgDnnjiJSXHPJhURGY6O0DMkGsnh5a+cO2iMl16tnTo5KiKHR4GeQWbGny5OfP/Vhp0NfOPRjfRorBcRSZICPcPyozlcsGDKoMfg/d9713DHc2/x5p6mDFUmImGjQM8wM+N7nziVV24+L+F6jcYoIslSoI8Rhxrr5WsPb1Cwi8iwdJXLGBHJMaI5RteA4L7+F7F7uZYvmMKfVE3IRGkiEhI6Qh9D3vzHC/jS+SdkugwRCSkF+hhiZlx31pyE657fXMuJf/c4+5raR7kqEQkLBfoYdPX7qwYtu+3pLbR2drPm3SHGDBCRo54CfQz6h4tP4qkvfDjhukgObNGljCKSgAJ9jJozYHjdXnc9/zbnfvtZqrftH+WKRGSsU6CPYf92+cJBy17cug+Ad/e3jHY5IjLGKdDHsEtPncFrt5yfcF1bZw/ffPwN2jTmi4gEdB36GFeSH6W8KJe6lv6Pp/vWE29woKWTaeWFfGLZMRmqTkTGEh2hh0D1V87l7z82v9+yA0HA6w5SEemlQA+BaCSHz5xxbMJ1b+9tpurGR9iyp3GUqxKRsUaBHiI/uWbpoGX3rnoHgIfW6ZncIkc7BXqIfHBuJW/+4wX9lnV2x7pccgzueHarTpKKHMWGDXQz+6GZ7TGz14ZYf6aZ1ZvZ2uB1c+rLlF65kZyE4738+pUdfOOxN/j+s1szUJWIjAXJHKH/CFg+TJvn3X1R8PrqkZclh3LdWXN4+K8/0G/ZO/ti16U3t3dR39qZ6G0ikuWGDXR3fw7QbYljzILpZZwxZ+Kg5Vv2NLHwlif573U7M1CViGRSqvrQTzezdWb2mJmdlKJtyjDuvXYZt125uN+yP74V++59ZlNtJkoSkQxKRaCvAY5x94XAvwO/Gaqhma0ws2ozq66tVeCkwsULpzF/6ri++dbgpOj+5naqbnyElRt2Z6o0ERllRxzo7t7g7k3B9KNArplVDNH2Tndf4u5LKisrj/SjJfDr697P1z++oN+yP7wVG/PlvpffzURJIpIBRxzoZjbFzCyYXhpsc9+RbleSlx+N8Illx/DBuQe/R9s6ewBo6ejWkbrIUSKZyxZ/DvwBOMHMaszsGjP7rJl9NmhyGfCama0DbgOucHfdj54B93xmKd+9YlG/Zb2jM/7gubcyUZKIjKJhB+dy9yuHWX87cHvKKpIRy8kxLlk0nQ07G7hjQIC3d/dQdeMj3HblYi5eOC1DFYpIOulO0Sx004Un8uTnP9Rv2brtsUfX/cfTWzJRkoiMAgV6ljp+cik/vea0Qcs37W6k6sZH+FX19gxUJSLppEDPYh+YW8Hmr1/AacdOGLTu1pWb6e5xdLpDJHso0LNcXjSHH31mKd/58/4nS3fWt3Hclx/lP5/R2C8i2UKBfhQozIvw8cXTuffawV0w33piE4+sf0/jv4hkAQX6UeSMORW8+g8f4U+qxvdbft3P1nDp917U0LsiIadAP8qUFuRy77XLBnXBbNnTxLy/e5xHX9WDMkTCSoF+FMqL5vDxxdN56gsfoiC3/6/AX927hktuf4GttU0Zqk5ERkqBfhSbM6mU129Zzg8+taTf8nU19Zzzb89y9wtv09zelaHqRORwKdCPcpEc47z5k3ntlvP57IeP67fuaw9v4KS/f4IXt+zV5Y0iIaBAFwBK8qPceME8Xv7KuYNOml511yrO/fazbN7dmKHqRCQZCnTpp7I0n1/+5ek896WzmFSa37d8a20zH7n1OT73szW8s685gxWKyFAsU39KL1myxKurqzPy2ZIcd+e1HQ1c9YM/0jigL/0j8ydzw/ITmDOpNEPViRydzGy1uy9JuE6BLsNxd17ZXsen735pULB/YE4FX77wROZPGzfEu0UklRTokjIb32vgxgfWs66mvt/y4yqLufljJ/H+4yaSG1FPnki6KNAl5epbO/n6wxv41eqaQev+30dP5E8XT2diSX6Cd4rIkVCgS9p0dfdwx3Nv8a0nNg1ad+YJlVx/zlwWzignJ8cyUJ1I9lGgy6hYu72O25/ewlMbBz+/9G/Omcvlp85g5oSiDFQmkj0U6DKq3J1nNtfyrcc3seG9hn7rojnGLZecxPknTaFCXTIih02BLhnT0tHFA6tr+NrDG+no7hm0/vPnHs/HFk5ldmVJBqoTCZ8jCnQz+yFwEbDH3RckWG/Ad4ELgRbgandfM1xRCvSjz676Nh58pYZvPj64vx3gggVT+OSyY1hSNYG8qK6UEUnkSAP9Q0AT8OMhAv1C4K+JBfppwHfdffCTFAZQoB/ddje08ezmWu54ditbawffeVqaH+WaDx7LRSdPY84kHb2L9DriLhczqwIeHiLQ7wCecfefB/ObgDPd/ZADayvQpVdbZzfra+r52ap3+M3anQnbvP+4iVy5dBZnz5tEcX50lCsUGTsOFeip+D9jOhD/CPmaYJmelCBJKciNsPTYCSw9dgK3/vkiag608j9b9vKD59/qO3p/ces+Xty6r+89f3bKdC5eOI0z5lToRiaRwKge6pjZCmAFwKxZs0bzoyUkzIyZE4q4Yuksrlg6i7bObt7c3cRv1u7g7hfe7mv34JodPLhmR9/81e+v4qKTp7J41ngiuuZdjlLqcpFQqWvpYF1NPb99ZQcPvrJjyHYfmT+Zc+dP5gNzKphaVkDs3L1I+KW7D/2jwOc4eFL0NndfOtw2FeiSCvWtnbzy7gF+tbqGR9YPfQxRUZLHVacdw6WnTGfWhCIFvITWkV7l8nPgTKAC2A38PZAL4O7fDy5bvB1YTuyyxc+4+7BJrUCXVHN3WjpiJ1hXbtjNg6/UUNfSmbDtgunj+D8fnM25J07WSVYJFd1YJEctd2dvUwcvbKnlFy9tZ9Xb+we1yTH48PGVXHbqTE4/biLji3J1BC9jlgJdJE5dSwcrN+zmp6veZd32uoRt5k4qYfGscs6eN4nFs8ZTUZKvk60yJijQRQ6hoa2T5zfv5bHX3uOVd+vYUdeasN308kLmTSll2eyJvH/ORI6tKKYoT901MroU6CKHaV9TOy9vO8DL2/azYWcDm3Y3sr+5Y1C7wtwI86aWcvykUuZPG8eSqvEcM7GYEvXLS5oo0EVSoLvH2bCzgRe37mVdTR1b9jSxtbaZ7p7B/w9VlOSx5JgJLJxZzhlzJnL85FIKciMZqFqyTbrvFBU5KkRyjPfNKON9M8r6LW9u72LLniZe3VHP6zsb2Ly7kW17m3n89V08/vquvnbHVhQzb0opcyaVMLuymGMrSjhmQhHlOgkrKaJAFzlCxflRFs4sZ+HM8n7LWzu6WVdTx+p3DvDGrkY272pk5YbdPPbarn7t8iI5TC0vYPK4AiaV5lNRks/E4jwmluRTWZrPrAlFzJxQqP56GZZ+Q0TSpDAvwrLZE1k2e2LfMndnT2M7m3c38s6+FrYfaGHHgVZ21sVe62vqaOscPG48QHlRLrMripk7qZTjp5Ry4pRSTppeRllh7mjtkoxxCnSRUWRmTB4XOxr/4NzEbVo7utnX3M6OA63samhjV30bO+paeXd/C2/vbWZN9fZ+7aeVFTB/2jjmTRnH3MklzK4o4ZiKIsYVKOiPNgp0kTGmMC/CjLwiZoxP/PzVts5u3t7bzMb3Gti8u4nNuxt5Y1cjT23c06/d+KJcZk4oYlpZIVPK4rp0SvOpKMljfFEeZYW5FOVF1IefJRToIiFTkBvhxKnjOHHquH7LWzu62VrbxLZ9zbHunP2xLp03djXwzOY9Q3blRHKMssJcyotyqSjJZ/K4AqaVFTCtvJBp5YVMLy9kxoRCSvOjCv4xToEukiUK8yIsmF7Ggullg9a5O43tXdQ2tlPb2M6+pg72t3RQ39JBfWsndS2d7G/uYG9TOy+9vY/dDe2DtjGuIMq08kImjytgfFEuk8sKmF5eSFlhLmWFsS+DkvwoxflRxhVGyYvk6AtglCnQRY4CZsa4glzGFeRyXBIP5O7pcXbWt/JefRs761rZUdca69Ovb2NXQxtba5vY3dBGZ/fQ97GYQW4kh8LcCCX5UXIjRl40h9KCXPKjORTkRqgoyWNcQewLYfr4QiaVFjCxJI9ZE4o0aNoI6F9MRAbJyTFmjB+6Hx9iob+3uZ3Gti7qWjqobWynpaOb5vYu6ls7ae/qoaO7h7aObhrbu+jqdtq7umls66Kjq4e6lk5e31lPY1sXLR3dg7ZfXpTL5NICxhfHvojGFfb+jMbNRykpiOIOHd09dHb10NntdHb30NHVQyTHMIt1KxXkRijOi7UvyY/2vbcgGiEnS8bpUaCLyIjk5BiTSguYVHrk22rr7GZnXSt7mzrY09jGO/ta2FXfxu6GNupaOnl3fwsNrZ00tHXR1N515B8YJ5pjFOVFKA66i4rzoxTnRSjKi1KSH6EwL0pRXoSivAgFub2vHAqiEQrzgldubH1+NEJ+NIe84JUbySE/GnuNRveTAl1EMq4gN8LsyhJmVw7ftqu7h6b2Lhpau2ho66SxrYtIjpEbMXIjQZhGcsiN5tDa0U13T+yIva2zm+aOblrau2hs76Ip+HJobo//GfsLozk439DS0U1zR+wviI6uxCeVk2EGRbmx8M+PRviLZbP4qzPnjHh7Q1Ggi0ioRCM5lBflUV6UN2qf6e509Thtnd20dca+HNq7emjv6qats5uWjm5aO7pp6+qhvbObjqDLp/fV2tlNa2c3Le3dtHd1c8yE4rTUqUAXERmG2cG/AEoLMl3N0HIyXYCIiKSGAl1EJEso0EVEskRSgW5my81sk5ltMbMbE6y/2sxqzWxt8Lo29aWKiMihDHtS1MwiwH8A5wE1wMtm9pC7bxjQ9D53/1waahQRkSQkc4S+FNji7m+5ewfwC+CS9JYlIiKHK5lAnw7ED8BcEywb6FIzW29m95vZzJRUJyIiSUvVSdH/Bqrc/WRgJXBPokZmtsLMqs2sura2NkUfLSIikNyNRTuA+CPuGcGyPu6+L272LuCbiTbk7ncCdwIEJ1HfOaxqD6oA9o7wvWON9mVs0r6MPdmyH3Bk+3LMUCuSCfSXgblmdiyxIL8CuCq+gZlNdff3gtmLgY3DbdTdkxi1ITEzq3b3JSN9/1iifRmbtC9jT7bsB6RvX4YNdHfvMrPPAU8AEeCH7v66mX0VqHb3h4C/MbOLgS5gP3B1qgsVEZFDS2osF3d/FHh0wLKb46ZvAm5KbWkiInI4wnqn6J2ZLiCFtC9jk/Zl7MmW/YA07Yu5D/0IKRERCY+wHqGLiMgACnQRkSwRukAfbqCwscbMfmhme8zstbhlE8xspZm9GfwcHyw3M7st2Lf1ZnZK5irvz8xmmtnvzWyDmb1uZtcHy8O4LwVm9pKZrQv25ZZg+bFmtiqo+T4zywuW5wfzW4L1VZmsPxEzi5jZK2b2cDAfyn0xs21m9mowyF91sCx0v2MAZlYe3Dn/hpltNLPT070voQr0uIHCLgDmA1ea2fzMVjWsHwHLByy7Efidu88FfhfMQ2y/5gavFcD3RqnGZHQBX3T3+cAy4Lrg3z6M+9IOnO3uC4FFwHIzWwb8C3Cru88BDgDXBO2vAQ4Ey28N2o0119P//o8w78tZ7r4o7jrtMP6OAXwXeNzd5wELif33Se++uHtoXsDpwBNx8zcBN2W6riTqrgJei5vfBEwNpqcCm4LpO5sWAdcAAAKtSURBVIArE7Ubay/gt8RG4Az1vgBFwBrgNGJ37kUH/q4Ruwfj9GA6GrSzTNcetw8zgnA4G3gYsBDvyzagYsCy0P2OAWXA2wP/bdO9L6E6Qif5gcLGusl+8M7aXcDkYDoU+xf8mb4YWEVI9yXoolgL7CE2/tBWoM7du4Im8fX27Uuwvh6YOLoVH9J3gBuA3sfSTyS8++LAk2a22sxWBMvC+Dt2LFAL/FfQFXaXmRWT5n0JW6BnHY99HYfm2lEzKwEeAP7W3Rvi14VpX9y9290XETu6XQrMy3BJI2JmFwF73H11pmtJkQ+4+ynEuiCuM7MPxa8M0e9YFDgF+J67LwaaOdi9AqRnX8IW6MMOFBYSu81sKsTGwSF2lAhjfP/MLJdYmN/r7g8Gi0O5L73cvQ74PbFuiXIz6717Or7evn0J1pcB+xgbzgAuNrNtxJ5VcDaxvtsw7gvuviP4uQf4NbEv2zD+jtUANe6+Kpi/n1jAp3VfwhbofQOFBWftrwAeynBNI/EQ8Olg+tPE+qN7l38qOOO9DKiP+/Mso8zMgLuBje7+7bhVYdyXSjMrD6YLiZ0L2Egs2C8Lmg3cl959vAx4Oji6yjh3v8ndZ7h7FbH/H552978ghPtiZsVmVto7DXwEeI0Q/o65+y5gu5mdECw6B9hAuvcl0ycPRnCy4UJgM7E+z69kup4k6v058B7QSexb+xpifZa/A94EngImBG2N2FU8W4FXgSWZrj9uPz5A7M/D9cDa4HVhSPflZOCVYF9eA24Ols8GXgK2AL8C8oPlBcH8lmD97EzvwxD7dSbwcFj3Jah5XfB6vff/7zD+jgX1LQKqg9+z3wDj070vuvVfRCRLhK3LRUREhqBAFxHJEgp0EZEsoUAXEckSCnQRkSyhQBcRyRIKdBGRLPH/AYAqfDbSoAgXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81.05499999999999\n"
     ]
    }
   ],
   "source": [
    "accuracy_train = (1-(np.sum(np.abs((Y_pred >= np.max(Y_pred,axis=0)) * 1 - Y_train)))/2/Y_train.shape[1]) * 100\n",
    "print(accuracy_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred_test,cache = forward_nn_network(X_test,cache,h_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81.65\n"
     ]
    }
   ],
   "source": [
    "accuracy_test = (1-(np.sum(np.abs((Y_pred_test >= np.max(Y_pred_test,axis=0)) * 1 - Y_test)))/2/Y_test.shape[1]) * 100\n",
    "print(accuracy_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now lets mess around with a bigger network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "cache,h_params = initialize_parameters(X_train,[50,25,10],['sigmoid','sigmoid','softmax'],learning_rate =1e-4,lam = 1e-5)\n",
    "cost = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.145302510845956\n",
      "3.159844550600732\n",
      "3.1745890799795404\n",
      "3.189435419363853\n",
      "3.204451749672674\n",
      "3.219650762884007\n",
      "3.2351261195426435\n",
      "3.2507284442109525\n",
      "3.266508266893154\n",
      "3.2824915528827106\n",
      "3.298522507815438\n",
      "3.3147238745244736\n",
      "3.3311666716110526\n",
      "3.347838056706813\n",
      "3.3647380986222406\n",
      "3.3818144164550827\n",
      "3.3990390636928876\n",
      "3.416536609546848\n",
      "3.4341635143302116\n",
      "3.4520141587255684\n",
      "3.4701539962972525\n",
      "3.488451004375045\n",
      "3.506975382965728\n",
      "3.525698529842197\n",
      "3.5447190533665895\n",
      "3.5639090327278646\n",
      "3.5833225044894297\n",
      "3.602954885866052\n",
      "3.6227872917760116\n",
      "3.6429186901865562\n",
      "3.6633378466530835\n",
      "3.683930514179124\n",
      "3.704745661341942\n",
      "3.7258025099446366\n",
      "3.747118599607311\n",
      "3.768690555173102\n",
      "3.790441693781797\n",
      "3.8123924567739884\n",
      "3.8345888231215115\n",
      "3.85714951712089\n",
      "3.8799086568990733\n",
      "3.9028657847094896\n",
      "3.926077277358401\n",
      "3.949538452630031\n",
      "3.973252516558196\n",
      "3.997198657222616\n",
      "4.021414117913486\n",
      "4.045889927848236\n",
      "4.070550549116836\n",
      "4.095560802271491\n",
      "4.120822314274599\n",
      "4.14625592999475\n",
      "4.17199780294073\n",
      "4.198017497624051\n",
      "4.224425189313783\n",
      "4.251021425509909\n",
      "4.27790032323791\n",
      "4.30509185829499\n",
      "4.332606868368146\n",
      "4.3603823734685365\n",
      "4.3884139736253625\n",
      "4.416765833134427\n",
      "4.445217594147286\n",
      "4.474070935570078\n",
      "4.503271620555369\n",
      "4.532796924228474\n",
      "4.562512239593101\n",
      "4.592424250004837\n",
      "4.622626515359103\n",
      "4.653258902267998\n",
      "4.684057286649053\n",
      "4.715255502112589\n",
      "4.7467445179154\n",
      "4.778584213200673\n",
      "4.810700377985834\n",
      "4.8431646111306845\n",
      "4.875951639397477\n",
      "4.908972932957793\n",
      "4.942381476519849\n",
      "4.976087355217435\n",
      "5.010124893181955\n",
      "5.044488058388396\n",
      "5.079181199549848\n",
      "5.1139789359043295\n",
      "5.14916042198715\n",
      "5.184695311805992\n",
      "5.220569594863634\n",
      "5.25674450610026\n",
      "5.293226035972556\n",
      "5.330039862382985\n",
      "5.367372774790095\n",
      "5.404989505377593\n",
      "5.442798884011798\n",
      "5.480971066474777\n",
      "5.51957421256243\n",
      "5.558514706642796\n",
      "5.5978167799386584\n",
      "5.637441989543805\n",
      "5.677473474619101\n",
      "5.71786803320964\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    Y_pred,cache = forward_nn_network(X_train,cache,h_params)\n",
    "    loss = cost_function(Y_pred,Y_train,'cross-entropy')\n",
    "    for j in range(len(h_params['nn_arch'])):\n",
    "        loss += 0.5 * h_params['lam'] * np.sum(np.square(cache['W'+str(j)])) / Y_train.shape[1]\n",
    "    print(np.sum(loss))\n",
    "    cache = backward_nn_network(X_train,Y_pred,Y_train,'cross-entropy',cache,h_params)\n",
    "    cache = update_weights(cache,h_params,'L2')\n",
    "    cost.append(np.sum(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0xb40f95198>]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxddZ3/8dcne5qkTdOkabqmK6UsXShlFVkVKQIKKj8XZMSpIqj8Rn8qP2eYEZ3fb9TfjOAwDpsiikCRRUpFFFkVgdLSpi3d9zZN23RJs2/3fn5/3FtIQ9Ik7b333Hv7fj4eeeTcc07veXNa3jn5nuWauyMiIqkvI+gAIiISGyp0EZE0oUIXEUkTKnQRkTShQhcRSRNZQW24tLTUKysrg9q8iEhKWrJkyV53L+tpWWCFXllZyeLFi4PavIhISjKzrb0t05CLiEiaUKGLiKQJFbqISJpQoYuIpAkVuohImlChi4ikCRW6iEiaUKGLiCTQHX9ex+sb98XlvVXoIiIJsutgK3f8eT2Lt+yPy/ur0EVEEuTZFTUAXHZqRVzeX4UuIpIgv19Rw9QRRUwsK4zL+6vQRUQSYMeBZpZsPcBHp4+M2zZU6CIiCbBweWS45aOnqtBFRFLaM1U7mT6mmLHDBsVtGyp0EZE421jbyDs767kijsMtoEIXEYm7Z6p2YgZzT4nP1S2HqNBFROLI3VlQtZM5lSWMGJIX1231q9DNbIuZrTCzZWb2vo8ZMrPzzexgdPkyM7st9lFFRFLPqpp6NtU2ccWM+A63wMA+gu4Cd997hOV/cffLjzWQiEg6eaaqhqwM4yMnx3e4BTTkIiISN+7OM1U7OXdyKSUFOXHfXn8L3YE/mdkSM5vXyzpnmVmVmf3BzE6KUT4RkZT19rY6quta4nrteVf9HXI5192rzWw48LyZrXH3V7ssfxsY5+6NZnYZ8Dtgcvc3if4wmAcwduzYY4wuIpLcnqnaSU5WBh86qTwh2+vXEbq7V0e/7wGeAuZ0W17v7o3R6WeBbDMr7eF97nX32e4+u6ys7JjDi4gkq1DYWbi8hgtPGE5RXnZCttlnoZtZgZkVHZoGPgSs7LbOCDOz6PSc6PvG54G/IiIp4I1N+9jb2JaQq1sO6c+QSznwVLSvs4CH3f05M/sygLvfDVwD3GhmnUALcK27e5wyi4gkvSffrqYoN4sLpw5P2Db7LHR33wRM72H+3V2m7wLuim00EZHU1NzeyXMra7j81JHkZWcmbLu6bFFEJMb+9M5umtpDfGzWqIRuV4UuIhJjTy6tZlRxPnMqSxK6XRW6iEgM7alv5a/ra/nYzFFkZFhCt61CFxGJoaeX7STsJHy4BVToIiIx9eTSaqaPKY7b54YeiQpdRCRGVtfUs7qmno/PTPzROajQRURi5qml1WRlWFw/CPpIVOgiIjEQCjtPL6vm/BOGJ+TJij1RoYuIxMDfNu5ld30bHw/gZOghKnQRkRh48u1qBucl9lb/7lToIiLHqKG1g+dW7mJugm/1706FLiJyjBYur6GlI8QnZ48ONIcKXUTkGM1/aztTyguZMaY40BwqdBGRY7BudwPLttfxydljiD5mPDAqdBGRY/DYW9vJzjQ+FtDNRF2p0EVEjlJ7Z5gnl1Zz8YnlDCvMDTqOCl1E5Gi9uGY3+5va+eTsMUFHAVToIiJHbf5b2xkxOI/zpiTHh96r0EVEjsKug628sq6Wa04bTWaCn3veGxW6iMhReOLtHYQdPhHwtedd9avQzWyLma0ws2VmtriH5WZmPzWzDWa23MxmxT6qiEhyCIedxxZv58wJJYwbVhB0nHdlDWDdC9x9by/LPgJMjn6dAfx39LuISNpZtGU/W/c1c8vFk4OOcphYDblcCfzKI94Ais2sIkbvLSKSVB5ZtI2ivCwuPSm5aq6/he7An8xsiZnN62H5KGB7l9c7ovMOY2bzzGyxmS2ura0deFoRkYDta2zjDyt2cfWs0eTnBPcgrp70t9DPdfdZRIZWbjKz845mY+5+r7vPdvfZZWXJcZmPiMhAPL5kB+2hMJ85Y2zQUd6nX4Xu7tXR73uAp4A53VapBrpeWT86Ok9EJG2Ew87Di7YxZ3wJk8uLgo7zPn0WupkVmFnRoWngQ8DKbqstAK6LXu1yJnDQ3WtinlZEJECvbdzL1n3NSXl0Dv27yqUceCr6FLEs4GF3f87Mvgzg7ncDzwKXARuAZuDv4hNXRCQ4v3ljGyUFOVx68oigo/Soz0J3903A9B7m391l2oGbYhtNRCR57K5v5fnVu/niB8aTm5VcJ0MP0Z2iIiL9MP+t7YTCzqfnJOdwC6jQRUT61BkK88iibXxgcmlS3RnanQpdRKQPL6+tpeZgK585Y1zQUY5IhS4i0offvLmV8sG5XHTi8KCjHJEKXUTkCLbta+bldbV86vSxZGcmd2UmdzoRkYD96vUtZJol7bXnXanQRUR60dTWyfzF2/nIKRWUD84LOk6fVOgiIr14cmk1Da2dXH92ZdBR+kWFLiLSA3fnwb9t4dTRQ5g1tjjoOP2iQhcR6cFrG/axYU8j159dSfTRJ0lPhS4i0oNf/m0zpYU5zD01uT7E4khU6CIi3Wzd18QLa/bw6Tljk/a5LT1RoYuIdPOr17dGLlU8M7nvDO1OhS4i0kVDawePvbWdy1LkUsWuVOgiIl3Mf2s7DW2dfPED44OOMmAqdBGRqI5QmF/8dTNnjC/h1NGpcaliVyp0EZGoZ1fUsPNgK/POmxB0lKOiQhcRIXIj0X1/2cTEsgIuOCG5n6rYGxW6iAjw+qZ9rKyu5+8/MIGMjNS4kai7fhe6mWWa2VIzW9jDsuvNrNbMlkW/vhjbmCIi8XXfq5soLczhqpmjgo5y1Pr8kOguvg6sBgb3sny+u9987JFERBJr3e4GXlpbyzcumUJedurcSNRdv47QzWw0MBe4P75xREQS7/6/bCIvO4PPptiNRN31d8jlDuBbQPgI61xtZsvN7HEzG9PTCmY2z8wWm9ni2tragWYVEYm5PQ2t/G7pTj5x2hiGFuQEHeeY9FnoZnY5sMfdlxxhtWeASnc/FXgeeLCnldz9Xnef7e6zy8rKjiqwiEgsPfDaFjrCYW44N/VuJOquP0fo5wBXmNkW4FHgQjN7qOsK7r7P3duiL+8HTotpShGRODjY3MGvX9/KZadUUFlaEHScY9Znobv7re4+2t0rgWuBF939s13XMbOuz5e8gsjJUxGRpPbg61tobOvkpvMnBR0lJgZylcthzOx2YLG7LwC+ZmZXAJ3AfuD62MQTEYmPprZOfvHaZi6aOpxpI3u7eC+1DKjQ3f1l4OXo9G1d5t8K3BrLYCIi8fTIom3UNXdw04XpcXQOulNURI5DrR0h7n11E2dNGMassUODjhMzKnQROe48vmQHexrauDmNjs5BhS4ix5mOUJi7X9nIjDHFnD1xWNBxYkqFLiLHlQXLdrLjQAs3XTAJs9R8CFdvVOgictwIhZ2fvbyBqSOKuGhqaj4i90hU6CJy3Fi4fCcba5u46YJJKfuI3CNRoYvIcSEUdu58YT1TyguZe0pF338gBanQReS4sKCqmk21TfzPi6ek5dE5qNBF5DjQGQpz55/Xc2LFYD580oig48SNCl1E0t5TS6vZsq+Z/3nx5LQ9OgcVuoikuY5QmJ++uJ6TRw3mkmnlQceJKxW6iKS1J5bsYPv+Fv7hkilpd915dyp0EUlbrR0h/vPFDUwfU8wFJ6TfdefdqdBFJG099MZWquta+NaHT0j7o3NQoYtImqpv7eCulzbwgcmlnDOpNOg4CaFCF5G0dO8rm6hr7uDbl04NOkrCqNBFJO3sqW/l53/dzEenj+TkUUOCjpMwKnQRSTs/fXE9HaEw37hkStBREkqFLiJpZfPeJh5ZtJ3/MWcslaUFQcdJqH4XupllmtlSM1vYw7JcM5tvZhvM7E0zq4xlSBGR/vrxH9eQk5nBVy9Kr08j6o+BHKF/HVjdy7IbgAPuPgn4CfDDYw0mIjJQizbv59kVu/jSBycwvCgv6DgJ169CN7PRwFzg/l5WuRJ4MDr9OHCRHQ8XfYpI0giHne8vXMWIwXnMO29C0HEC0d8j9DuAbwHhXpaPArYDuHsncBB434f1mdk8M1tsZotra2uPIq6ISM+eWlrNiuqDfOvSExiUkxV0nED0Wehmdjmwx92XHOvG3P1ed5/t7rPLysqO9e1ERABobu/kR39cw6mjh3DVjFFBxwlMf47QzwGuMLMtwKPAhWb2ULd1qoExAGaWBQwB9sUwp4hIr+55ZRO769v4p8unpfXjcfvSZ6G7+63uPtrdK4FrgRfd/bPdVlsAfD46fU10HY9pUhGRHtQcbOGeVzcy95QKTq8sCTpOoI56oMnMbgcWu/sC4OfAr81sA7CfSPGLiMTdj59bSzgM3/nI8XOLf28GVOju/jLwcnT6ti7zW4FPxDKYiEhflm2v48ml1Xz5gxMZUzIo6DiB052iIpKSQmHnn363kuFFudx0wcSg4yQFFbqIpKRHFm1jRfVBvjv3RIrysoOOkxRU6CKScvY3tfPjP67lzAklXDF9ZNBxkoYKXURSzo+eW0NTWye3X3nycfFJRP2lQheRlPL2tgM8+tZ2vnDueKaUFwUdJ6mo0EUkZYTCzm1Pr6R8cC5fu2hy0HGSjgpdRFLGA69tZmV1Pf84dxqFucfn81qORIUuIilh+/5m/v1P67hw6nAuP7Ui6DhJSYUuIknP3fnu71aSYfD9q3QitDcqdBFJek8v28mr62r55odPYFRxftBxkpYKXUSS2v6mdm5fuIoZY4q57qzKoOMkNRW6iCS1H/x+FfUtHfzb1aeQeRw/Grc/VOgikrT+sr6WJ9+OPHxr6ojBQcdJeilZ6L/462au/K/Xgo4hInHU0NrBd55YwYTSAm6+cFLQcVJCSl7IefvCVUFHEJE4+8HC1dQcbOHxG88mLzsz6DgpISWP0EUkvb24ZjfzF2/nSx+cyKyxQ4OOkzJU6CKSVA40tfPtJ1ZwQnkRt1ys2/sHIiWHXEQkff3zgnc40NTOA9efTm6WhloGQkfoIpI0fr+8hgVVO/naRZM5edSQoOOknD4L3czyzGyRmVWZ2Ttm9r0e1rnezGrNbFn064vxiXu4cNgTsRkRSYDquhZufXI508cUc+P5+ki5o9GfIZc24EJ3bzSzbOCvZvYHd3+j23rz3f3m2EfsXWfYydGNBiIpLxR2bnl0KWGHn147g+xMDR4cjT4L3d0daIy+zI5+JcWhcdiTIoaIHKO7XtzAW1sO8JNPTWfcsIKg46Ssfv0YNLNMM1sG7AGed/c3e1jtajNbbmaPm9mYXt5nnpktNrPFtbW1xxA7olNDLiIpb/GW/dz5wjqumjGSj80cHXSclNavQnf3kLvPAEYDc8zs5G6rPANUuvupwPPAg728z73uPtvdZ5eVlR1LbgBCIRW6SCo72NLB1x9dxqih+Xz/qu61IgM1oIEqd68DXgIu7TZ/n7u3RV/eD5wWm3hHFtKQi0jKcne++dsqdte3cue1MynKyw46Usrrz1UuZWZWHJ3OBy4B1nRbp+vHh1wBrI5lyN50hsOJ2IyIxME9r27i+VW7ufWyE3U3aIz05yqXCuBBM8sk8gPgMXdfaGa3A4vdfQHwNTO7AugE9gPXxytwV+pzkdT0xqZ9/Oi5Ncw9pYIvnFMZdJy00Z+rXJYDM3uYf1uX6VuBW2MbrW86QhdJPXvqW7n54aVUlhbww2tO1cfJxVBK3/qvPhdJLZ2hMDc/spSmtk4e/vszKMxN6QpKOim9N3WELpJafvzHtSzavJ87PjWDKeVFQcdJOyl9O1ZI16GLpIynlu7gnlc38bkzx3HVzFFBx0lLqV3oumxRJCUs3XaAbz+xgjMnlHDbR6cFHSdtpXShd+rGIpGkV3OwhXm/XkL54Fx+9pnT9JyWOErpPatnuYgkt5b2EPN+tYTmtk5+/vnTKSnICTpSWkvxk6IqdJFk5e5864nlrNx5kPs+N1snQRMgpY/QdVJUJHnd+cJ6nqnayf/68AlcPK086DjHBRW6iMTcbxdv544/r+fqWaO58YP6sIpEUaGLSEz9ZX0ttz65gnMnlfJ/P36K7gRNIBW6iMTMqp313PjQ20waXsh/f3YWOVkpXTEpJ6X3tgpdJHlU17XwhV++RWFuFg/83el6HG4AdJWLiByzvY1tfO7+N2lq7+SxL51FxZD8oCMdl3SELiLH5GBLB9f9fBE7D7bwwPWnc2LF4KAjHbdU6CJy1FraQ9zwy7dYv6eBez43m9mVJUFHOq6ldqHrTlGRwLR3hvnyQ0t4e9sB7rx2Jh+ccuyfEyzHJqXH0EN6fK5IIDpCYW6Zv5RX1tXyw6tP4bJTKvr+QxJ3KX2ErodziSReRyjM1x9dyrMrdvGPc0/kU6ePDTqSRKX0EboeziWSWB2hMF97ZCl/WBkp8y9+YELQkaSLlC50XbYokjgdoTBffXgpz72zi3+6fBo3nDs+6EjSTZ9DLmaWZ2aLzKzKzN4xs+/1sE6umc03sw1m9qaZVcYjbHdhFbpIQrR2hLjxoSU8984ublOZJ63+jKG3ARe6+3RgBnCpmZ3ZbZ0bgAPuPgn4CfDD2MbsmY7QReKvobWDz/9iES+s2cP3rzyJL6jMk1afhe4RjdGX2dGv7k16JfBgdPpx4CJLwBN5dB26SHzta2zj0/e9yZKtB7jjUzP43FmVQUeSI+jXVS5mlmlmy4A9wPPu/ma3VUYB2wHcvRM4CAzr4X3mmdliM1tcW1t7bMlRoYvE0866Fj55z+us293AfdfN5soZ+mDnZNevQnf3kLvPAEYDc8zs5KPZmLvf6+6z3X12Wdmx34SgIReR+Fi3u4FP3P06e+rb+PUNZ3DB1OFBR5J+GNB16O5eB7wEXNptUTUwBsDMsoAhwL5YBDwSnRQVib2/rK/l6p/9jfZQmEfmncmc8bqdP1X05yqXMjMrjk7nA5cAa7qttgD4fHT6GuBF9/hfJK4jdJHYemTRNq5/4C1GDc3n6ZvO4eRRQ4KOJAPQn+vQK4AHzSyTyA+Ax9x9oZndDix29wXAz4Ffm9kGYD9wbbwCd/05oTF0kdgIh50fPreGe17dxPknlHHXp2dRmJvSt6kcl/r8G3P35cDMHubf1mW6FfhEbKP1rGuJ6+FcIseuobWDbzxWxZ9W7ea6s8Zx2+XTyMpM6aeCHLdS7kdw14NyHaGLHJsNexqY9+slbN3XzD9/dBrXn12pzwBNYSlY6BpyEYmF51bu4huPLSM/J5PffPEMzpzwviuNJcWkXKEfNuSiQhcZsFDY+Y/n1/JfL21k+phi7v7sLH1kXJpIuULveoTeqeehiwzIroOt3DJ/KW9s2s+1p4/he1eeRG5WZtCxJEZSr9C7dHhIfS7Sby+s3s03f1tFW2eY//eJ6Vxz2uigI0mMpVyhhw4bQ1eji/SlrTPEv/1hDQ+8toVpFYP5z0/PZGJZYdCxJA5SrtAPPykaYBCRFLC6pp5vPFbFqpp6rj+7klsvm6ohljSWeoUe1hG6SF86Q2HufmUjd76wniH52dx/3WwunlYedCyJs5Qr9NBhJ0V1lYtId+t2N/CNx6pYUX2Qj04fyfeuOImSgpygY0kCpFyhd+1wfaaoyHvaOkPc88om7npxA4V5WfzsM7O47JSKoGNJAqVeoXdp9M6QCl0E4I1N+/juUyvYWNvE3FMq+N6VJ1FamBt0LEmwlCv0rjcTdeisqBzn9je183+eXc3jS3Ywemg+D/zd6Vxwgp5dfrxKuULvOszS1BYKMIlIcDpCYR5+cxs/+fM6Gls7+cr5E/nqhZPJz9EVLMezlC70+taOAJOIBOPltXv4we9Xs2FPI2dPHMa/XHESU8qLgo4lSSDlCv3QKEtuVgaNbZ3BhhFJoHW7G/jX36/mlXW1VA4bxH3XzebiE4fr6YjyrpQr9ENH6EPys1XoclzYsreJO/68jqerdlKYm8U/zj2R686qJCdLzyyXw6VcoR86KTo4P5ste5twdx2hSFqqrmvhP19Yz2+X7CA70/jSeRP50nkTGKpryqUXKVfoXY/QO8NOW2eYvGydCJL0seNAM/e9uolHFm0H4HNnjuMrF0xkeFFewMkk2aVgoUe+D8nPBqChtVOFLmlhza567nllEwuqdmLANaeN5qsXTWZUsZ5VLv3TZ6Gb2RjgV0A54MC97n5nt3XOB54GNkdnPenut8c2asShIZf3Cr2DsiLdQCGpyd15a8sB7n5lIy+u2cOgnEyuP7uSG84dz0gVuQxQf47QO4FvuPvbZlYELDGz5919Vbf1/uLul8c+4uHcDy90nRiVVNTaEWJB1U5+9foWVlbXU1KQwz9cMoXrzhpH8SCNkcvR6bPQ3b0GqIlON5jZamAU0L3QE+Ldk6J5keiNrSp0SR3b9zfz0Btbmb94O3XNHUwpL+QHV53M1bNG66YgOWYDGkM3s0pgJvBmD4vPMrMqYCfwTXd/p4c/Pw+YBzB27NiBZgXee9ri4ENDLjpClyTX2hHi+VW7eeLtHbyyrpYMMz58UjnXnVXJGeNLdJWWxEy/C93MCoEngFvcvb7b4reBce7eaGaXAb8DJnd/D3e/F7gXYPbs2Uf1ZC3vdlJUR+iSjNydpdvreHzJDhZW7aS+tZORQ/K4+YJJfPqMsfpQZomLfhW6mWUTKfPfuPuT3Zd3LXh3f9bMfmZmpe6+N3ZRI3o6KSqSDNydVTX1/GHFLp5dUcOmvU3kZWfwkZMruOa00Zw1YRgZGToal/jpz1UuBvwcWO3u/9HLOiOA3e7uZjYHyAD2xTRp1KEhl5KCHLIzjZr61nhsRqRf3J3lOw7y7Moanlu5i637mskwOGviMOadN4G5p1ZQlJcddEw5TvTnCP0c4HPACjNbFp33v4GxAO5+N3ANcKOZdQItwLXu8fn0iUNvm52ZwYTSQtbvbozHZkR61dTWyd827uPltXt4eW0t1XUtZGUYZ08q5cYPTuSSaeUM07PIJQD9ucrlr8ARf09097uAu2IV6kgOPZwrw4zJ5YVU7ahLxGblOObubKxt5OW1tby8tpZFm/fTHgpTkJPJ2ZNKueXiyVwyrVyXG0rgUu5O0UNj6BkZMKW8iN+vqKG5vZNBOSn3nyJJyt3ZsKeRNzbt443N+3lz0372NrYBMHl4IdefU8n5U8qYXVmiB2RJUkm5Fjw05JKZYUwpL8QdVtc0cNq4oQEnk1TV1hlidU0Dy7YdYNGWSIHva2oHYMTgPM6dNIwzJgzjA5NLGT10UMBpRXqXcoV+6KRohhlnThhGVobxp1W7VOjSL+FwZPikasdBqrbXUbWjjtU19XREP592VHE+H5xSxpkThnHGhBLGlgzSdeKSMlKu0M+ZWMqTXzmbMUMHkZ+TyTmTSllYVcPXL5qsYRc5TENrB+t2N7BmVwNrahpYu6uBVTX17z4uojA3i1NGDeGGcycwffQQTh1TzMgheSpwSVkp14BDC3IOex70DeeO5/MPLOLjP/sb0yoGk5OVQXZmBpkZRoYZmRmQkWFkZRiZZgwtyGFsySDGlxYwblgBmbouOOXVNbezaW8Tm2ub2LS3kbW7Glhd00B1Xcu76xTmZnHCiCI+NnMU08cUM2PMECaUFuq6cEkrKVfo3Z03pYyfXjuTX7y2mbe27qetI0xHKEwo7IQ9chI15B75Hj78Ssr87EymjChiWkUR0yoGc2LFYKZWDKYwN+V3S1oJhZ3ahjaq61rYWdfCtv3NbKptYvPeRjbvbeJA83s3l2VmGBPLCpg1biifPmMsJ5QXMbWiiFHF+TrylrRncbpcvE+zZ8/2xYsXJ3Sb7s6+pna27W9m455GVtc0sLqmnlU19Rxsea8UxpYMYlrFYKaNjJT8tJGD9at4nLR2hNjb2Mbexnb2NrRRU9/KzroWaupa2FnXSnVdC7vrW+ns9sN4xOA8xpcWML6sgAmlBVQOi0yPGTpIV55IWjOzJe4+u6dlx9WhqJlRWphLaWEus8a+dxLV3ak52Bop9531rN5Vz+qaBp57Z9e76wzJz+bEiiImDS9k9NBBjB6az+ihgygryqVkUI6elEfkSLqhtYODLe991TV3sLexjX2N7exrihT3vsb3vje1h973PlkZxogheYwszmfO+BIqotOjivOpKM5jzNBBFOi3KJH30f8VRIp+ZHE+I4vzuejE8nfnN7Z1snZXPatqGli1M3Ikv3B5DXXN739+TH52JiUFOQwtyGbooJzI9KAchhXkUJiXxaCcTPKyM8nPzmRQThb5ORnkZ2eRnxOZl5uVQVamkZ353jmAeAqFnY5QmPZQmI7OMK2dYVraO2luD9HUFqKlIzLd3B6iua2T5o4QLdFlze2d1Hcr7oPNHTS0ddLbL3wZBiUFuZQW5lBamMvMsYMYVpBLaVEOpQW5DCvMYVhhLhVD8igtzNW5DZGjoEI/gsLcLE4bV8Jp40oOm9/Q2kF1XQs79rewr6mN/U0d7I9+P9Dczv7osM7+pnYajvJpkGaQnREp+awMIycrg6zo664jP90LtKdCPay8Q2HaO8OEj2KkzQwGZWeSn5PFkPwshuRnU1aYy6SyQobkZ0e+BuW8N52fTfGgbEoLcynOz9YJSJE4U6EfhaK8bKaOyGbqiMF9rtveGaY5euTbEj3K7el7a0eIzpDTEQ7TGXI6Q2Hao987o4XcGYp8fx/r/vLwGVkZRnZW5Og/JzPj3SuBIl9GblZk3qCcyG8S+TmZ701nZ1KQG5nOzcrQeQSRJKZCj7OcrAxysnIo1g2GIhJnuhxARCRNqNBFRNKECl1EJE2o0EVE0oQKXUQkTajQRUTShApdRCRNqNBFRNJEYE9bNLNaYOtR/vFSYG8M48RSsmZTroFRroFRroE72mzj3L2spwWBFfqxMLPFvT0+MmjJmk25Bka5Bka5Bi4e2TTkIiKSJlToIiJpIlUL/d6gAxxBsmZTroFRroFRroGLebaUHEMXEZH3S9UjdBER6UaFLiKSJlKu0M3sUjNba2YbzOw7AWfZYmYrzGyZmS2Ozisxs+fNbH30+9C+3icGOX5hZnvMbGWXeT3msIifRryyBUMAAARcSURBVPffcjObleBc/2Jm1dF9tszMLuuy7NZorrVm9uE45hpjZi+Z2Soze8fMvh6dH+g+O0KuZNhneWa2yMyqotm+F50/3szejGaYb2Y50fm50dcbossrE5zrl2a2ucs+mxGdn7B//9HtZZrZUjNbGH0d3/3l7inzBWQCG4EJQA5QBUwLMM8WoLTbvB8B34lOfwf4YQJynAfMAlb2lQO4DPgDkQ+uOxN4M8G5/gX4Zg/rTov+feYC46N/z5lxylUBzIpOFwHrotsPdJ8dIVcy7DMDCqPT2cCb0X3xGHBtdP7dwI3R6a8Ad0enrwXmJzjXL4Frelg/Yf/+o9v7B+BhYGH0dVz3V6odoc8BNrj7JndvBx4Frgw4U3dXAg9Gpx8Eror3Bt39VWB/P3NcCfzKI94Ais2sIoG5enMl8Ki7t7n7ZmADkb/veOSqcfe3o9MNwGpgFAHvsyPk6k0i95m7e2P0ZXb0y4ELgcej87vvs0P78nHgIrPYfyDtEXL1JmH//s1sNDAXuD/62ojz/kq1Qh8FbO/yegdH/gcfbw78ycyWmNm86Lxyd6+JTu8CyoOJ1muOZNiHN0d/3f1FlyGpQHJFf7WdSeTILmn2WbdckAT7LDp8sAzYAzxP5DeCOnfv7GH772aLLj8IDEtELnc/tM/+NbrPfmJmud1z9ZA51u4AvgUc+mT3YcR5f6VaoSebc919FvAR4CYzO6/rQo/8/hT4daHJkiPqv4GJwAygBvj3oIKYWSHwBHCLu9d3XRbkPushV1LsM3cPufsMYDSR3wSmBpGju+65zOxk4FYi+U4HSoBvJzKTmV0O7HH3JYncbqoVejUwpsvr0dF5gXD36uj3PcBTRP6R7z70K1z0+56A4vWWI9B96O67o/8DhoH7eG+IIKG5zCybSGn+xt2fjM4OfJ/1lCtZ9tkh7l4HvAScRWTIIquH7b+bLbp8CLAvQbkujQ5fubu3AQ+Q+H12DnCFmW0hMjR8IXAncd5fqVbobwGTo2eKc4icPFgQRBAzKzCzokPTwIeAldE8n4+u9nng6SDyHSHHAuC66Nn+M4GDXYYZ4q7beOXHiOyzQ7mujZ7tHw9MBhbFKYMBPwdWu/t/dFkU6D7rLVeS7LMyMyuOTucDlxAZ438JuCa6Wvd9dmhfXgO8GP2tJxG51nT5wWxExqm77rO4/126+63uPtrdK4n01Ivu/hnivb9ieUY3EV9EzlKvIzJ+990Ac0wgcoVBFfDOoSxExr1eANYDfwZKEpDlESK/incQGZe7obccRM7u/1d0/60AZic416+j210e/Udc0WX970ZzrQU+Esdc5xIZTlkOLIt+XRb0PjtCrmTYZ6cCS6MZVgK3dfn/YBGRE7K/BXKj8/OirzdEl09IcK4Xo/tsJfAQ710Jk7B//10yns97V7nEdX/p1n8RkTSRakMuIiLSCxW6iEiaUKGLiKQJFbqISJpQoYuIpAkVuohImlChi4ikif8PD1+dHKRLD60AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18.51166666666667\n"
     ]
    }
   ],
   "source": [
    "accuracy_train = (1-(np.sum(np.abs((Y_pred >= np.max(Y_pred,axis=0)) * 1 - Y_train)))/2/Y_train.shape[1]) * 100\n",
    "print(accuracy_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred_test,cache = forward_nn_network(X_test,cache,h_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18.75\n"
     ]
    }
   ],
   "source": [
    "accuracy_test = (1-(np.sum(np.abs((Y_pred_test >= np.max(Y_pred_test,axis=0)) * 1 - Y_test)))/2/Y_test.shape[1]) * 100\n",
    "print(accuracy_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}