10_Training_Deep_Neural_Networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "173110e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Most Important\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "## less Important\n",
    "import os\n",
    "from scipy import stats\n",
    "import missingno as msno\n",
    "import joblib\n",
    "import tarfile\n",
    "import shutil\n",
    "import urllib\n",
    "\n",
    "## Sklearn\n",
    "from sklearn import datasets\n",
    "## Preprocessing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "## Metrics\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "## tensorflow & Keras\n",
    "import tensorflow as tf    ## i will use tf for every thing and for keras using tf.keras"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1218bc8b",
   "metadata": {},
   "source": [
    "## Vanishing or Exploding the Gradients\n",
    "1. Choose a best Activation Function\n",
    "2. Choose a best Initialization technique\n",
    "3. try using a faster Optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef1c761c",
   "metadata": {},
   "source": [
    "### Loading `Fashion MNIST Dataset` "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "73f44acd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_full shape => (60000, 28, 28)\n",
      "y_train_full shape => (60000,)\n",
      "=============================================\n",
      "X_train shape => (54000, 28, 28)\n",
      "y_train shape => (54000,)\n",
      "=============================================\n",
      "X_val shape => (6000, 28, 28)\n",
      "y_val shape => (6000,)\n",
      "=============================================\n",
      "X_test shape => (10000, 28, 28)\n",
      "y_test shape => (10000,)\n",
      "===========================================================================\n",
      "X_train , y_train -- dtypes => (dtype('float32'), dtype('int32'))\n",
      "X_val , y_val -- dtypes => (dtype('float32'), dtype('int32'))\n",
      "X_test , y_test -- dtypes => (dtype('float32'), dtype('int32'))\n"
     ]
    }
   ],
   "source": [
    "### Building a Sequential API for multiclass Classification\n",
    "\n",
    "## loading the Fashion MNIST \n",
    "(X_train_full, y_train_full), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "\n",
    "## show shapes, dtypes and transfer datatypes to (float32, enough for NN)\n",
    "## and rescale the inputs to be from (0-1)\n",
    "\n",
    "X_train_full = (X_train_full/255.0).astype('float32')\n",
    "y_train_full = y_train_full.astype('int32')\n",
    "\n",
    "X_test = (X_test/255.0).astype('float32')\n",
    "y_test = y_test.astype('int32')\n",
    "\n",
    "## split train to train & val\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train_full, y_train_full, \n",
    "                                                  test_size=0.1, shuffle=True)\n",
    "\n",
    "\n",
    "print('X_train_full shape =>', X_train_full.shape)\n",
    "print('y_train_full shape =>', y_train_full.shape)\n",
    "print('==='*15)\n",
    "print('X_train shape =>', X_train.shape)\n",
    "print('y_train shape =>', y_train.shape)\n",
    "print('==='*15)\n",
    "print('X_val shape =>', X_val.shape)\n",
    "print('y_val shape =>', y_val.shape)\n",
    "print('==='*15)\n",
    "print('X_test shape =>', X_test.shape)\n",
    "print('y_test shape =>', y_test.shape)\n",
    "print('==='*25)\n",
    "print('X_train , y_train -- dtypes =>', (X_train.dtype, y_train.dtype))\n",
    "print('X_val , y_val -- dtypes =>', (X_val.dtype, y_val.dtype))\n",
    "print('X_test , y_test -- dtypes =>', (X_test.dtype, y_test.dtype))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "11230df0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAAFDCAYAAAD/OFz6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCHklEQVR4nO3debxdVX3//88SCGSe5wRCyASEyJwGmUIoAoK21q80FKFqTVMf9kEVvrbV/oqiVR9iEUqLCiKVoQIVCFODWhADBAIkzBAgJGQezURCwuT+/XHj1/t575Wzzrm595517n09H4882s+95+69z9nrrLM4vvdnh6IoDAAAAMjVB+p9AAAAAEAlLFgBAACQNRasAAAAyBoLVgAAAGSNBSsAAACyxoIVAAAAWWPBCgAAgKyxYN1DIYRRIYRfhxDeCiEsDCGcWu9jQn5CCN8IITwfQngvhPC1eh8P8hRCGBRC+FkIYVUIYUsI4dEQwuR6Hxfys+tzZ30IYWsI4dkQwsfqfUzIWwjhpBBCEUL4Zr2PpSVYsO65n5nZ02bW38y+amY/DyEMrO8hIUOLzOzLZnZfvQ8EWethZk+a2VFm1s/Mfmpm94UQetT1qJCjC81saFEUvcxshpndFEIYWudjQqZCCPuY2ZVmNq/ex9JSDbdgDSH83xDC7fKzq0IIV9ThWMaZ2ZFmdklRFDuKorjdzJ43sz9r72OBl9M4MTMriuKnRVHMNrM367F/7F5OY6UoisVFUVxeFMXqoijeL4riGjPrYmbj2/tY4OU0TszMiqJ4riiK935fmtk+ZjayHscCL7exsstFZvZLM1tYx2PYIw23YDWzm8zs9BBCHzOzEMLeZnaOmd0Ye3AI4d4Qwubd/Lu31seJQ81scVEUzRchz+76Oeorp3GCvGU7VkIIh1vTgnXRHj5H7Lnsxsmuv91pTd+aPWRmT7XOU8UeymqshBAOMLPPmNmlrfkk29ve9T6AWhVFsTqEMMfM/o+ZXWtmp5vZhqIo5u/m8WdVud2qHid6mNkW+dkWMxvegm2hFWU2TpCxXMdKCKGXNX3Afb0oCp1n0M5yHCdFUZy163/qPdXMJhRF8buWbgutJ8Ox8m9m9v8VRbEthNDCTdRfI37DataU6zpv1/9/nu3mv1rawTYz6yU/62X8z765yGWcIH9ZjZUQQlczu8fMHi+K4tv1PBY4WY0TM7OiKN7dFTf6cAjho/U+Hvw/WYyVEMLZZtazKIpb67H/1tSoC9ZZZjYphDDRzM4ys5t398AQwuwQwrbd/Jtd6+PEi2Y2OoTQs9nPPrjr56i/WZbHOEH+ZlkmYyWEsO+u41lpZn/dWk8QrWKWZTJOIvY2s4Na+sTQ6mZZHmNlmpkdHUJYE0JYY03RhL8LIdzVek+1fYSiKOp9DC0SQrjWzCZb09fsp9TxOB43s0fM7J/M7Awzu97MxhZFsb5ex4Q/yGic7GNme5nZT8xssZl908zeLYri/XodE7wcxsqucXKHmb1vZp9odlENMpHJOJlgZgdaU271PWtahPzEzP6oKIoF9TgmlGUyVnqaWfdmP7rSzFaZ2TeKothYj2NqqUb9htWs6ev2w6z+/5PMn5vZ0Wa2ycy+Y00fMixW85HLOLnWzHaY2XRran+2w8w+VdcjgsphrBxnTd/GnGZmm5t9e3JCHY8JXg7jJJjZ18xsnZmtt6YWV+ewWM1O3cdKURRvFkWx5vf/rOmzZ3ujLVbNGvsb1v2tqT3DkKIottb7eJAnxgmqxVhBNRgnqBZjpXU15DesIYQPmNmXzOwWBgF2h3GCajFWUA3GCarFWGl9DdfWKoTQ3czWmtlSa2oVAZQwTlAtxgqqwThBtRgrbaNhIwEAAADoHBoyEgAAAIDOIxUJyO7r1yeffNLVM2bMKD1m+vTprh41apSrJ0+e7Op33nnH1StXrnT1e+/5zjJXXnllaZ+9e/d29RVXXOHqAQMGlP6mDtryFhfZjRX11FPluxbefru73bONGzfO1Tt27HD1Xnvt5eru3bu7+gMf8P8NuH379tI+P/e5z6UPtv7aaqxkN05effVVV7/4YrmNsp7HzZs3u7pLly6u3rlzp6t/9zt/A6L99tvP1ccdd1xpn5MmTYofcF6ynlP0ddf3p/4vjE888YSrt23b5urXX3/d1W++Wb5HzJFHHunqqVOnVnewe+Cll15ytT4PpZ9H/fr1c3VsPLaCTjOntMTChQtdvffee1es9ZwPHDjQ1cccc0wrHl272u044RtWAAAAZI0FKwAAALLGghUAAABZS3UJqHs2ZPDgwa5+6623XB1COe6gmVPNIf7t3/6tqzW3qL9XvXr1Kv1Ms1Kai9XslB5TO8k6b9bW/vIv/7L0M82wDhkyxNWrV692tWZWu3Xr5uquXbtW/HszszfeeMPVmn/ORIfJm23c6G/o8txzz7n6Rz/6katvueWW0jZ07Gie7L//+79dfcop/i6MOq40+xjLOus2zjjjDFcfdFBtt43XuT42d7ZA1nPK22+/7ep9993X1XqtwXe/+11X9+/f39U6r8c+P999911XL1u2zNX6+XH00Ue7eujQoa5++OGHK27frDzv6HhT+nmkz+vOO++saXtV6jBzSq00C21WPgc33HCDq59++mlXr1u3ztWamz/77LNd/ed//uelfW7ZssXVOr4zQYYVAAAAjYkFKwAAALLGghUAAABZy+7WrJo52rRpk6s1QxjLYWmuUDOtixYtcnUsP9ac5nd0e2blPIo+5vDDD6+4D7Q9zZKZmQ0bNszVmift27evq3v06OHqffbZx9WakdNss5nZK6+84upjjz12N0eMaixZssTVN998s6v19dYeqJoVXbx4cWkf9957r6s/8YlPuPrDH/6wq0eMGOFqzd5rdkxz9GZmy5cvd/UPf/hDV+tY1Uyr9qNupcxqQ0k9Z80Fav9c/WzQ97/2ZTYrz/3jx493tc4ZmrHWjLuO10GDBpX2qfOUZqw3bNjgas1Uvv/++67W16WVMqwdlp5znZNinwMHHHCAq3UNoeuUBx980NWaUdVzpGPAzGzVqlWuXrFihavHjBnjal1v1RvfsAIAACBrLFgBAACQNRasAAAAyFp2GVbNAGrWq5r+pfoYzWHMnj274t/rfZU11xSj+RPNrPTp0ye5DbStZ599tvQzzbBqlktrHVt6nrUvo46L2HGQYa2Nvn7aS1ff75of17yZ5ua/+c1vlvZ53XXXuXrp0qWunjZtmqs1e680cxjL0Wt2UbOPOtbmzp3ras3AfvnLX654TB1RLMfXnI6NmTNnunrs2LGu1hyx9rU0K/fH7Nmzp6s1e6yfeTp+tWevjlezciZVn7de13HJJZe4Wnu9ZtqfM1sLFy6s+PsBAwaUfqbvb50Dpk6d6uqjjjrK1TqPxfrDK5139PNMs7cTJ05MbrM98Q0rAAAAssaCFQAAAFljwQoAAICssWAFAABA1rK76CqlmotaYj9rbuDAga7W5tLvvPNOC4/uD/Q4tTk02p9evGBWvpAldZFGrFF4cxqE1wsszMyefvrpittAZfPnz3e1XlCiF7m8++67rtZzrOfsxRdfLO3zk5/8pKt1LK1evdrV2txdL87bunWrq2MNulMXWSm9WYEek95AQRvad0SpzwI9r1qrn/3sZ67Whvxm5Qtb9PNEa72ISsdr7CIrpWNYLwq89NJLXX3GGWdUrFEbPUd6IZ1evBuj40ab+g8fPtzVeo4PPvhgV69Zs6a0j5deesnVesMSnZdywzesAAAAyBoLVgAAAGSNBSsAAACyln2GNZX9SOW6YjQjVKvYPlPZ2j3dJ2qnTZFjY0mbfGueWXOEqRtEaK3NoWPHhdpoNiv2GjeXyh3rOdU8oJnZokWLXK3jRHOxqdyhHrOOG7P03PbWW2+5WrO7uo9HH33U1Z0hw5qic4LO0/oannjiia6ONYzXc683nunRo4er9eYDmn/Ux8fmj9RxH3/88aW/aU6POTUvdnaa9UytU+6+++7Sz2bMmOHqMWPGuPqZZ55xtealDznkEFfr+z92U4uRI0e6WucYnfs0q6/7aG98wwoAAICssWAFAABA1liwAgAAIGvZZVg1y6V5nVQerT1Uk+fR7OPy5cvb6nCwG6lso1k6s5rq0VtrPjK2TdRm06ZNrtaep5q/01yWnpNq+i6nMn06DjRTmMq4xcZJKsO6bds2V/fr18/VmjdbuXJlxe11RrV+nmiG9d///d9Lj9HzpvlmPW86lvQzUP9e+3WaxXPXzen4VboP5qjKtm/f7uo+ffq4WnuoxjLEes50HtPfb9682dUnnXSSq5977jlXDxs2rLTPtWvXulrnKf380/w1GVYAAACgAhasAAAAyBoLVgAAAGQtuwzrnDlzXK15H82KtKQPa3vQzJDmkjQbUk3eErVJ5QzNyudF70uvGWrNJo0ePdrVmoeMZRM519WLvb8126XnSHNWet51my3JxacyqXqONStWzThJ7UN7JGoOUbe5evXqittD2gc/+EFX9+rVq/SY1GdSrOduczoeu3bt6upYHlX7qGrGcsCAARX3mevnaK60x6nWjzzyiKtPOOGE0jYmTpzo6v79+7ta++/qmkEff80117j6oosuKu3zqaeecrWOrSlTprha55iBAweWttme+IYVAAAAWWPBCgAAgKyxYAUAAEDWssuwfupTn6r4e83/xLI3sTxYa4rtU/OSmjPauHGjq0877TRXa3YXrU/zOmblPKSeR80R6v2chwwZUvHvta+dmVn37t3TBwszK7/eZuV7reucsG7dOlcfcMABFfeh51jzgNX8jdI5Qutq8tXab1bz1fo8U9nIap5XZ5fqP9q7d29Xa37arDw29LqL1HhLfZbExorOY+PHj3d1as7Job95I9E1hmbUFyxY4Oq33nqrtI2pU6e6Ws/z4MGDXf3KK6+4WsfVpZde6upVq1aV9qnj94UXXnD1cccd52rNzdYb37ACAAAgayxYAQAAkDUWrAAAAMhadhnWD33oQ66+9dZbKz4+lidN5cf2VCznlMqPVZNZQ9uK5bg0T6ZZI+1199d//deu1vE5atQoV8fGhd7zHbsXy7BqllPzZHq/bM166euv5zyWT9WMmmYGU/dqVzouYnOK5iM1f6a9H3WbmvXV54CyWq9/OP3000s/u/fee12t503HtPYN1s8r7YUZm8c0K6/9YlNS2d3OTt9bWuu40XN83nnnlbb57LPPuvrQQw91tfbOffXVV12t89jcuXNdrTlmM7O+ffu6WucEnXPIsAIAAAA1YMEKAACArLFgBQAAQNayy7B+4hOfcLVmBKvpw9rW90VO5VVj9N7hRxxxRGsdDqqkfe3Myj0QNc940EEHuVrH55VXXunqww47zNWbN28u7bPe92NuJPq+MSvnxbTWbKf2K9XeuameqmbljGCtGfRUhj12DPq89Bj0tdl7bz+d6zxFH9bWd9ddd5V+pveA1/Ok516zijpeNRevmVaz8lhZs2ZN/IB3I5XJ7Oz0vaMZdv295ktjvaA1c5rKIWue9Le//a2rddwce+yxpX3qcWn2fvv27a7O7VobRiUAAACyxoIVAAAAWWPBCgAAgKyxYAUAAEDWsrvoasyYMRV/rxcnxBp2ayC6Hk2RUzcviIWw0baGDh1a+tnixYtdrWMldcGE0kbMsWbt2sgeuxd7/fSCkF69elXcht5oQC9uaAk9Bp2XdA7S97+Os9gcpdvUbegFfXqjAL0QpJobqtBAvrK///u/d7Ve4GdmNnz48IqP0bHTtWtXV2/dutXVep70wpiYhQsXuvqGG25w9fnnn+9qzntlqZtu6OtXzefA/vvv7+pHHnnE1R//+MddrRfr3nLLLa7WC4L1BiqxbeicsWnTJlfrWNYbC+jYbWt8wwoAAICssWAFAABA1liwAgAAIGvZZVi1Wa5mLDTPE2vir9kvbaitUjcaqOZGAboPzaxoxkWbl6PtjR49uvSz119/3dWaUdW82THHHFNxH9Xc2CKVucQfaGbKrHyONKsZu9lApW1W08S/VjpuNNOqx7jffvuVtqHHpXXqBgia79c5Sm+SYWbWvXv30s86s+XLl7v6tttuc7U2Yjcrn0tt8K5jQ89Lajz37ds3eZwjRoxw9Ve+8hVXk2GtjX6eaw5es586x8Ru2qGvud7YZunSpa7Wzxa94YRuT28kYGY2aNAgV6cyrDqO9HmQYQUAAACaYcEKAACArLFgBQAAQNayy7BqDkPzPZrTiuVLU5nUWlWzPT3u1N+0Ri9I1Gbs2LGln82ePdvVeh41N5jKQyvNq5mZ9e7du6ZtdGax97fmyTSrqZk/7cPaFvQ4a+3XGxsTqdyr/j6Vn9axu23bttI+ybB6X/rSl1ytud9YH3Cl501zf/pZoJ9x+vfaS9PMbNmyZa4+6KCDXK1ZRe35efzxx7s61Te4s9H3q54j7Xk6btw4V8fmA8256jUW8+fPd7We42nTprla+/327NmztE89bs1bt0e+f0/wDSsAAACyxoIVAAAAWWPBCgAAgKxll2FV2nNOsyKazTFL5000V1hNn9XU42vNsKL9xbJfqdxfrdmtVL8+s3jPTcTF3mua6dNemHoOUj1Rq8llxbLIlY5Tt5m6t7gek1l5LOpjdBzp66KZVXLz5ddQXyPNAb744ouu1tcwlmlP9VHVnLDmBDUnu3PnzoqPj+1Ds4ia477yyitdrRnWzp5ZVfr66uuj5+iII45wdWwe07Gktfbr1rGrax+dU2J59NRY3L59u6tTeer2xjesAAAAyBoLVgAAAGSNBSsAAACyln2GVXvWaaYilj+rNT9aa26xmgxrSnv0hoSnmSCzdF/VVN5Ux6dmx2J/r/eAx+5pNixGX/MePXq4etiwYa5ev369q3UMxPaZGif6/k/VOm/Fcomam9V8pc4hOi/pMev2Yvc37+hSWeSrrrrK1Vu3bnW1vsax97dmUPVvtHerfv7o2NDxXQ3NGmpPTu3DmhL7zEu9lh2JPv9UDl6vvalmfaDneeTIka6eMmWKqx9//HFXDxo0yNWx85M6Tp0TUtcDtbfOM+IAAADQkFiwAgAAIGssWAEAAJC17DOsqR6q1WRYW7snaktyHKn7eqPtab9Os/J50KxSnz59Km5Ts2HVZBNj/WARF8uTar9CzX5t2rTJ1Zoz1HOk5zyW/dLzqNku/b2Oq9TjY+NEc2/6vCZMmOBqzU5qvlpzjdpzsTNI5S5vvvnmio+vJsO6ceNGV9f6+aNjIdXjN0bP7YABA1zdv39/V8+bN8/VkydPTu6jM9P3pp4zfa/F3t96XnUeS2XYNV+tmXb9bDIrj0WdS3Vs5dZPnm9YAQAAkDUWrAAAAMgaC1YAAABkLfsgpWYsUvd+r0asp1xzqdxSNVJ912L3DkfbimV6UnmxwYMHV9ym5gR1PGrPRbPO1b9wT8UyrPqabt682dV6DjXbpe893V7svZnKsGr2S9//us3Uvd/Nyse9ePFiV5922mmuXrRoUcVjSuXsOqJUj+0VK1a4OnW/dt1ebKzo2EjlHVXqPFXTB1zzjmPHjnX1hg0bXP3zn//c1Zph7exzVur9rPOU/j62HtDzqu9XHXsq1c839v7Wbaby/fo8WrIWak2dexQCAAAgeyxYAQAAkDUWrAAAAMgaC1YAAABkLfuLrrQ5rl6EFQsz1yMYXGtgujNc8NAIUhfYpZp0ayhda24QsWfefvvt0s/0Qrd169a5WpuiDx8+3NVz5851td4cIjZ/pC620XGSunGAio0zvfhGbxwwYsQIV7/22msV99EZx2LqoqvrrrvO1dpwv3v37q6u5rym9qnbSD1ex0ZsrOg29SYSelGfXtD3+OOPl7aJP0i9v/VzRC/wreYia10z6DjQfeg2169f7+qRI0eW9qHbSN2sQI+h3jcS4BtWAAAAZI0FKwAAALLGghUAAABZyz7U9Oabb1b8fSxTkcqwajYs1Ry3mtyGZpk0Z6fWrFmT3CZaVzXnUc99aixpFln3Ue/MT6OL5fX0NdVzNGjQIFdrg21tqt6rVy9Xx/LnqfOYulGIimVzleYMNc8/YMAAV2uuTvdR6zF2BKmG97fccourdd5OZfhiGVb9fNFj0FrnEL1Oo5qbFSh9jGYVtWH8c889l9xmZ5a6+YOes759+7o6Nk50HOh5TuVJdV5L5Zhj29TjXLp0qat1Lkxl8dsa37ACAAAgayxYAQAAkDUWrAAAAMhadhnWhQsXunrbtm2u7tGjh6tjeZ5as1qpfmdt0df1gQcecPWMGTNafR/wYr1v9dxqNklrlVufuo4m9t7T11x7Zx599NGu1j6rO3furLjPWE5LM2qarU3NEalcfGzO0p/pcWu+TLO7y5cvr3gMqbHdEelrojlf/XzRsaDnpJrxqa9z6vNJf6917DNP96HjM/U8Bw8e7Op58+a5evLkyRWOuOPT3LuOi9R8UM3ngmZSdZv6+bVy5UpX9+7d29Vjx44t7UOzyxs2bHC1Zp01B1tNfrot8Q0rAAAAssaCFQAAAFljwQoAAICsZZdhveeeeyr+viWZwVRvzFRfx2qkMiuagXv44Ydr3gf2TCzDmpLKm6XOe1vkn+G99tprrv6rv/orV6fyfdq/sJoMfOp+76l7jVfTh1W3qWNLM2+aQ1y0aJGrtW+rZn87gxtvvNHV+hqnel+2RKovq84Rtd5D3qw83nSbmrlM5WrvuusuV3f2DKvmQfX1mzRpkquXLFni6k2bNpW2qWOv1jlFe6hqxj02TrSv/bhx41x9xBFHuFqzzjrntDe+YQUAAEDWWLACAAAgayxYAQAAkLXsMqy/+tWvKv6+mv5mqXxO6v7SqZxsNf0L9THa/2z16tXJbaB1tUWGVXNCmjvS3nioTez9ra+x5vM0l6V5UZ1DtNasmFn5POvfaLYr1a9Q/z6WadV5KvW8Dz74YFfPnj3b1ZpH056LnYFeI6HnWq810PNSzTUTqc8XPfepeamavuCp8ZbqE6o9fefPn19xe51N9+7da3q8jpPYmNCxVuv1Dv3793f15s2bXR17f3fr1s3Vuk454IADajqG9sY3rAAAAMgaC1YAAABkjQUrAAAAspZdhlVzGKqaDGuqt55uQx+fEsua6D5S9ytH+4udZ/1ZKnucksqvoTax11OzWfqYPn36uFrvua2ZQc2Cxug4SWUGYznY5jQbGXueW7durbgNfR7ah1XnxlSOsTN44oknXK2539RYqOY1a43erZXEjlHPrWaq9Zh0/Oo899xzz+3JIXY4+l5KnWOdo2KfIzpH6D50jtBzpseg80FsrOrn25YtW1ytY0tztvXGpysAAACyxoIVAAAAWWPBCgAAgKxll2FN5baq6W+Wup976h7dqT6ssdys5ks0f1JrThatr5p8qZ6nWvvvpXp8ojax16+avqnNpXpntiRzmOrFrMeo/Q9TebXYz7SP6v77719xn5pp0xxdZxibTz/9tKv1NdOsZ+qzQOeQavqy6jZ0nzp+dWxpj9TY9RE6T6V6fOpx6+OHDBni6pdffrm0T83/dmS1zhHV9H7XNUOt173oONqwYYOrJ02alNxGai2UG75hBQAAQNZYsAIAACBrLFgBAACQNRasAAAAyFp2F11pwFxpODwWZtYQe+pv9OKEVLNcDTubpcP6+jd6DHrDBG1+jj0XC5SnwvS9e/eu+Hsdr9VcFIjqxS6o0osVevXqVXEbqQukUjcW2d3PKu1Dx5U25NYLoN56663kPsePH+9qPe7UBUF6gUVbN7jPweWXX+5qbZSeOvd63lLN283SDeE3btzoaj33+vjt27e7OnYBr16srNvQ8anb0HlMj+lHP/pRaZ9XXHFF6WedVa1N/s3KY0/nBF2X6BpCz5FeeBdbS6UuvtNtxtY69cSnKQAAALLGghUAAABZY8EKAACArGWXYU3dOODNN99spyNpXZpr0ozLI4884uqzzjqrzY+ps4mNHc2oqVhD9+bWrVvn6sGDB7u61mbQ8DTbbVY+J5r1UtpwX7epuS3NDJqVc+06llJZSH28ZiljGVkdm/fcc0/pMZXo69S1a1dXL168uPQ306ZNq2kfuTvllFNcvXTpUlf369fP1Zr1XLRokavXrFnj6gEDBpT2qeNn1apVrp4wYYKr169f72odC/pZoefRzOzYY491tTaN1/yz3hjg+uuvd7VeQzFq1KjSPvEHqfe/vt/NyvOW5kn1/a9rI83F6jymNxIwK+dkNeeauglLvfENKwAAALLGghUAAABZY8EKAACArGUXWBg5cqSrDz30UFcPHz7c1QMHDixto3///q6O5eCaGzp0qKt37Njhas0hxvp5amZFc0maJ5s3b56rNUuF1nf44YeXfqb5xr59+7p6ypQpFbd58803u/pb3/qWqzUrhtoMGjSo9DPNcsUe01zPnj1dPXPmzIqPT+WWzco5xVTuULepWclY72fNKp5xxhnJ42puzJgxrtZ57cADD6xpe43o05/+dMUaTc4999x6H0JD0/fv5MmTXa3zgVk515rqEZx6/06cONHVej2FWfnzrlapfvNtjW9YAQAAkDUWrAAAAMgaC1YAAABkLcTymAAAAEAu+IYVAAAAWWPBCgAAgKyxYAUAAEDWWLACAAAgayxYAQAAkDUWrAAAAMgaC1YAAABkjQUrAAAAssaCFQAAAFljwQoAAICssWAFAABA1liwAgAAIGssWAEAAJA1FqwAAADIGgtWAAAAZI0FKwAAALLGghUAAABZY8EKAACArLFgBQAAQNZYsAIAACBrLFgBAACQNRasAAAAyBoLVgAAAGSNBSsAAACyxoIVAAAAWWPBCgAAgKyxYAUAAEDWWLACAAAgayxYAQAAkDUWrAAAAMgaC1YAAABkjQUrAAAAssaCFQAAAFljwQoAAICssWAFAABA1liwAgAAIGssWAEAAJA1FqwAAADIGgtWAAAAZI0FKwAAALLGghUAAABZY8EKAACArLFgBQAAQNZYsAIAACBrLFgBAACQNRasAAAAyBoLVgAAAGSNBeseCiG8EULYEULYtuvfL+t9TMhTCOHCEMKSEML2EMLLIYRx9T4m5CWEsH+zueT3/4oQwkX1PjbkJYRweAjh4RDClhDCihDCP9f7mJCnEMJxIYQnQghvhhCeCyEcX+9jagkWrK3j7KIoeuz6d1q9Dwb5CSH8lZl91sw+YmY9zOwsM9tQ14NCdoqiWNZsLulhZoeZ2e/M7PY6Hxry819mNsfM+pnZSWb2NyGEj9b3kJCbEEI/M7vbzC4zsz5m9l0zuyeE0Leex9USDbdgDSH83xDC7fKzq0IIV9TpkJChnMZJCOEDZnaJmX2xKIqXiiavF0Wxsb2PBWU5jZWI881sTlEUb9T7QDq7DMfJKDO7uSiK94uieN3MHjGzQ+t0LGgms7FynJmtLYriv3eNlZvMbL2ZfbwOx7JHGm7BamY3mdnpIYQ+ZmYhhL3N7BwzuzH24BDCvSGEzbv5d2+tj9uNm0MI60MIvwwhfLCVnif2TE7jZMSufxNDCMt3xQK+vmshi/rLaayo883sp3v29NBKchsnV5jZ+SGEfUII481sipn9bys9V+yZnMZK2PVPfzZxT59kuyuKouH+mdlsM/vcrv//LDN7qY7H8iEz62pm3czsH81sjZn1qfdrxL98xok1/RduYWb3WdP/JDPKzF79/bHxr/7/chkrckwnmNk2M+tR72PhX37jZNe8ssjM3ts1v3y93q8P//IbK2bW38w2m9l0M9vHzC6wppjRj+r9GtX6r1G/4fmpmZ236/8/z3bzXy3toSiKR4ui2FEUxVtFUXzbmgbGCfU6Hji5jJMdu/7vd4ui2Fw0/c+7PzKzM+t0PCjLZaw0d4GZ3V4UxbZ6Hwj+nyzGya5c4v1mdqmZ7WdmI83swyGEz9fjeBCVxVgpiuK3ZvYxM/uSma01s9Ot6Zv4FfU4nj3RqAvWWWY2KYQw0Zr+y+Xm3T0whDA7lK+6/f2/2bU+rgqFlb9+R33MsjzGyStm9o41jQ3kaZblMVZ+/7ddzez/GHGA3MyyPMbJaDN7vyiKG4qieK8oihVmdovxH8E5mWV5jBUriuI3RVEcUxRFPzP7lJmNN7MnWu2ZtpOw6yvjhhNCuNbMJpvZhqIoTqnTMexvTf9l+6Q1Lf7/1sy+bGYTdv1XDeosh3Gy6zhusKareaebWW9r+i/cy4qiuK5exwQvl7Gy61jONbNvm9moolEn6Q4qh3ESQuhlZsvM7PPWtFAdZGZ3mtmDRVF8tR7HhLIcxsqu4zjCzF6wpvjipWZ2TFEUH6rX8bRUo37Datb0zcNhVt//6a6nmf3AzDaZ2Upr+qr9DBarWclhnJiZfcGa8oirzOwxa2pJ85O6HhFULmPFrCkOcAOL1SzVfZwURbHVmq7y/qI1ff48Y00Lkn+p1zEhqu5jZZcvW1MbxeVmNtTM/rS+h9MyjfwN6/5mttDMhux68wIljBNUi7GCajBOUC3GSutqyG9YQ1M7oC+Z2S0MAuwO4wTVYqygGowTVIux0vr2rvcB1CqE0N2arnRbak3/EzxQwjhBtRgrqAbjBNVirLSNho0EAAAAoHNoyEgAAAAAOo9UJICvXzuWtuwP2+5j5f3333f1XnvtVfM25syZ4+qvfOUrFbf5zjvvuLpLly6u/trXvubqqVOnJo9B/1cOrT/wgbr8d2VbjRXmlI6lQ80p7WHVqlWufuaZZ1y9ceNGV/fr18/VsTmla9eurXNwbYs5BdXY7TjhG1YAAABkjQUrAAAAssaCFQAAAFlLdQkgG9KxdOq82Y03lm82cu2117p6+fLlru7du7er3377bVeH4F/SSZMmufoLX/hCaZ/HH398+mCb+d3vfufqdsq0kjdDNTr1nBKzadMmV995552uXrhwoav79u1bsV6/fn3F2szszDPPdPXpp2fZSYk5BdUgwwoAAIDGxIIVAAAAWWPBCgAAgKyRYe1cOlTe7Le//a2rr776alc///zzrtZ8qpnZ5MmTXf3www+7evXq1a7u37+/q8eOHevqbt26uXrvvcutjvfbbz9XT5w40dWx3Gtz7ZRpJW+GanSoOaVW7777bulnl19+uau3bdvm6r/7u79ztc4pKdo72szsuuuuc/XMmTNdPWXKFFfr575m8dsIcwqqQYYVAAAAjYkFKwAAALLGghUAAABZI8PauTR03mzBggWuvuqqq1w9cuRIV/fp08fVQ4cOLW3ztttuc/XOnTtdre8Pzah26dKlYn3kkUeW9qm52HfeecfV27dvd/X3v/99V3fv3r20zTZA3gzVaOg5JXkAiazn3XffXfqb3/zmN67+zne+4+p99tnH1e+//76rNaOu9b777lva50MPPeTqe+65x9X/+q//WnGfe+21V2mbbYA5pRXpNRovv/yyqzdv3pzcRurzbfDgwa7ef//9Xa2fd9q3PLaPrl27ujryeUaGFQAAAI2JBSsAAACyxoIVAAAAWWPBCgAAgKxx0VXn0tAXSEyfPt3VGt7WwLfeWOC0004rbVMvcHrggQdcrRcnaJP+4cOHu3rMmDGu3rRpU2mfeoHDW2+95epnnnnG1UcddZSrv/71r5e22Qa4QALVaOg5pVb6eXnxxReXHjNt2jRXn3nmma5+++23Xa0XUaXmHL0Iy6w8p1xyySWuPuGEE1x96qmnVtxnG12ExZxSQa03hDn88MNdrRfzxcbJa6+95mq94FfHZq0++MEPln6mY+uFF15w9datW13ds2dPLroCAABAY2LBCgAAgKyxYAUAAEDW9q73AaDJ7bff7upf/OIXrv7e975X+ptevXq16THlZseOHa7u2bOnq9euXevq9957r+Lvzco5n+OOO87VGzZscPWyZctcrY2Wly9fXvEYzMpZW92HZlr1ZgYA2kbqRgF6U4A333yztI2pU6dW3Mfee1f+2NXsoh5DNRlWnRvnz5/vas2wtobUa4fKan29zj//fFc/8sgjrtZ8qpnZscce62rNrOrY0s/cdevWuVrPeSwDO378eFcfeuihrl66dKmrJ06cWNrG7/ENKwAAALLGghUAAABZY8EKAACArHXIDGuqn5n2/frOd77j6oceesjVU6ZMcXUsl6jZRe3vuXLlSldr/lSPSX+vx2hm9q1vfav0s45EX+cuXbq4OpX50T6t2pfVzGzgwIEV96kZtf3228/VmuHRY9Q8qpnZli1bXL1+/fqKx6BjCUBZoqd4lOb89LND55DFixe7WrOiZuX3vG5D91lrprUaOqek+mtqBvbdd991dewYdf4ls7pnas0Az5kzx9Wvvvqqq2NZ51WrVrl62LBhFfehj9c+rgMGDHB17P2g12joZ26lzKriG1YAAABkjQUrAAAAssaCFQAAAFnrlBnWxx57zNU33XSTq/UevdojNZZh7dGjh6tfeeUVV48cOdLVmjG64IILXP3888+7+r777ivtM5VhrfXexLlZvXq1qzdv3uxqfc01O6Z5mljPRO2jqq+RZrd0n/r3et9kvb+zWTmTqr3t9Bhi2Vs0Fh27mmOO5ZQPOeSQPdpnO90fPhstyVDuu+++NT3+xz/+saunTZtWekyqP7ZmWmtVzXmcN2+eqzU3mMpLxuatFB1vqX6y2DMbN2509ahRo1yt11OYlfu7H3DAARX/RjOsw4cPd/W4ceNcHXs/6XFqrZ9v/fv3L23j9xprBQMAAIBOhwUrAAAAssaCFQAAAFlr+AxrrNdYysknn+zq//zP/3T1Kaec4mrtYRfLFPbt29fVd955p6s1t3Taaae5WvOXF154YcXfm5l973vfc/XFF1/s6kbLrCrtbasZKa31vGgeJ9YjTh+jWcNYXrk5zYJpviyWN9u0aZOrdXzpcer9nNF4NLOu7+dnn3229Dc6nk844QRXpzLqOvZakmnVHPnQoUOTf5OzJ5980tUrVqxw9bZt21yt7z29vkEz7WZmP/3pT12tmTzN+fXp08fVqZ6o2rPbrJwt1G1ohnXBggWuXrZsmat1Hozlck8//XRXd+/e3dUt6YvbmdWa8dU1xdq1a10dG5s6DnRO0M+7CRMmuFrnKf2s0rFqZjZ69GhXL1myxNU6x5BhBQAAQMNiwQoAAICssWAFAABA1ho+wxrLaaaym5oh0sxq6vGp+++amZ177rnJxzT3D//wD67WPq4nnXRS6W/uuusuV8+YMcPVqX6AudNsi/Y81R6p2mdV8zWxXpc7d+50tWYLNZOjmddU5i02FvUxug8db2RY21csh5XqS3nvvfe6evr06a4+/vjjXa3v7x/84Aelbb744ouu1vzomDFjKh6TqiazeuaZZ7r6nHPOcbX2i64nzX7HekDqHDl79mxXDx482NX6ftY+lfpe1H6nZuW+ygcddJCre/fu7erU2NLxGJvHND86aNAgVy9atMjV//Iv/+JqvSe8vi7vvPNOaZ+apf30pz/tavqu1qbW10s/D/WzKzZOjjzySFdrhlUz3Tp2NRe7dOlSV+t70qz8vtS89cSJE0t/szt8wwoAAICssWAFAABA1liwAgAAIGsNl2Gtpu9qKsOq29Bas5H6+2oybqltPvTQQ65euXKlqzX3oT1Xzcz+6Z/+ydU33HCDq7/whS+U/qaRaJ857XWb6kOpPeU0n2aWzgHpMWhvwQ0bNrhac7Ox/rna41DpWImNN7SdWKZQz+Mll1zi6l/+8peu1pzWyy+/7GrNk2rfZTOzU089teJxam6x1h6pmrM1K+diH3jgAVfnlGGN3Stdvfbaa67WOUTPtZ43nR8ee+wxV8fe3x/96Eddrf1Ktb9urf1yY3lSzcXOmjXL1U8//bSr9XX47Gc/W3GfsTloy5Ytrtb+0roP7Jnrr7/e1W+88YarDzzwQFfr/GBmtt9++7lac/IvvfSSq7VnqvZUPeyww1ytWWizcn/Y1DVDlfANKwAAALLGghUAAABZY8EKAACArLFgBQAAQNYa7qKr1AVVLdlGapv6+1iD6tRFVtrE9+yzz3a1NhbXi3dix6ih62XLlrm60S+60ibd+hpqU29taqwXN40cObK0D73oQps3p24coOddb0SgF9OZlcdP6oI9pReT6VjrbFKvV61zxr/927+Vfvb973/f1XqedSzqRSu13gTDzOzHP/6xq6+++mpX33///a7WCwAfeeQRV+sFVAMHDiztU19LvcBHG4uPGDGitI22ou9Ffd/ozRvMzObOnevqyZMnu1ovmtKbj+jroReM6IVGZuULUxYuXOhqvegqdWMQHb+xBvN9+vRxtT5PnTP69+9fsV6+fLmrdbyalS+maY3P5npJXehWj33q57teVD127FhX6+dd7LNH5xm9GO9DH/qQq/Wc6rjQGweceOKJpX3qe2pPXtvGHWEAAADoFFiwAgAAIGssWAEAAJC1iuG3VGP2tlCPfbaG1HFqVmTq1Kmu1vzlf/3Xf7k6ltPTTNrJJ5/s6qeeesrVRx99dMVjzI3mrjT7uWrVKld/8pOfdPUdd9zh6q1bt5b2oRk8zQFpZvXtt992td5YoFevXq7WxsxmZuecc46rH3744YrHoJkfzc3FsoidSa1zhGYI//RP/9TVa9asKf1Nv379XK35vXXr1rlam8Wnmsfr35uZ3Xfffa7W5u7atFub/uvY/cxnPuPqjRs3lvapjcN1XtKsY3tKnedx48aVfpa6kUfq2gPNKg8ePNjVsfeeZvZ0ztCxpOdBm7vrMejvzcrnWsfXpEmTKh7j66+/XnEfOs/FjquR6RyrN4iJ5YYrqeY6g1SW85prrnG13hhA/14/NzTXbFZ+P/zRH/2Rq/UzMnVTCx27sTGh7znNlcdumrI7jbEaBAAAQKfFghUAAABZY8EKAACArFXMsKYyQ22RN22UzGrKTTfd5OrLLrvM1cOGDXP1K6+8UnF7N954Y+lnQ4YMcXU982VtQfMymgXTPnWa8ZswYYKrNUdsVu6ZmMqwpXqo6uNjOcFzzz3X1ZpX1hysHpPmzzp7hlXpeb7iiitcrf0MtT9vbA7SPquaQ9Z+pdp7U/PkAwYMcHUsl3jWWWe5WjPr2mdRM6xK55hYLl7fM5qLe/zxx12tWfx6ivVE1feOvs7du3d3tZ57Pa/V9DzWfejfPPPMM67u3bu3qzVnWA39LNC5M9WHVecxzXDGjknnndgYzpE+t9jPdBzoe0Uzrfr31YyT9evXu/oXv/iFq7Xn6ZIlS1ytc4iOo9j7W9/PJ5xwgqv/53/+x9XPP/+8q/UcawZW1zVm5bGmmdZadIzVIQAAADosFqwAAADIGgtWAAAAZG2PbkLeUfKmbeHJJ590tfax0/zPrbfeWnF7sR6imqvTnp+HH3546jCzpj3gUjkrzRUdccQRrn7wwQeT+9Csl9L8jeZNq+lLp5kezYfp+yrV+05zuI1k27Ztrtb3Qez+8DrOtSeqvl6HHnqoq/U+65ol+8hHPlLap76XfvOb37j6tttuc7We0+OOO87VgwYNKu1DadbxyCOPdPWUKVNcvWLFCle/9tprrh46dKir9XUyK+fe9DHz5s1zdXtmWDVPqnPq9u3bS3+j75XHHnvM1fpe0364qfei5upjNFP9j//4j67W11T762q/6dWrV5f2sWjRIlf37dvX1am5U+nfx3Ly+np//OMfr7jN9pLKm1bTU1XPc6pnqm5zy5YtrtYMu1m59+0LL7zgan199doFnbe0D3Es0629XOfMmeNqnbd0jtG+zOPHj3f12LFjS/vUz0B9D9aCFScAAACyxoIVAAAAWWPBCgAAgKztUYYVf6C9NPVeztonT3umaY/Fq666ytWxPIr2jzzzzDNdXU0vuJxpnkzzNZo31ZzR/vvv72rtoWpm1q1bN1fXeh9vzbDpa67ZMbNyvknvT65jRzNZsbHQKLSPp75v9D7rmtM2S2cxFy9e7Grt16v5PH09Y/16Z8+e7epp06a5evr06a7WbPP8+fNdrbnFWK5Os4pHH320qzXfq/kx7ZFYzdjUx+hYjd2fvL3E7mff3KOPPlr6WZcuXVytr6k+X82samY9lvtVOifoGNbzdO2117pa++/ecccdrv7qV79a2qd+Xug1D/ra6bnXfrQ7duyoWJuVe25+7GMfc/U111zjap2P20pLrq3R918qs6r9THXO0Dknlq8ePny4qw855BBXa2Zb56k1a9a4Wj+rYu9VHRc6DjSjqp89Ok40N69569jf6HUL+jx0rdQc37ACAAAgayxYAQAAkDUWrAAAAMham4ccNQORypfU+viW0IxRLNtYq6VLl7pas1O6D+3T+utf/9rVBx98sKtjffC0B+LMmTOrO9gGoVlOPW+pfnpz5851teYKzcw2bNjg6lTuV3NFmkXW8Robv//7v//ras1L6XtAeyZqfqqR3Hfffa7WTJW+FrE+tpoH06zW6aef7uobb7zR1dpTVfsbLlu2rLRPPY/ah1WzX3ofcB3Lmh3Te3qbmX3mM59xtfZy1dfuBz/4gasXLFjgas1Kx3qIat5MX2udl+pJM33/8R//UXrMBRdc4GqdM/XcV9Pbtbl333239DOdUzSbqL0wNdOq+9TH6/bMyj1QNZuf+lzV3+v1Ato72qzc9/Oyyy5ztfYVba8Mqz4Xff+vXLmy9Df6ea19kvX10M97vZ5ixIgRro59VmmWU997Bx10kKv1s0nz/pqTj/V61s8SHQeptZDOW9oLNtb7Xbepc6Nec0CGFQAAAA2LBSsAAACyxoIVAAAAWWvzDKtmJDTrkeqD1xb2NLP6xBNPlH720EMPuVrv433aaae5Wu//rhkXzSlqzzuzjpdZVZoX1bGjeTPt57ZkyZKKjzcr91VMZSo1A6QZHh1bmhs0M3v55Zddred27dq1rtbekY3ch1XfB88//7yrFy5c6OpXX321tA3Ng2qvTM2PaXZOe0rqe03vP29Wziqm7sV+3nnnuVrvHx/LIaak5s4xY8a4+uc//7mrtYew5vbMyq+VZve012Y96WdFLH+u71+dl1N9VXVsKX19zMr5Ru0brGNcj0H3qedN55zYPvW10OPUfepcq5lVve+9WTm/r5nJ22+/3dXaJ7ytXHzxxRV/r/2Jzcq5Vn1v6OupvZx1nGnWUz9XzMpZZZ3XdJt6DYbuY8CAAa7Wzxmz8vPUvK/m9zVHq9fS6DiJXXOgc2U143l3+IYVAAAAWWPBCgAAgKyxYAUAAEDWWLACAAAga21+0VVph4nG7HpBROxCmfZ24YUXulovsDIzO/TQQ119xx13uFqbAisNpOuFN/fff3/qMDscHSsaMtcAuI4dDXPHxpJeyKIXv2ljcT0GvchKH6/hfDOzl156ydXaKFkvgNALJvT3jWTKlCmuXrVqlas/97nPuTp2swdt0q8XZukFItrIXS+y0ovYYg23p0+f7uo//uM/Lj1mT8TOqTYW17Gmf3P00UdXrDsavegn1pheL/jQm0woveBJL3zRc6DnyKx8I4Bnn33W1TpPHXPMMa7Wi2/0eV1//fWlfeocoXNj6iJBfS11bMXmMb3xhF5IHLsgrS3ohXR68a3Or3rjFrPyzRn09dDPhdSNQnRO0XnOrPz66WeLXuipj9dzqmMxdrMhvUhKP/90/OvroO8f/YzWzz+z8ntIL0LU174SvmEFAABA1liwAgAAIGssWAEAAJC1ioFSzTdoo+aWNPnX/N66detcrXm0GTNm1LyPWmk25G/+5m9crcd86623lrYxYcKEVj2Ggw8+2NWxPEpHpxkdzddoJlXHjmaZJk6cWNqH5sk0R6TZYz0m/XttxKzZJ7NyplLfR5oz0jyzZpsaiWaezj77bFf/6le/crXm08zKuVbNk2lGSrOcejOHWD4vRRuB61jTPJmeM30ddNyYlceFjn/NGeoND3Tc6FjVrKRZuZm55sAvuOACV59xxhmlbdRLLKepDeC1abl+xnXr1q3i4/WcxHLx+p4fNWqUq7Vpvx6DjhU9z+PHjy/tU7ehteYIdfzp61TNHKPjUTOr+tneVi666CJX6zjQMR1bt2jGdMGCBa7WOUJzsfp66s1e9PPdrHxedU7ROUH3oeNIm/b369evtE89zzp3rlixouIx6vOK3ShA6Tyjr/WcOXNcPWnSpN1ui29YAQAAkDUWrAAAAMgaC1YAAABkrWKGNdUztSUefPBBV7/wwguuHjZsmKt/+MMfunrmzJl7fAyaadEei5qRmzt3rqtbkt1N0dd68uTJyb/RXIz2CGx0mtHRWp9vKj8aG8+6Dc0R6bnWfWiOSzNvsVydHodmpDXzpnky7fnXyDQz+LGPfazmbWh+dOnSpa5etGiRq5966qmK24v1RNWxpJlAzXLpeddstP59bJxoPlLHQWqsaf9OzeGNGDGitM9aeiLWm85/sZxgSioXqJm/ajJ7On5S85j2BdaxotvTLHKMPi8dv/q8dJ7TeVHHq1n5eeg+2yvDOmbMGFdrn1V9n2hu06z8/PS9lpqD9boD3Z6eU7PyOkRzx5pB1c8ezajrONG1lZnZli1bXK3nWZ+nfr7pOa7muibt3arv08WLF5f+Znf4hhUAAABZY8EKAACArLFgBQAAQNZaNaRaTbbm85//vKunTp3qas0QXnbZZa7WXMw///M/J/f50EMPufrb3/62qzU7cvfddye32dq0z2o1fVdjuaKORPONep60R2oqbxPrdZnKwek+NCekeSjNAGke2sxsyZIlrta+oHpetU7dD72z0Z6IWqNj0vkglhNUqZxrKm+qn3F6DGblOSI27zSn2WP9e/19bHt6HLoNzRZqrlPnylRW0aw816XmsbZy+eWXu3rWrFmu/slPfuLql19+ubQNPVZ9fbTHrL4+y5Ytq7g9/RwxS6+XNJuv/btjfZSb056pZmannHKKqzdt2lTxmDTjmnqPxcZJKmsf+4zc7farfiQAAABQByxYAQAAkDUWrAAAAMhaxQyr9kD95je/6eoTTzzR1XoPbzOz3r17u/rggw92tfYz095ixxxzjKvfeOONisdoVs6szJ8/39UXXnihqzUD0xY0d6RZD72/rmZ5Y/R+5R0t26g5Ic3bDB8+3NWaM0r10jMrnxetU/fk1tdcc0XV5GYPOeQQV2uvR33e2qcV6Ix0foj119T3vGY7dRvaGzPVfzf2/k7NITr3a09Upc8h9njdpj5PnVP0GHWbmjuM9SbWz2J9/XU+bi9/8id/UrF+8sknS39z9dVXu1p7OWut9BzpuIp99uhjNB/65ptvulr7Kn/xi1909amnnurqiRMnVjjiJt/4xjdcrdcMaS/olvSg1xysfv4tWLCg6m3xDSsAAACyxoIVAAAAWWPBCgAAgKxVzLDOnDnT1TNmzHD1nXfe6epY3y/tHTZnzhxX/8Vf/IWrX3zxRVdrTlFzm7/+9a9L+zz33HNd/fjjj7u6JTmM5mK992L3qa9ln/q8qukpqNmQjkYzUZoPHT9+vKu1D6vmtmLnSLNamhvSv9HzpHkczbzF6FgYNGiQq7UXnubqpk2bltwH0NFpDlM/a8zKPR/1/a73Vtdtan68mh6rmlfUeUj7leq8pjlZnYNi1zdoL0t93jovpXK21cxj+jyfe+45V+v1J7mIHdf1119f8W+0H+/DDz/s6pdeesnVixYtcnUsw6pjUTOnEyZMcPVRRx3l6tSaoxpf/epXXa3rLx0n+tmkOWV9nczK2dsDDzzQ1Z/97GerO1jjG1YAAABkjgUrAAAAssaCFQAAAFmrKQSh2bs/+7M/a9WDaRStkR1RTzzxRM1/oz3SOhrNywwcONDV2gNV7+esv9csqFk6o6Z9hDWjo/kzzafp9mPb1Htbd+vWzdWjR492tfYq1pw30BFddNFFrtb5IZYn1YyqZjP1/auP37x5s6t1DonNKZprV3qth75/9doEnXMGDBhQ2qbmIfdUNfd7177r+pgjjzyyVY+ppfS5aG2WvsZEc8Unn3xyxbo9aC5Wx3bsWhsdJ/q8b7nlllY6urbBN6wAAADIGgtWAAAAZI0FKwAAALLGghUAAABZa/2rh4BWohdRvfrqq67WZs16gYSKBeu1ab9eEKEX2OkFFXrhm968QOsY3YfevEBfh6efftrVuVzcALSlvn37ulovhIk1hNf3s150snz5clevWbPG1StXrnS1NkrXm5uYlS/+0jlGn8f69esrHrPeRCZ2wxg9Lp3rdBs65+gFO7q9JUuWlPapF4PpPnKZl/S5VXNThEaQurlQW1wcXm98wwoAAICssWAFAABA1liwAgAAIGsh1kS3mYq/RMNpy/BOq4+VBx980NX33nuvq88//3xXr1q1ytUf+chHXD116tTSPrSxv2bS+vfv7+qtW7dWOOJyPir2+Pnz57t61qxZrtYm1I8++mjF37fRDSTaaqwwp3Qs7TanaLZb5wPNuJuZ3X///a7WfOlJJ53k6g0bNlQ8oJ07d7o6lmHdtm2bq/UGJnrjkGOPPdbVms3V3G2XLl0qHqNZuhG+5htjTeab0+cd24be2GXChAmu3meffZhTUI3djhO+YQUAAEDWWLACAAAgayxYAQAAkDUyrJ1LQ2VY99TatWtdrRlXM7N58+a5es6cOa7Wnoe6Tc2nDRkyxNWTJ08u7fOQQw5x9WGHHVZ6TAbIm6EaDTWnaKZ19OjRrtYMq/ZR1qynZmLNyplTfYz2OI3NER0UcwqqQYYVAAAAjYkFKwAAALLGghUAAABZS2VYAQAAgLriG1YAAABkjQUrAAAAssaCFQAAAFljwQoAAICssWAFAABA1liwAgAAIGv/P9xUcePo0vkiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "target_only_plot = pd.DataFrame(y_train, columns=['target'])\n",
    "\n",
    "## get first index only i need only one unique value to test to \n",
    "## what this number belong to what photo (i hope you understand me)\n",
    "## although the images are diffrent for each class_name < like (coats have diffrent images)\n",
    "## but still have the same number of target, that what i only want\n",
    "\n",
    "img_indicies = {}\n",
    "\n",
    "for i in range(len(np.unique(y_train))):\n",
    "    img_indicies[i] = target_only_plot[target_only_plot['target']==i].index[0]   \n",
    "\n",
    "    \n",
    "keys = list(img_indicies.keys())\n",
    "\n",
    "fig, ax = plt.subplots(2, 5, figsize=(12, 6))\n",
    "\n",
    "plt.sca(ax[0,0])\n",
    "plt.imshow(X_train[img_indicies[0]], cmap='binary')\n",
    "plt.title(f'y == {keys[0]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[0,1])\n",
    "plt.imshow(X_train[img_indicies[1]], cmap='binary')\n",
    "plt.title(f'y == {keys[1]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[0,2])\n",
    "plt.imshow(X_train[img_indicies[2]], cmap='binary')\n",
    "plt.title(f'y == {keys[2]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[0,3])\n",
    "plt.imshow(X_train[img_indicies[3]], cmap='binary')\n",
    "plt.title(f'y == {keys[3]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[0,4])\n",
    "plt.imshow(X_train[img_indicies[4]], cmap='binary')\n",
    "plt.title(f'y == {keys[4]}')\n",
    "plt.axis('off')\n",
    "\n",
    "###################\n",
    "plt.sca(ax[1,0])\n",
    "plt.imshow(X_train[img_indicies[5]], cmap='binary')\n",
    "plt.title(f'y == {keys[5]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[1,1])\n",
    "plt.imshow(X_train[img_indicies[6]], cmap='binary')\n",
    "plt.title(f'y == {keys[6]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[1,2])\n",
    "plt.imshow(X_train[img_indicies[7]], cmap='binary')\n",
    "plt.title(f'y == {keys[7]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[1,3])\n",
    "plt.imshow(X_train[img_indicies[8]], cmap='binary')\n",
    "plt.title(f'y == {keys[8]}')\n",
    "plt.axis('off')\n",
    "plt.sca(ax[1,4])\n",
    "plt.imshow(X_train[img_indicies[9]], cmap='binary')\n",
    "plt.title(f'y == {keys[9]}')\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3fd67308",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_train in row 1 => 4\n",
      "get its corresponding class => Coat\n"
     ]
    }
   ],
   "source": [
    "### you can now know for each target value belong to what class\n",
    "class_names = ['T_Shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', \n",
    "               'Sandal', 'Shirt', 'Sneaker','Bag', 'Ankle_Boot']\n",
    "\n",
    "\n",
    "## you can now work like this\n",
    "\n",
    "print('y_train in row 1 =>', y_train[0])\n",
    "print('get its corresponding class =>', class_names[y_train[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0b922cb",
   "metadata": {},
   "source": [
    "## `Kernel Initialization with diffrent Activation Functions`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2732b352",
   "metadata": {},
   "source": [
    "##### 1. `Golorot Initialization` ==> preferable with (None, tanh, sigmoid, softmax)\n",
    "\n",
    "* here ==> kernel_initializer='glorot_uniform' ==> default  <br />\n",
    "* you can also use ('glorot_normal')\n",
    "* glorot_normal use initialization mean=0, variance=(1/fan_avg), fan_avg = 0.5(fan_in+fan_out)<br />\n",
    "* glorot_unifrom use initialization r=sqrt(3*variance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "68e3a94e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAATx0lEQVR4nO3dfYwcd33H8feXOCkmB3HSJFtjohrKg0pjEfARaGmrOwLIJIiABKgIoliADlCJgLoqLqhqKEIKDwZahFSFh2IgcAoQmijhKURZaCQIvUuTXFKHBogLMalNSmLYyAI5fPvHjulxufPO7s7erX95v6TT7c78dubj8d7n5mbm5iIzkSSV5xFrHUCSNBoWvCQVyoKXpEJZ8JJUKAtekgq1bjVXduqpp+bmzZsBeOCBBzjxxBNXc/V9M2MzzNgMMzbjWMw4Pz9/b2ae1veCMnPVPrZu3ZpHXH/99TnuzNgMMzbDjM04FjMCczlA53qIRpIKZcFLUqEseEkqlAUvSYWy4CWpUBa8JBXKgpekQlnwklQoC16SCrWqtyqQxtXCvoNs33lNz3F7LzlvFdJIzXAPXpIKZcFLUqEseEkqlAUvSYWy4CWpUBa8JBXKgpekQlnwklQoC16SCmXBS1KhLHhJKpQFL0mF6lnwEfHIiPhuRNwSEbdHxDur6RdHxL6IuLn6OHf0cSVJddW5m+QvgedmZicijgduiIivVPM+mJnvH108SdKgehZ8ZibQqZ4eX33kKENJkoYX3f7uMSjiOGAeeCLwkcx8W0RcDGwHfg7MATsy875lXjsDzAC0Wq2ts7OzAHQ6HSYmJpr5VwxgYd/BnmNa62H/Idiy6aRVSDSYtd6OdRwLGQ/87CD7D/Uet5bvhWNhO5qxGUszTk9Pz2fmZL/LqVXwvxkcsQH4EnAR8FPgXrp78+8CNmbma472+snJyZybmwOg3W4zNTXVb97GbK7xxx12bDnMroV1Y/1HHtZ6O9ZxLGT88GVXsmuh9xHLtXwvHAvb0YzNWJoxIgYq+L6uosnM+4E2sC0z92fmg5n5a+CjwNn9rlySNDp1rqI5rdpzJyLWA88D7oiIjYuGvRS4bSQJJUkDqXMVzUZgd3Uc/hHA5Zl5dUR8OiLOonuIZi/w+pGllCT1rc5VNLcCT19m+gUjSSRJakSdPXg9zNQ5AX3EOJ+Alh7uvFWBJBXKgpekQlnwklQoC16SCmXBS1KhLHhJKpQFL0mFsuAlqVAWvCQVyoKXpEJZ8JJUKAtekgplwUtSoSx4SSqUBS9JhTpm7gffzz3KJUnuwUtSser80e1HRsR3I+KWiLg9It5ZTT8lIq6NiDurzyePPq4kqa46e/C/BJ6bmU8DzgK2RcSzgZ3AdZn5JOC66rkkaUz0LPjs6lRPj68+Ejgf2F1N3w28ZBQBJUmDiczsPSjiOGAeeCLwkcx8W0Tcn5kbFo25LzMfcpgmImaAGYBWq7V1dnYWgE6nw8TERO2gC/sO1h7blNZ62H8Itmw6adXXDfX+za31cPopzebrZ1vX2TadToe7Dj7Y2PJG4cDPDrL/UO9xa5UP+v+aWQtmbMbSjNPT0/OZOdnvcmpdRZOZDwJnRcQG4EsRcWbdFWTmpcClAJOTkzk1NQVAu93myOM6tq/BVTQ7thxm18I69r5qatXXDfX+zTu2HOYVfWzHptZ7RJ1t02632XXDA40tbxQ+fNmV7Fro/eWwVvmg/6+ZtWDGZjSVsa+raDLzfqANbAP2R8RGgOrzgaHTSJIaU+cqmtOqPXciYj3wPOAO4CrgwmrYhcCVI8ooSRpAnUM0G4Hd1XH4RwCXZ+bVEfFt4PKIeC3wI+DlI8wpSepTz4LPzFuBpy8z/X+Bc0YRSpI0vGPmVgUl8bYLOpb083795LYTR5hE/fJWBZJUKAtekgplwUtSoSx4SSqUJ1kLUPck2N5LzhtxktXzcPw3Pxz5/zwc9+AlqVAWvCQVyoKXpEJZ8JJUKAtekgrlVTQ1lHIm31skPDyU8n7V8NyDl6RCWfCSVCgLXpIKZcFLUqE8ydogT2JKGifuwUtSoer80e0zIuL6iNgTEbdHxJur6RdHxL6IuLn6OHf0cSVJddU5RHMY2JGZN0XEo4H5iLi2mvfBzHz/6OJJkgZV549u3wPcUz3+RUTsATaNOpgkaTh9HYOPiM3A04Ebq0lviohbI+ITEXFy0+EkSYOLzKw3MGIC+Cbw7sy8IiJawL1AAu8CNmbma5Z53QwwA9BqtbbOzs4C0Ol0mJiYqB10Yd/B2mOb0loP+w+t+mr7stYZt2w6qeeYTqfDXQcfbGx50Pz7oe52rJtvFOp+zdTdNqPY1mv1fuzn/6Xf7lkLSzNOT0/PZ+Zkv8upVfARcTxwNfC1zPzAMvM3A1dn5plHW87k5GTOzc0B0G63mZqaqh10LS5B3LHlMLsWxvtK0rXOWOd+Ju12m+1ffaCx5UHz74e623Et799S92um6XvR9LOt1+r92M//S7/dsxaWZoyIgQq+zlU0AXwc2LO43CNi46JhLwVu63flkqTRqfOt9jnABcBCRNxcTXs78MqIOIvuIZq9wOtHkE+SNKA6V9HcAMQys77cfBxJUlPG+wCzNGb6OR7d9DHuT247sfa6JfBWBZJULAtekgplwUtSoSx4SSqUJ1mlY8TCvoNs928OqA/uwUtSoSx4SSqUBS9JhbLgJalQFrwkFcqraKQRWYtbXEuLuQcvSYWy4CWpUBa8JBXKgpekQnmSVXqY8iTwypr+u7ZrxT14SSqUBS9JhepZ8BFxRkRcHxF7IuL2iHhzNf2UiLg2Iu6sPp88+riSpLrq7MEfBnZk5h8Czwb+MiKeCuwErsvMJwHXVc8lSWOiZ8Fn5j2ZeVP1+BfAHmATcD6wuxq2G3jJiDJKkgYQmVl/cMRm4FvAmcCPMnPDonn3ZeZDDtNExAwwA9BqtbbOzs4C0Ol0mJiYqL3uhX0Ha49tSms97D+06qvty1pn3LLppJ5jOp0Odx18cBXSDG6tt2MdZlxZnfchdHtk3DPCQ/txenp6PjMn+11n7YKPiAngm8C7M/OKiLi/TsEvNjk5mXNzcwC0222mpqZqB12LS7p2bDnMroXxvpJ0rTPWuUys3W6z/asPrEKawa31dqzDjCure7ni5p3XjH1GeGg/RsRABV/rKpqIOB74InBZZl5RTd4fERur+RuBA/2uXJI0OnWuogng48CezPzAollXARdWjy8Ermw+niRpUHV+TnkOcAGwEBE3V9PeDlwCXB4RrwV+BLx8JAklSQPpWfCZeQMQK8w+p9k4ktQ/b7uwPH+TVZIKZcFLUqEseEkqlAUvSYWy4CWpUBa8JBXKgpekQlnwklQoC16SCmXBS1Khxvveoxp7dX5FfMeWw/hWk1afe/CSVCgLXpIKZcFLUqEseEkqlAUvSYWy4CWpUBa8JBWqzh/d/kREHIiI2xZNuzgi9kXEzdXHuaONKUnqV509+E8C25aZ/sHMPKv6+HKzsSRJw+pZ8Jn5LeBnq5BFktSgyMzegyI2A1dn5pnV84uB7cDPgTlgR2bet8JrZ4AZgFartXV2dhaATqfDxMRE7aAL+w7WHtuU1nrYf2jVV9sXMzbDjM0w48q2bDqp9til/Tg9PT2fmZP9rnPQgm8B9wIJvAvYmJmv6bWcycnJnJubA6DdbjM1NVU7aJ17njRtx5bD7FoY73uomLEZZmyGGVe295Lzao9d2o8RMVDBD3QVTWbuz8wHM/PXwEeBswdZjiRpdAYq+IjYuOjpS4HbVhorSVobPX9OiYjPAVPAqRFxN/D3wFREnEX3EM1e4PWjiyhJGkTPgs/MVy4z+eMjyCJJapC/ySpJhbLgJalQFrwkFcqCl6RCWfCSVCgLXpIKZcFLUqEseEkqlAUvSYWy4CWpUBa8JBXKgpekQlnwklQoC16SCmXBS1KhLHhJKpQFL0mFsuAlqVA9Cz4iPhERByLitkXTTomIayPizurzyaONKUnqV509+E8C25ZM2wlcl5lPAq6rnkuSxkjPgs/MbwE/WzL5fGB39Xg38JJmY0mShhWZ2XtQxGbg6sw8s3p+f2ZuWDT/vsxc9jBNRMwAMwCtVmvr7OwsAJ1Oh4mJidpBF/YdrD22Ka31sP/Qqq+2L2ZshhmbYcaVbdl0Uu2xS/txenp6PjMn+13nun5f0K/MvBS4FGBycjKnpqYAaLfbHHlcx/ad14wg3dHt2HKYXQsj30RDMWMzzNgMM65s76umao/ttx9XMuhVNPsjYiNA9fnA0EkkSY0atOCvAi6sHl8IXNlMHElSU+pcJvk54NvAUyLi7oh4LXAJ8PyIuBN4fvVckjRGeh6IysxXrjDrnIazSJIa5G+ySlKhLHhJKpQFL0mFsuAlqVAWvCQVyoKXpEJZ8JJUKAtekgplwUtSoSx4SSqUBS9JhbLgJalQFrwkFcqCl6RCWfCSVCgLXpIKZcFLUqEseEkqVM8/2Xc0EbEX+AXwIHA4MyebCCVJGt5QBV+Zzsx7G1iOJKlBHqKRpEINW/AJfD0i5iNipolAkqRmRGYO/uKIx2bmTyLidOBa4KLM/NaSMTPADECr1do6OzsLQKfTYWJiova6FvYdHDjnoFrrYf+hVV9tX8zYDDM2w4wr27LppNpjl/bj9PT0/CDnOIcq+N9aUMTFQCcz37/SmMnJyZybmwOg3W4zNTVVe/mbd14zZML+7dhymF0LTZymGB0zNsOMzTDjyvZecl7tsUv7MSIGKviBD9FExIkR8egjj4EXALcNujxJUrOG+TbWAr4UEUeW89nM/GojqSRJQxu44DPzh8DTGswiSWqQl0lKUqEseEkqlAUvSYWy4CWpUBa8JBXKgpekQlnwklQoC16SCmXBS1KhLHhJKpQFL0mFsuAlqVAWvCQVyoKXpEJZ8JJUKAtekgplwUtSoSx4SSqUBS9JhRqq4CNiW0R8LyK+HxE7mwolSRrewAUfEccBHwFeCDwVeGVEPLWpYJKk4QyzB3828P3M/GFm/gqYBc5vJpYkaViRmYO9MOJlwLbMfF31/ALgWZn5piXjZoCZ6ulTgO9Vj08F7h1o5avHjM0wYzPM2IxjMePvZ+Zp/S5k3RABYplpD/lukZmXApc+5MURc5k5OcT6R86MzTBjM8zYjIdTxmEO0dwNnLHo+eOAnwwXR5LUlGEK/t+BJ0XE4yPiBOAvgKuaiSVJGtbAh2gy83BEvAn4GnAc8InMvL2PRTzksM0YMmMzzNgMMzbjYZNx4JOskqTx5m+ySlKhLHhJKtRICz4iTomIayPizurzySuMW/aWBxFxVkR8JyJujoi5iDh73DJW8y6q5t0eEe8dx4zV/L+OiIyIU8ctY0S8LyLuiIhbI+JLEbGhoVy9tklExD9V82+NiGfUfW1TBs0YEWdExPURsad677153DIumn9cRPxHRFw9jhkjYkNEfKF6D+6JiD8ew4xvrf6fb4uIz0XEI3uuMDNH9gG8F9hZPd4JvGeZMccBPwCeAJwA3AI8tZr3deCF1eNzgfYYZpwGvgH8TvX89HHLWM0/g+4J8f8GTh23jMALgHXV4/cs9/oBMh11myx6X32F7u91PBu4se5rG9puw2TcCDyjevxo4L/GLeOi+X8FfBa4uul8TWQEdgOvqx6fAGwYp4zAJuAuYH31/HJge691jvoQzfnVhjuyAV+yzJij3fIggcdUj09iNNfZD5vxjcAlmflLgMw8MIYZAT4I/A3L/DLaOGTMzK9n5uFq3Hfo/l7FsOrcTuN84FPZ9R1gQ0RsrPnaJgycMTPvycybADLzF8AeukUwNhkBIuJxwHnAx0aQbeiMEfEY4M+BjwNk5q8y8/5xyljNWwesj4h1wKOo0YejLvhWZt4DUH0+fZkxm4AfL3p+N///Jn0L8L6I+DHwfuBvxzDjk4E/i4gbI+KbEfHMccsYES8G9mXmLSPI1kjGJV5Ddy9mWHXWt9KYulnXMuNvRMRm4OnAjc1HHDrjh+juXPx6BNnqrL/XmCcAPwX+pTqM9LGIOHGcMmbmProd+CPgHuBgZn691wqHuVUBABHxDeD3lpn1jrqLWGbakb3MNwJvzcwvRsQr6H6Hfd6YZVwHnEz3x6lnApdHxBOy+jlqrTNGxKOqZbygnzzLrmC02/HIOt4BHAYu6y/dYOs7yphat+JowDAZuzMjJoAvAm/JzJ83mK3W+o82JiJeBBzIzPmImGo6WK/11xyzDngGcFFm3hgR/0j3MOPfNRtxqO14Mt29+8cD9wOfj4hXZ+ZnjrbCoQs+M1cs3IjYf+RHyerHjOUOXxztlgcXAkdOHH2eAX/EG3HGu4ErqkL/bkT8mu6Ngn46Jhn/gO6b4paIODL9pog4OzP/Z0wyHlnGhcCLgHP6/Qa5gjq301hpzAk1XtuEYTISEcfTLffLMvOKEeQbNuPLgBdHxLnAI4HHRMRnMvPVY5Qxgbsz88hPP1+gW/BNGybj84C7MvOnABFxBfAnwFELvvGTHUtOGLyP3z7x9t5lxqwDfki3hI6cePijat4eYKp6fA4wP4YZ3wD8Q/X4yXR/vIpxyrhk3F5Gc5J12O24DfhP4LQGM/XcJnSPDS8+qfXdfrbnGmcM4FPAh5rO1VTGJWOmGN1J1qEyAv8GPKV6fDHwvnHKCDwLuJ3usfege57rop7rHPEb43eB64A7q8+nVNMfC3x50bhz6V4B8APgHYum/ykwX22IG4GtY5jxBLrfRW8DbgKeO24ZlyxrL6Mp+GG34/fpfnO8ufr454ZyPWR9dL8pv6F6HHT/cM0PgAVgsp/tuZYZq6+PBG5dtN3OHaeMS5YxxYgKvoH/67OAuWpb/itw8hhmfCdwB92u+TTVlXtH+/BWBZJUKH+TVZIKZcFLUqEseEkqlAUvSYWy4CWpUBa8JBXKgpekQv0fBymkoiWMs74AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.Dense(300, activation='tanh', kernel_initializer='glorot_uniform'))\n",
    "model_fashion.add(tf.keras.layers.Dense(100, activation='tanh', kernel_initializer='glorot_uniform'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "## check for example layer 1\n",
    "weights_1, biases_1 = model_fashion.layers[1].get_weights()\n",
    "\n",
    "## check for feature of these features\n",
    "pd.DataFrame(weights_1).iloc[:,0].hist(bins=30);  ## distribution of feature 1 ==> uniform distr."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad0939db",
   "metadata": {},
   "source": [
    "##### 2. `He Initialization` ==> preferable with (relu, elu, leakyrelu, parametricrelu, randomizedrelu)\n",
    "\n",
    "* here ==> kernel_initializer='he_normal'   <br />\n",
    "* you can also use ('he_uniform')\n",
    "* he_normal use initialization mean=0, variance=(2/fan_in)   <br />\n",
    "* he_unifrom use initialization r=sqrt(3*variance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4b359563",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPOElEQVR4nO3df4hl91nH8ffTtKUhk2azJpmOSXEUQzV0aW3GGgjIjGlKTIobwUhLrVuMLAUj/WMDrvafSimuQvxDKOKqwfFXx6ANWVJ/ratjKETNrjbZhKRuW9eYZNml6W7shFBdefxjznZm58feM/f3M/t+weXec+45c5/77J3PnvmeHzcyE0lSPW8adQGSpO4Y4JJUlAEuSUUZ4JJUlAEuSUW9eZgvdt111+X09PQwX3Kd119/nauuumqkNYwLe3Ex+7HCXqwYh14cO3bsG5l5/dr5Qw3w6elpjh49OsyXXGdxcZHZ2dmR1jAu7MXF7McKe7FiHHoREf+50XyHUCSpqFZb4BFxEvgW8H/A+cyciYidwJ8B08BJ4Kcz8+xgypQkrbWVLfC5zHxvZs400/uBI5l5M3CkmZYkDUkvQyi7gfnm8Txwb8/VSJJaizbXQomI/wDOAgn8TmYejIhzmblj1TJnM/PaDdbdC+wFmJycvHVhYaFftXdlaWmJiYmJkdYwLuzFxezHCnuxYhx6MTc3d2zV6Md3tD0K5fbMfCUibgAOR8QLbV84Mw8CBwFmZmZy1Htzx2GP8riwFxezHyvsxYpx7kWrIZTMfKW5PwM8CrwfOB0RUwDN/ZlBFSlJWq9jgEfEVRFx9YXHwAeBZ4FDwJ5msT3AY4MqUpK0XpshlEng0Yi4sPyfZuZfR8RTwCMRcT/wInDf4MqUJK3VMcAz8+vAezaY/ypwxyCKki4n0/u/2Gq5kwfuGXAlqsYzMSWpKANckooywCWpKANckooywCWpKANckooywCWpqKF+I49UXdtjtsHjtjV4boFLUlEGuCQVZYBLUlGOgUsDspXxcqkbboFLUlEGuCQVZYBLUlEGuCQV5U5MbWt+WYK2M7fAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJamo1gEeEVdExL9FxOPN9M6IOBwRJ5r7awdXpiRpra1sgX8SeH7V9H7gSGbeDBxppiVJQ9IqwCPiJuAe4PdWzd4NzDeP54F7+1qZJOmSIjM7LxTx58CvAVcDD2bmhyLiXGbuWLXM2cxcN4wSEXuBvQCTk5O3Liws9Kv2riwtLTExMTHSGsbF5dCL4y+/1nrZySvh9BsDLKZHu268ZmivdTl8Ntoah17Mzc0dy8yZtfM7fit9RHwIOJOZxyJidqsvnJkHgYMAMzMzOTu75R/RV4uLi4y6hnFxOfTi4y2/lR5g367zPHS846/EyJz86OzQXuty+Gy0Nc69aPNpvR34iYi4G3gb8PaI+GPgdERMZeapiJgCzgyyUEnSxTqOgWfmL2fmTZk5DXwY+PvM/BngELCnWWwP8NjAqpQkrdPLceAHgDsj4gRwZzMtSRqSLQ34ZeYisNg8fhW4o/8lSZLa8ExMSSrKAJekogxwSSrKAJekogxwSSrKAJekosb3vGH13fSa08r37Tq/4anmJw/cM6yS1llb42ZGWaM0LtwCl6SiDHBJKsoAl6SiDHBJKsqdmFIRo9rB2/Z1B/HaujS3wCWpKANckooywCWpKANckooywCWpKANckooywCWpKI8DV0+8+JQ0Om6BS1JRBrgkFWWAS1JRjoFvA1u5VsV2cTm+57b60ZvNvuxD48UtcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqqmOAR8TbIuJfIuLpiHguIn61mb8zIg5HxInm/trBlytJuqDNFvi3gR/LzPcA7wXuiojbgP3Akcy8GTjSTEuShqRjgOeypWbyLc0tgd3AfDN/Hrh3EAVKkjYWmdl5oYgrgGPA9wOfy8xfiohzmblj1TJnM3PdMEpE7AX2AkxOTt66sLDQr9q7srS0xMTExEhr6LfjL7/W1XqTV8LpN9bP33XjNQN/7XG0WT8uR932YiufnSrGITPm5uaOZebM2vmtAvw7C0fsAB4FfhH4UpsAX21mZiaPHj3a+vUGYXFxkdnZ2ZHW0G/dXn1u367zPHR8/QUpt/LtOdvpqoCb9eNy1G0vtuM3L41DZkTEhgG+paNQMvMcsAjcBZyOiKnmh08BZ3ovU5LUVpujUK5vtryJiCuBDwAvAIeAPc1ie4DHBlSjJGkDbf5GmgLmm3HwNwGPZObjEfEk8EhE3A+8CNw3wDolSWt0DPDMfAb4oQ3mvwrcMYiiJEmdeSamJBVlgEtSUQa4JBVlgEtSUQa4JBVlgEtSUQa4JBXlhR8k9U3ba+Nsx2umjIJb4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlNdC0Tptr2chabTcApekogxwSSrKAJekogxwSSrKnZhjzJ2Jki7FLXBJKsoAl6SiDHBJKsox8BFwbFtSP7gFLklFGeCSVJQBLklFGeCSVFTHAI+Id0bEP0TE8xHxXER8spm/MyIOR8SJ5v7awZcrSbqgzRb4eWBfZv4gcBvwCxFxC7AfOJKZNwNHmmlJ0pB0DPDMPJWZ/9o8/hbwPHAjsBuYbxabB+4dUI2SpA1EZrZfOGIaeAJ4N/BiZu5Y9dzZzFw3jBIRe4G9AJOTk7cuLCz0WHJvlpaWmJiYGGkNx19+baSvf8HklXD6jVFXMT7sx4px6cWuG68ZdQljkRlzc3PHMnNm7fzWAR4RE8A/Ap/NzC9ExLk2Ab7azMxMHj16dGuV99ni4iKzs7MjrWFcTuTZt+s8Dx33XK4L7MeKcenFyQP3jLqEsciMiNgwwFsdhRIRbwH+AviTzPxCM/t0REw1z08BZ/pVrCSpszZHoQTw+8Dzmfmbq546BOxpHu8BHut/eZKkzbT5G+l24GPA8Yj4cjPvV4ADwCMRcT/wInDfQCqUJG2oY4Bn5peA2OTpO/pbjiSpLc/ElKSiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKsoAl6SiDHBJKqrNt9JL0lib3v/FVsudPHDPgCsZLrfAJakoA1ySijLAJakox8D7qO04nKTR2G5j5W6BS1JRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFWWAS1JRHgcuSWusPl58367zfHyT48dHfbx4xy3wiHg4Is5ExLOr5u2MiMMRcaK5v3awZUqS1mozhPIHwF1r5u0HjmTmzcCRZlqSNEQdAzwznwC+uWb2bmC+eTwP3NvfsiRJnXS7E3MyM08BNPc39K8kSVIbkZmdF4qYBh7PzHc30+cyc8eq589m5obj4BGxF9gLMDk5eevCwkIfyu7e0tISExMTA/nZx19+bSA/d1Amr4TTb4y6ivFhP1aMSy923XhNq+UG+bt3qV60ra9Xc3NzxzJzZu38bo9COR0RU5l5KiKmgDObLZiZB4GDADMzMzk7O9vlS/bH4uIig6phsz3V42rfrvM8dNwDkS6wHyvGpRcnPzrbarlB/u5dqhdt6xuUbodQDgF7msd7gMf6U44kqa02hxF+HngSeFdEvBQR9wMHgDsj4gRwZzMtSRqijn8jZeZHNnnqjj7Xckn9vhC7X74gqTpPpZekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekokZ/rmyfdTq++1IXZ5c0Xsb9fI2t1DeIL39wC1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJakoA1ySijLAJamongI8Iu6KiK9ExFcjYn+/ipIkddZ1gEfEFcDngB8HbgE+EhG39KswSdKl9bIF/n7gq5n59cz8H2AB2N2fsiRJnURmdrdixE8Bd2XmzzfTHwN+JDMfWLPcXmBvM/ku4Cvdl9sX1wHfGHEN48JeXMx+rLAXK8ahF9+TmdevnfnmHn5gbDBv3f8GmXkQONjD6/RVRBzNzJlR1zEO7MXF7McKe7FinHvRyxDKS8A7V03fBLzSWzmSpLZ6CfCngJsj4nsj4q3Ah4FD/SlLktRJ10MomXk+Ih4A/ga4Ang4M5/rW2WDMzbDOWPAXlzMfqywFyvGthdd78SUJI2WZ2JKUlEGuCQVtS0DPCJ2RsThiDjR3F+7yXIPR8SZiHi2m/Ur2EIvNrwsQkR8OiJejogvN7e7h1d9f3S65EMs+63m+Wci4n1t162mx16cjIjjzefg6HAr778WvfiBiHgyIr4dEQ9uZd2hycxtdwN+A9jfPN4P/Pomy/0o8D7g2W7Wr3Br815Y3gn9NeD7gLcCTwO3NM99Gnhw1O+jh/e/6XtbtczdwF+xfG7DbcA/t1230q2XXjTPnQSuG/X7GGIvbgB+GPjs6t+BcfpcbMstcJZP6Z9vHs8D9260UGY+AXyz2/WLaPNetvNlEdq8t93AH+ayfwJ2RMRUy3Ur6aUX203HXmTmmcx8Cvjfra47LNs1wCcz8xRAc3/DkNcfJ23ey43Af62afqmZd8EDzZ/TDxccTur03i61TJt1K+mlF7B8pvXfRsSx5hIZlfXybzs2n4teTqUfqYj4O+AdGzz1qWHXMmp96MWlLovw28BnmunPAA8BP7fVGkeozSUfNlum1eUiCumlFwC3Z+YrEXEDcDgiXmj+iq2ol3/bsflclA3wzPzAZs9FxOmImMrMU82ff2e2+ON7XX+o+tCLTS+LkJmnV/2s3wUe70/VQ9Pmkg+bLfPWFutW0ksvyMwL92ci4lGWhxKqBngvlwIZm8uIbNchlEPAnubxHuCxIa8/Ttq8l00vi7Bm/PMngWc3WH+ctbnkwyHgZ5sjMG4DXmuGm7bb5SK67kVEXBURVwNExFXAB6n3WVitl3/b8flcjHpv8CBuwHcBR4ATzf3OZv53A3+5arnPA6dY3knxEnD/pdaveNtCL+4G/p3lveufWjX/j4DjwDMsf0inRv2euujBuvcGfAL4RPM4WP5ykq8173WmU1+q3rrtBctHXDzd3J67THrxjiYX/hs41zx++zh9LjyVXpKK2q5DKJK07RngklSUAS5JRRngklSUAS5JRRngklSUAS5JRf0/EcN9LLE7towAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.Dense(300, activation='relu', kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.Dense(100, activation='relu', kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "## check for example layer 1\n",
    "weights_1, biases_1 = model_fashion.layers[1].get_weights()\n",
    "\n",
    "## check for feature of these features\n",
    "pd.DataFrame(weights_1).iloc[:,0].hist(bins=30);  ## distribution of feature 1 ==> normal distr."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa81f462",
   "metadata": {},
   "source": [
    "##### 3. `LeCun Initialization` ==> preferable with (Selu)\n",
    "\n",
    "* here ==> kernel_initializer='lecun_normal'   <br />\n",
    "* you can also use ('lecun_uniform')\n",
    "* lecun_normal use initialization mean=0, variance=(1/fan_in)   <br />\n",
    "* lecun_unifrom use initialization r=sqrt(3*variance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bd501932",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAARi0lEQVR4nO3dfYwcd33H8fe3NmlMjvihSa6uE9VQBdSUEyE+Ai1tdYcJCkmE8wcgUECOGmSBSgTUqDJFlUqlqoY0La1UqbWA9igP1wBJEyV9wLg5KBIE7JDESR1qEtwQx7WB2imHItCVb//YuXA9r72zu7N3+7PfL+m0s/Ow87n1+nOzszOzkZlIksrzM8sdQJLUGwtckgplgUtSoSxwSSqUBS5JhVq5lCu74IILcuPGjT0v/8Mf/pDzzjuvuUANMVd3zNUdc3VvWLP1mmvfvn3fy8wLT5qQmUv2s2nTpuzHvffe29fyg2Ku7pirO+bq3rBm6zUXsDfbdKq7UCSpUBa4JBXKApekQlngklQoC1ySCmWBS1Khah0HHhGHgB8A/wvMZeZ4RKwD/h7YCBwC3piZxwcTU5K0WDdb4JOZeXlmjlf3dwB7MvNSYE91X5K0RPrZhbIFmKqGp4Dr+04jSaotssYXOkTEt4HjQAJ/nZm7IuJEZq5ZMM/xzFzbZtltwDaA0dHRTdPT0z2HnZ2dZWRkpOflB8Vc3Tlbcu0//HSt+cY2rD7t9LPl+WrSsGbrNdfk5OS+BXs/nlX3WiivzMynIuIiYHdEPFp3xZm5C9gFMD4+nhMTE3UXPcnMzAz9LD8o5urO2ZLrxh331Jrv0A2nX+fZ8nw1aVizNZ2r1i6UzHyquj0G3AFcCRyNiPUA1e2xxlJJkjrqWOARcV5EPG9+GHgN8DBwF7C1mm0rcOegQkqSTlZnF8oocEdEzM//qcz854j4OnBbRNwEPAG8YXAxJUmLdSzwzHwceEmb8d8HNg8ilCSpM8/ElKRCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWqeyq9VKSNdU9n33ntgJNIzXMLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQnkcuPrS9HHWHrct1ecWuCQVygKXpEJZ4JJUKAtckgrlh5hSF+p+yCotBbfAJalQFrgkFcoCl6RCuQ9cS2LhvuPtY3Pc6L5kqW9ugUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKVbvAI2JFRHwjIu6u7q+LiN0RcbC6XTu4mJKkxbrZAn8XcGDB/R3Ansy8FNhT3ZckLZFaBR4RFwPXAh9ZMHoLMFUNTwHXN5pMknRakZmdZ4r4LPDHwPOA92bmdRFxIjPXLJjneGaetBslIrYB2wBGR0c3TU9P9xx2dnaWkZGRnpcflLM51/7DT3e9zOgqOPpMf+sd27C61nx1841tWF3r+erl962z7tM5m19fvRrWbL3mmpyc3JeZ44vHd7yYVURcBxzLzH0RMdHtijNzF7ALYHx8PCcmun6IZ83MzNDP8oNyNufq5aJU28fmuHV/f9dRO3TDRK356uY7dMNEredrEBfh6vS7nM2vr14Na7amc9X5X/RK4HURcQ1wLnB+RHwCOBoR6zPzSESsB441lkqS1FHHfeCZ+b7MvDgzNwJvAv41M98C3AVsrWbbCtw5sJSSpJP0cxz4TuCqiDgIXFXdlyQtka52RGbmDDBTDX8f2Nx8JElSHZ6JKUmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSofq7IIW0TDYO4Joky6XT77J9bI4bd9zDoZ3XLlEilcItcEkqlAUuSYWywCWpUO4Dl2jth57f1yyVwi1wSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVB+J6ZOstHvhRxKy/XvcmjntcuyXnXWcQs8Is6NiK9FxIMR8UhEfKAavy4idkfEwep27eDjSpLm1dmF8iPgVZn5EuBy4OqIeAWwA9iTmZcCe6r7kqQl0rHAs2W2uvuc6ieBLcBUNX4KuH4QASVJ7dX6EDMiVkTEA8AxYHdm3geMZuYRgOr2ooGllCSdJDKz/swRa4A7gJuBL2fmmgXTjmfmSfvBI2IbsA1gdHR00/T0dM9hZ2dnGRkZ6Xn5QTnTcu0//PQA0vzU6Co4+sxAV9ETc7U3tmF12/HD+rqH4c3Wa67Jycl9mTm+eHxXR6Fk5omImAGuBo5GxPrMPBIR62ltnbdbZhewC2B8fDwnJia6zf6smZkZ+ll+UM60XDcO+GiH7WNz3Lp/+A6AMld7h26YaDt+WF/3MLzZms5V5yiUC6stbyJiFfBq4FHgLmBrNdtW4M7GUkmSOqrzZ309MBURK2gV/m2ZeXdEfAW4LSJuAp4A3jDAnJKkRToWeGY+BLy0zfjvA5sHEUqD4Qk60pnFU+klqVAWuCQVygKXpEIN3zFTelbdfdZebEg6O7kFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoTwOXNJpnep8hO1jcyddethzEpaWW+CSVCgLXJIKZYFLUqHcB34G6GYfpaQzh1vgklQoC1ySCmWBS1Kh3Acuacl5rftmuAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUqI4FHhGXRMS9EXEgIh6JiHdV49dFxO6IOFjdrh18XEnSvDpb4HPA9sz8ZeAVwG9HxGXADmBPZl4K7KnuS5KWSMcCz8wjmXl/NfwD4ACwAdgCTFWzTQHXDyijJKmNyMz6M0dsBL4EvBh4IjPXLJh2PDNP2o0SEduAbQCjo6Obpqenew47OzvLyMhIz8sPyqBy7T/8dF/Lj66Co880FKZB5upOSbnGNqyutWzd13bdx1vsTOuKycnJfZk5vnh87QKPiBHgi8AfZebtEXGiToEvND4+nnv37u0u+QIzMzNMTEz0vPygDCpX3W8tOZXtY3Pcun/4vnTJXN0pKVfdb9AZ9DfynGldERFtC7zWUSgR8Rzgc8AnM/P2avTRiFhfTV8PHOs6lSSpZ3WOQgngo8CBzPzTBZPuArZWw1uBO5uPJ0k6lTrvy14JvBXYHxEPVON+D9gJ3BYRNwFPAG8YSEJJUlsdCzwzvwzEKSZvbjaOJKkuz8SUpEJZ4JJUqOE7NmkIdTrkafvYHDfuuKfnQ54kqRdugUtSoSxwSSqUBS5JhXIfeIMGfXqwJC3kFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUJ7II6kx/X6Pq7rjFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYXyOHBJQ6vX48rnv2h8sTPty1TcApekQlngklQoC1ySCuU+8GXg9SIkNcEtcEkqlAUuSYWywCWpUGfcPvC6+5fPtONBJXV2pvVDxy3wiPhYRByLiIcXjFsXEbsj4mB1u3awMSVJi9XZhfK3wNWLxu0A9mTmpcCe6r4kaQl1LPDM/BLw34tGbwGmquEp4PpmY0mSOonM7DxTxEbg7sx8cXX/RGauWTD9eGa23Y0SEduAbQCjo6Obpqenew47OzvLyMjIaefZf/jpWo81tmF17fV2eszRVXD0mdoPt2TM1R1zdWdYc0H/2brph27U6bB2Jicn92Xm+OLxA/8QMzN3AbsAxsfHc2JioufHmpmZodPy7S5g086hG+rn6PSY28fmuHX/8H0ebK7umKs7w5oL+s/WTT90o06HdaPXwwiPRsR6gOr2WGOJJEm19FrgdwFbq+GtwJ3NxJEk1VXnMMJPA18BXhQRT0bETcBO4KqIOAhcVd2XJC2hjjuJMvPNp5i0ueEsS8oLSkk6lUH0wyBODvJUekkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCjWcFzJoY+OOe9g+Nlf7WieSdKZzC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCtVXgUfE1RHxzYj4VkTsaCqUJKmzngs8IlYAfwm8FrgMeHNEXNZUMEnS6fWzBX4l8K3MfDwzfwxMA1uaiSVJ6iQys7cFI14PXJ2Zb6vuvxV4eWa+c9F824Bt1d0XAd/sPS4XAN/rY/lBMVd3zNUdc3VvWLP1musXM/PCxSNX9hEk2ow76a9BZu4CdvWxnp+uMGJvZo438VhNMld3zNUdc3VvWLM1naufXShPApcsuH8x8FR/cSRJdfVT4F8HLo2I50fEOcCbgLuaiSVJ6qTnXSiZORcR7wT+BVgBfCwzH2ksWXuN7IoZAHN1x1zdMVf3hjVbo7l6/hBTkrS8PBNTkgplgUtSoYaqwCNiXUTsjoiD1e3aU8zX9hT+iLg8Ir4aEQ9ExN6IuHIYclXTbq6mPRIRH2oiV1PZqunvjYiMiAuGIVdE3BIRj0bEQxFxR0Ss6TNPp98/IuIvqukPRcQVdZddjlwRcUlE3BsRB6rX1LuGIdeC6Ssi4hsRcfew5IqINRHx2ep1dSAifnVIcr2n+jd8OCI+HRHn1l5xZg7ND/AhYEc1vAP4YJt5VgCPAS8AzgEeBC6rpn0eeG01fA0wMyS5JoEvAD9b3b9oWJ6zavoltD6M/k/ggmHIBbwGWFkNf7Dd8l1kOe3vv+D18k+0zm94BXBf3WWXKdd64Ipq+HnAfwxDrgXTfwf4FHB3g6/1vnIBU8DbquFzgDXLnQvYAHwbWFXdvw24se66h2oLnNap+FPV8BRwfZt5TncKfwLnV8Orae649H5zvQPYmZk/AsjMYw3laiIbwJ8Bv0ubE7GWK1dmfj4z56r5vkrrPINe1bnswxbg49nyVWBNRKyvueyS58rMI5l5P0Bm/gA4QKsMljUXQERcDFwLfKShPH3niojzgd8EPgqQmT/OzBPLnauathJYFRErgefSRW8NW4GPZuYRgOr2ojbzbAC+s+D+k/z0hftu4JaI+A7wJ8D7hiTXC4HfiIj7IuKLEfGyhnL1nS0iXgcczswHG8zUd65FfovW1kuv6qznVPPUzbjUuZ4VERuBlwL3DUmuD9PaIPhJQ3mayPUC4LvA31S7dj4SEectd67MPEyrq54AjgBPZ+bn6664n1PpexIRXwB+vs2k99d9iDbj5rcc3wG8JzM/FxFvpPXX9tVDkGslsJbWW6eXAbdFxAuyes+0XNki4rnVY7ym5uMsSa5F63g/MAd8srt03a3nNPPUumREj/rJ1ZoYMQJ8Dnh3Zv7PcueKiOuAY5m5LyImGsrTdy5a/wevAG7OzPsi4s9p7dr7/eXMVX02tAV4PnAC+ExEvCUzP1FnxUte4Jl5ykKNiKPzbw+rtxftdjWc7hT+rcD8hzmfoYu3cAPO9SRwe1XYX4uIn9C6qM13lznbL9F64TwYEfPj74+IKzPzv5Yx1/xjbAWuAzbX/WN3CnUu+3Cqec6psexy5CIinkOrvD+Zmbc3lKnfXK8HXhcR1wDnAudHxCcy8y3LnCuBJzNz/l3KZ2kVeBP6yfVq4NuZ+V2AiLgd+DWgVoE38uFCUz/ALfz/D74+1GaelcDjtIpn/gODX6mmHQAmquHNwL4hyfV24A+r4RfSeisVw5Bt0XyHaO5DzH6fs6uBfwcubCBLx9+f1j7bhR8yfa2b524ZcgXwceDDTWRpKteieSZo9kPMvnIB/wa8qBr+A+CW5c4FvBx4hNa+76D1edHNtdfd9D9+n0/EzwF7gIPV7bpq/C8A/7hgvmtofer+GPD+BeN/HdhXPYH3AZuGJNc5tP6iPgzcD7xqWJ6zRY91iOYKvN/n7Fu0/tA9UP38VZ95TloPrT+sb6+Gg9YXlDwG7AfGu3nuljpX9VpP4KEFz9E1y51r0WNM0GCBN/DveDmwt3rO/gFYOyS5PgA8Sqsf/o7qaLU6P55KL0mFGrajUCRJNVngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVD/ByJiR/q8LUj2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.Dense(300, activation='selu', kernel_initializer='lecun_normal'))\n",
    "model_fashion.add(tf.keras.layers.Dense(100, activation='selu', kernel_initializer='lecun_normal'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "## check for example layer 1\n",
    "weights_1, biases_1 = model_fashion.layers[1].get_weights()\n",
    "\n",
    "## check for feature of these features\n",
    "pd.DataFrame(weights_1).iloc[:,0].hist(bins=30);  ## distribution of feature 1 ==> normal distr."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92cea0c4",
   "metadata": {},
   "source": [
    "#### you can apply `LeakRelu` like this\n",
    "#### as i said, it is preferable for Relu and its variants to be 'he_normal or he_uniform' initailization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "59219ef2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPV0lEQVR4nO3df4wc91nH8ffT0KiWL41tnGyNU3GgWoWIU39kKZEioTvSViGpsJEIahXKRQSdKhHUPxwJQ/+gqKowSOYPpAphIOIA0SNCRLbCj9YYjqpSKbGhySVKgttiQhzLVlPb9KKoYPTwx83lzvfDO3v787t+v6TV7szO7D7zZP3J3HdnZiMzkSSV5y2DLkCStDUGuCQVygCXpEIZ4JJUKANckgr1Pf18s927d+f4+Hg/33JTr7/+Otu3bx90GUPBXqywFyvsxbUG2Y/Tp09/KzNvWzu/rwE+Pj7OqVOn+vmWm5qfn2dycnLQZQwFe7HCXqywF9caZD8i4j83mu8QiiQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFaqvZ2JKw2r80F8DcHDiKg9Xjzdy9vAD/SpJaqlWgEfEWeA7wP8BVzOzGRG7gL8AxoGzwM9m5qXelClJWqudIZSpzHxvZjar6UPAyczcB5yspiVJfdLJGPh+YLZ6PAsc6LgaSVJtUedHjSPiP4BLQAK/n5lHI+JyZu5YtcylzNy5wbozwAxAo9G4a25urlu1d2RxcZGxsbFBlzEU7AUsnLsCQGMbXHijO685sffW7rzQgPi5uNYg+zE1NXV61ejHm+p+iXlPZr4aEbcDJyLixbpvnJlHgaMAzWYzh+XylF4qc4W94M0vLg9OXOXIQne+2z/70GRXXmdQ/Fxcaxj7UWsIJTNfre4vAk8CHwAuRMQegOr+Yq+KlCSt1zLAI2J7RNyy/Bj4MPAccByYrhabBo71qkhJ0np1/lZsAE9GxPLyf56ZfxcRTwNPRMQjwMvAg70rU5K0VssAz8xvAu/ZYP5rwL29KEqS1Jqn0ktSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCeT1wacDGr3P98dW8FrnWcg9ckgplgEtSoQxwSSqUY+AaaXXHl6USuQcuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSobweuDri7zlKg+MeuCQVygCXpEIZ4JJUKANckgpVO8Aj4qaI+LeIeKqa3hURJyLiTHW/s3dlSpLWamcP/JPAC6umDwEnM3MfcLKaliT1Sa0Aj4g7gAeAP1w1ez8wWz2eBQ50tTJJ0nVFZrZeKOIvgd8EbgEey8yPRMTlzNyxaplLmbluGCUiZoAZgEajcdfc3Fy3au/I4uIiY2Njgy5jKHTSi4VzV2otN7H31i29fqfq1ressQ0uvNGd9667ze3W2K33bcV/I9caZD+mpqZOZ2Zz7fyWJ/JExEeAi5l5OiIm233jzDwKHAVoNps5Odn2S/TE/Pw8w1LLoHXSi4frnsjz0NZev1N161t2cOIqRxa6c35b3W1ut8ZuvW8r/hu51jD2o84n9R7gpyLifuBtwNsj4s+ACxGxJzPPR8Qe4GIvC5UkXavlGHhm/mpm3pGZ48BHgX/IzJ8DjgPT1WLTwLGeVSlJWqeT48APAx+KiDPAh6ppSVKftDXYl5nzwHz1+DXg3u6XJEmqwzMxJalQBrgkFcoAl6RC+YMOUo/U/bELaavcA5ekQhngklQoA1ySCuUYuIrk+LLkHrgkFcsAl6RCGeCSVCgDXJIKZYBLUqEMcEkqlAEuSYXyOHBpxNQ9Rv7s4Qd6XIl6zT1wSSqUAS5JhTLAJalQjoGrL7x2idR97oFLUqEMcEkqlAEuSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAtAzwi3hYR/xIRz0TE8xHxG9X8XRFxIiLOVPc7e1+uJGlZnT3w7wI/kZnvAd4L3BcRdwOHgJOZuQ84WU1LkvqkZYDnksVq8q3VLYH9wGw1fxY40IsCJUkbi8xsvVDETcBp4F3A5zLzVyLicmbuWLXMpcxcN4wSETPADECj0bhrbm6uW7V3ZHFxkbGxsUGXcV0L567UWm5i760dvV5jG1x4o/3Xu95rlmptL25ky71o5/MwygaZGVNTU6czs7l2fq0Af3PhiB3Ak8AvA1+uE+CrNZvNPHXqVO3366X5+XkmJycHXcZ1dfvHaTd7vYMTVzmysPLbHu382O2o/VDD2l7cyJZ74Y8fLxlkZkTEhgHe1lEomXkZmAfuAy5ExJ7qxfcAFzsvU5JUV52jUG6r9ryJiG3AB4EXgePAdLXYNHCsRzVKkjZQ52/FPcBsNQ7+FuCJzHwqIr4CPBERjwAvAw/2sE5J0hotAzwznwXet8H814B7e1GUJKk1z8SUpEIZ4JJUKI+XknRd7Rwq6iGH/eUeuCQVygCXpEIZ4JJUKMfAtc6onR4vjSr3wCWpUAa4JBXKAJekQjkGPgIcs5ZuTO6BS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcoAl6RCGeCSVCivBz4AXr9bUje4By5JhTLAJalQBrgkFcoAl6RCtQzwiHhnRPxjRLwQEc9HxCer+bsi4kREnKnud/a+XEnSsjp74FeBg5n5w8DdwC9FxJ3AIeBkZu4DTlbTkqQ+aRngmXk+M/+1evwd4AVgL7AfmK0WmwUO9KhGSdIG2hoDj4hx4H3AV4FGZp6HpZAHbu96dZKkTUVm1lswYgz4J+CzmflXEXE5M3esev5SZq4bB4+IGWAGoNFo3DU3N9eVwju1uLjI2NjYQN574dyVgbzvZhrb4MIbg65iONiLFVvpxcTeW3tTzBAYZGZMTU2dzszm2vm1Ajwi3go8BXwhM3+nmvcSMJmZ5yNiDzCfme++3us0m808derUljag2+bn55mcnBzIew/bmZgHJ65yZMGTcsFerLaVXpw9/ECPqhm8QWZGRGwY4HWOQgngj4AXlsO7chyYrh5PA8e6UagkqZ46/3u9B/g4sBARX6vm/RpwGHgiIh4BXgYe7EmFkqQNtQzwzPwyEJs8fW93y5FUsrrDg6M81NJPnokpSYUywCWpUAa4JBXKAJekQhngklQoA1ySCmWAS1KhPGdYUt91+3ISN+px5e6BS1KhDHBJKpQBLkmFcgy8i4btMrGSRpt74JJUKANckgplgEtSoQxwSSqUAS5JhTLAJalQBrgkFcrjwCXdMEbtNzvdA5ekQhngklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAeB16D1/mWNIzcA5ekQhngklQoA1ySCmWAS1KhWgZ4RDweERcj4rlV83ZFxImIOFPd7+xtmZKktersgf8xcN+aeYeAk5m5DzhZTUuS+qhlgGfml4Bvr5m9H5itHs8CB7pbliSplcjM1gtFjANPZeaPVNOXM3PHqucvZeaGwygRMQPMADQajbvm5ua6UHbnFhcXGRsbq7XswrkrPa5msBrb4MIbg65iONiLFTdyLyb23rpuXjuZ0W1TU1OnM7O5dn7PT+TJzKPAUYBms5mTk5O9fsta5ufnqVvLwyN+Is/BiascWfCcLrAXq93IvTj70OS6ee1kRr9s9SiUCxGxB6C6v9i9kiRJdWw1wI8D09XjaeBYd8qRJNXV8u+jiPg8MAnsjohXgF8HDgNPRMQjwMvAg70sUpL6aaPrHx2cuLpuOHXQv53ZMsAz82ObPHVvl2uRJLXBMzElqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoW7M82RZur7JqJ8iL6m32vm93F4cM+4euCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSqUAS5JhRq548DrHpd5cKLHhUhSj7kHLkmFMsAlqVAGuCQVygCXpEIV8yVmOxeNkaQbgXvgklQoA1ySCmWAS1KhDHBJKpQBLkmFMsAlqVAGuCQVygCXpEIZ4JJUKANckgplgEtSoQxwSSpURwEeEfdFxEsR8fWIONStoiRJrW05wCPiJuBzwE8CdwIfi4g7u1WYJOn6OtkD/wDw9cz8Zmb+DzAH7O9OWZKkViIzt7ZixM8A92XmL1bTHwd+LDMfXbPcDDBTTb4beGnr5XbVbuBbgy5iSNiLFfZihb241iD78f2ZedvamZ38oENsMG/d/w0y8yhwtIP36YmIOJWZzUHXMQzsxQp7scJeXGsY+9HJEMorwDtXTd8BvNpZOZKkujoJ8KeBfRHxAxFxM/BR4Hh3ypIktbLlIZTMvBoRjwJfAG4CHs/M57tWWe8N3bDOANmLFfZihb241tD1Y8tfYkqSBsszMSWpUAa4JBVqpAM8InZFxImIOFPd79xkuccj4mJEPLeV9UvQRi82vDxCRHw6Is5FxNeq2/39q747Wl36IZb8bvX8sxHx/rrrlqbDXpyNiIXqc3Cqv5V3X41e/FBEfCUivhsRj7Wzbs9l5sjegN8GDlWPDwG/tclyPw68H3huK+uXcKuzLSx9Gf0N4AeBm4FngDur5z4NPDbo7ehg+zfdtlXL3A/8LUvnONwNfLXuuiXdOulF9dxZYPegt6OPvbgd+FHgs6v/DQzD52Kk98BZOrV/tno8CxzYaKHM/BLw7a2uX4g62zLKl0eos237gT/JJf8M7IiIPTXXLUknvRg1LXuRmRcz82ngf9tdt9dGPcAbmXkeoLq/vc/rD5M627IX+K9V069U85Y9Wv05/XiBw0mttu16y9RZtySd9AKWzrj+YkScri6VUbJO/tsO/HPRyan0QyEi/h54xwZPfarftQxaF3pxvcsj/B7wmWr6M8AR4BfarXGA6lz6YbNlal02oiCd9ALgnsx8NSJuB05ExIvVX7El6uS/7cA/F8UHeGZ+cLPnIuJCROzJzPPVn38X23z5Ttfvqy70YtPLI2TmhVWv9QfAU92pum/qXPphs2VurrFuSTrpBZm5fH8xIp5kaSih1ADv5JIgA7+cyKgPoRwHpqvH08CxPq8/TOpsy6aXR1gz/vnTwHMbrD/M6lz64Tjw89URGHcDV6rhplG7bMSWexER2yPiFoCI2A58mPI+C6t18t928J+LQX8L3Msb8L3ASeBMdb+rmv99wN+sWu7zwHmWvqR4BXjkeuuXeGujF/cD/87St+ufWjX/T4EF4FmWPqR7Br1NW+jBum0DPgF8onocLP1IyTeqbW226kupt632gqUjLp6pbs/fIL14R5UL/w1crh6/fRg+F55KL0mFGvUhFEkaWQa4JBXKAJekQhngklQoA1ySCmWAS1KhDHBJKtT/A8G1fMnDDinFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.Dense(300, kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.LeakyReLU(0.01))  ## here alpha=0.01\n",
    "model_fashion.add(tf.keras.layers.Dense(100, kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "## check for example layer 1\n",
    "weights_1, biases_1 = model_fashion.layers[1].get_weights()\n",
    "\n",
    "## check for feature of these features\n",
    "pd.DataFrame(weights_1).iloc[:,0].hist(bins=30);  ## distribution of feature 1 ==> normal distr."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "648e4aef",
   "metadata": {},
   "source": [
    "#### you can apply `ParametricRelu` like this\n",
    "#### as i said, it is preferable for Relu and its variants to be 'he_normal or he_uniform' initailization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9e62e892",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPcUlEQVR4nO3df4hl91nH8fdj2tqQabNZk52u2+IqhGhw6Y9Ma6AgM6YtMSlsBFNaatxidCka6R9bZLV/pFCKUYiIIuKqwVG001Aadok/6ro6lkINndW2k5LWtHWNSZZd2m7WTgjV1cc/5mxnnL2z98z9/cy8X3C555x7zr3P+d6znz3zvedHZCaSpHq+Z9wFSJJ6Y4BLUlEGuCQVZYBLUlEGuCQV9bJRftiNN96Y+/fvH+VHdvTiiy9y3XXXjbuMsbMd1tgWq2yHNZPUFqdPn/5GZt60cfpIA3z//v0sLS2N8iM7WlxcZHZ2dtxljJ3tsMa2WGU7rJmktoiIf+803S4USSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSpqpGdiSpNq+bmLvO/oX3ad78xDd4+gGqkd98AlqSgDXJKKMsAlqSgDXJKKMsAlqSgDXJKKMsAlqahWx4FHxBng28D/AJcycyYidgMfB/YDZ4B3ZeaF4ZQpSdpoK3vgc5n5hsycacaPAqcy82bgVDMuSRqRfrpQDgLzzfA8cE/f1UiSWovM7D5TxL8BF4AE/iAzj0XEC5m5a908FzLzhg7LHgYOA0xPT9+2sLAwqNp7trKywtTU1LjLGDvbYc35b13k3Evd5zuw7/rhFzNGbhNrJqkt5ubmTq/r/fiuttdCeWtmPh8Re4CTEfHlth+cmceAYwAzMzM5CXd5nqS7TY+T7bDmd//8OA8vd//ncOa9s8MvZozcJtZUaItWXSiZ+XzzfB54DHgLcC4i9gI0z+eHVaQk6UpdAzwirouIV10eBt4BPAmcAA41sx0Cjg+rSEnSldp0oUwDj0XE5fn/IjP/JiI+BzwaEfcDzwD3Dq9MqTf7W1wiFuDIgSEXIg1B1wDPzK8Dr+8w/ZvAHcMoSpLUnWdiSlJRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFWWAS1JRBrgkFdX2crKSaH9tFYAzD909xEok98AlqSwDXJKKMsAlqSgDXJKKMsAlqSgDXJKKMsAlqSiPA5eGpO0x4x4vrl65By5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRbUO8Ii4JiL+JSIeb8Z3R8TJiHi6eb5heGVKkjbayh74B4Cn1o0fBU5l5s3AqWZckjQirQI8Il4L3A380brJB4H5ZngeuGeglUmSrqrtHvhvA78C/O+6adOZeRaged4z2NIkSVcTmXn1GSLeCdyVmb8YEbPABzPznRHxQmbuWjffhcy8oh88Ig4DhwGmp6dvW1hYGGD5vVlZWWFqamrcZYzdTmiH5ecutppv+lo499KQi9nEgX3Xj+eDO9gJ20Rbk9QWc3NzpzNzZuP0NgH+68B9wCXglcCrgU8CbwZmM/NsROwFFjPzlqu918zMTC4tLfW4CoOzuLjI7OzsuMsYu53QDm1vqnDkwCUeXh7P/U0m6YYOO2GbaGuS2iIiOgZ41y6UzPzVzHxtZu4H3g38fWb+DHACONTMdgg4PsB6JUld9HMc+EPA2yPiaeDtzbgkaUS29DdjZi4Ci83wN4E7Bl+SJKkNz8SUpKIMcEkqajw/u0vasrZH1EzSUS0aLvfAJakoA1ySijLAJako+8BVUtv+YGk7cw9ckooywCWpKANckooywCWpKANckooywCWpKANckoryOHBNFI/vltpzD1ySijLAJakoA1ySirIPfAfZ2L985MAl3tehz3kY15O2b1saPPfAJakoA1ySijLAJako+8AlDYz37Rwt98AlqSgDXJKKMsAlqSj7wKUdqlN/dadzA+yvnlzugUtSUQa4JBVlgEtSUV37wCPilcCnge9t5v9EZj4YEbuBjwP7gTPAuzLzwvBKlbYnrxOjXrXZA/8O8BOZ+XrgDcCdEXE7cBQ4lZk3A6eacUnSiHQN8Fy10oy+vHkkcBCYb6bPA/cMo0BJUmet+sAj4pqI+DxwHjiZmU8A05l5FqB53jO0KiVJV4jMbD9zxC7gMeCXgc9k5q51r13IzBs6LHMYOAwwPT1928LCQp8l929lZYWpqalxlzFyy89d/H/j09fCuZeunO/AvuuH/tmTZrO2qKjt99fpO+nUDlvZHtp+z8PYxgZtknJibm7udGbObJy+pQAHiIgHgReBXwBmM/NsROwFFjPzlqstOzMzk0tLS1v6vGFYXFxkdnZ23GWMXKcbOjy8fOXv2Dvxhg6btUVFbb+/zU7k2dgOW9kettPFrCYpJyKiY4B37UKJiJuaPW8i4lrgbcCXgRPAoWa2Q8DxgVUrSeqqzS7HXmA+Iq5hNfAfzczHI+KzwKMRcT/wDHDvEOvUhJr0PWtpO+sa4Jn5ReCNHaZ/E7hjGEVJkrrzTExJKsoAl6SitsfP7pK+y98ldg73wCWpKANckooywCWpKPvAJ9i4zmqzD1WqwT1wSSrKAJekogxwSSrKAJekogxwSSrKAJekogxwSSrK48AHyOO2JY2Se+CSVJQBLklFGeCSVJR94JKuyt9YJpd74JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUlAEuSUUZ4JJUVNcAj4jXRcQ/RMRTEfGliPhAM313RJyMiKeb5xuGX64k6bI2e+CXgCOZ+SPA7cAvRcStwFHgVGbeDJxqxiVJI9I1wDPzbGb+czP8beApYB9wEJhvZpsH7hlSjZKkDrbUBx4R+4E3Ak8A05l5FlZDHtgz8OokSZuKzGw3Y8QU8I/ARzPzkxHxQmbuWvf6hcy8oh88Ig4DhwGmp6dvW1hYGEjh/VhZWWFqamrg77v83MVW8x3Yd/1A369X09fCuZeG+hFl2BarJq0d2v5bGYZh5UQv5ubmTmfmzMbprQI8Il4OPA58KjN/q5n2FWA2M89GxF5gMTNvudr7zMzM5NLSUk8rMEiLi4vMzs4O/H0HfVPjYV9I/8iBSzy87D09wLa4bNLaYdA3AN+KYeVELyKiY4C3OQolgD8Gnroc3o0TwKFm+BBwfBCFSpLaafNf7VuB+4DliPh8M+3XgIeARyPifuAZ4N6hVChJ6qhrgGfmZ4DY5OU7BluOJKktz8SUpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqygCXpKIMcEkqanJuvTHBBn1nnGHfaUfaaQZ9N6wq3AOXpKIMcEkqygCXpKLsA5e0Y2y3vnL3wCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpKANckooywCWpqG13LZQ21zo4cuASs8MvRZKGquseeEQ8EhHnI+LJddN2R8TJiHi6eb5huGVKkjZq04XyJ8CdG6YdBU5l5s3AqWZckjRCXQM8Mz8NfGvD5IPAfDM8D9wz2LIkSd1EZnafKWI/8Hhm/mgz/kJm7lr3+oXM7NiNEhGHgcMA09PTty0sLAyg7M0tP3ex6zzT18Ke3dcP9D0rmr4Wzr007iomg22xynZY06YtDuxrnyP9mJubO52ZMxunD/1HzMw8BhwDmJmZydnZ2aF+3vta/oj5ri3U0eY9Kzpy4BIPL2+737F7Ylussh3WtGmLM++dHU0xm+j1MMJzEbEXoHk+P7iSJElt9BrgJ4BDzfAh4PhgypEktdX1b6WI+BgwC9wYEc8CDwIPAY9GxP3AM8C9wywS2t/LTpJ2iq4Bnpnv2eSlOwZciyRpCzyVXpKKMsAlqagde7yQfeqSqnMPXJKKMsAlqSgDXJKKMsAlqSgDXJKKMsAlqagdexihJPVrK4cjn3no7oF/vnvgklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRRngklSUAS5JRfUV4BFxZ0R8JSK+GhFHB1WUJKm7ngM8Iq4Bfg/4SeBW4D0RceugCpMkXV0/e+BvAb6amV/PzP8CFoCDgylLktRNZGZvC0b8NHBnZv58M34f8GOZ+cCG+Q4Dh5vRW4Cv9F7uwNwIfGPcRUwA22GNbbHKdlgzSW3xA5l508aJL+vjDaPDtCv+N8jMY8CxPj5n4CJiKTNnxl3HuNkOa2yLVbbDmgpt0U8XyrPA69aNvxZ4vr9yJElt9RPgnwNujogfjIhXAO8GTgymLElSNz13oWTmpYh4APgUcA3wSGZ+aWCVDddEdemMke2wxrZYZTusmfi26PlHTEnSeHkmpiQVZYBLUlHbMsAjYndEnIyIp5vnGzaZ75GIOB8RT/ayfAVbaIuOl0WIiA9HxHMR8fnmcdfoqu9ft8s9xKrfaV7/YkS8qe2y1fTZFmciYrnZBpZGW/ngtWiLH46Iz0bEdyLig1tZdqQyc9s9gN8EjjbDR4Hf2GS+HwfeBDzZy/IVHm3WhdUfob8G/BDwCuALwK3Nax8GPjju9ehx3Tddr3Xz3AX8NavnNdwOPNF22UqPftqiee0McOO412OEbbEHeDPw0fXb/6RtF9tyD5zVU/rnm+F54J5OM2Xmp4Fv9bp8EW3WZbteFqHNeh0E/jRX/ROwKyL2tly2kn7aYrvp2haZeT4zPwf891aXHaXtGuDTmXkWoHneM+LlJ0mbddkH/Me68WebaZc90PxJ/Uix7qRu63W1edosW0k/bQGrZ1n/bUScbi6PUVk/3+1EbRf9nEo/VhHxd8BrOrz0oVHXMm4DaIurXRbh94GPNOMfAR4Gfm6rNY5Jm8s9bDZPq0tFFNJPWwC8NTOfj4g9wMmI+HLzF2xF/Xy3E7VdlA3wzHzbZq9FxLmI2JuZZ5s/Ac9v8e37XX6kBtAWm14WITPPrXuvPwQeH0zVI9Hmcg+bzfOKFstW0k9bkJmXn89HxGOsdiVUDfB+LgMyUZcQ2a5dKCeAQ83wIeD4iJefJG3WZdPLImzoA/0p4MkOy0+qNpd7OAH8bHMExu3AxaarabtdKqLntoiI6yLiVQARcR3wDmptBxv1891O1nYx7l+Eh/EAvg84BTzdPO9upn8/8Ffr5vsYcJbVHyqeBe6/2vIVH1toi7uAf2X1F/YPrZv+Z8Ay8EVWN9S9416nLa7/FesFvB94fzMcrN6Y5GvNes50a5Oqj17bgtUjLr7QPL60Q9riNU0m/CfwQjP86knbLjyVXpKK2q5dKJK07RngklSUAS5JRRngklSUAS5JRRngklSUAS5JRf0flLygfnZ64gQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.Dense(300, kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.PReLU())  \n",
    "model_fashion.add(tf.keras.layers.Dense(100, kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "## check for example layer 1\n",
    "weights_1, biases_1 = model_fashion.layers[1].get_weights()\n",
    "\n",
    "## check for feature of these features\n",
    "pd.DataFrame(weights_1).iloc[:,0].hist(bins=30);  ## distribution of feature 1 ==> normal distr."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfb548fc",
   "metadata": {},
   "source": [
    "### `Conclusion`\n",
    "#### you can choose \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "75e86bb6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Activation</th>\n",
       "      <th>Initialization</th>\n",
       "      <th>variance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None, Sigmoid, tanh, softmax</td>\n",
       "      <td>Golort</td>\n",
       "      <td>1/fan_avg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Relu and its Variants</td>\n",
       "      <td>He</td>\n",
       "      <td>2/fan_in</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Selu</td>\n",
       "      <td>LeCun</td>\n",
       "      <td>1/fan_in</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Activation Initialization   variance\n",
       "0  None, Sigmoid, tanh, softmax         Golort  1/fan_avg\n",
       "1         Relu and its Variants             He   2/fan_in\n",
       "2                          Selu          LeCun   1/fan_in"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values_1 = ['None, Sigmoid, tanh, softmax', 'Relu and its Variants', 'Selu']\n",
    "values_2 = ['Golort', 'He', 'LeCun']\n",
    "values_3 = ['1/fan_avg', '2/fan_in', '1/fan_in']\n",
    "pd.DataFrame({'Activation': values_1, 'Initialization':values_2, 'variance':values_3})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82ee4841",
   "metadata": {},
   "source": [
    "###### **Selu Notes :** <br />\n",
    "This activation function was proposed in this [great paper](https://arxiv.org/pdf/1706.02515.pdf) by Günter Klambauer, Thomas Unterthiner and Andreas Mayr, published in June 2017. During training, a neural network composed exclusively of a stack of dense layers using the SELU activation function and LeCun initialization will self-normalize: the output of each layer will tend to preserve the same mean and variance during training, which solves the vanishing/exploding gradients problem. As a result, this activation function outperforms the other activation functions very significantly for such neural nets, so you should really try it out. Unfortunately, the self-normalizing property of the SELU activation function is easily broken: you cannot use ℓ<sub>1</sub> or ℓ<sub>2</sub> regularization, regular dropout, max-norm, skip connections or other non-sequential topologies (so recurrent neural networks won't self-normalize). However, in practice it works quite well with sequential CNNs. If you break self-normalization, SELU will not necessarily outperform other activation functions."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61da69cd",
   "metadata": {},
   "source": [
    "### Batch Normalization\n",
    "`this technique is used to solve the Problem of Vanishing of the Gradients and also works as a Regularizer`\n",
    "\n",
    "for each Batch ==> Normalize ==> called Batch Normalization\n",
    "$$mean = 'the-mean-of-this-batch'$$\n",
    "$$std = 'the-std-of-this-batch'$$\n",
    "$$X^{-} = (x - mean) / std$$\n",
    "$$Z_{new} = gamma * X^{-} + bias$$\n",
    "\n",
    "#### Model try to learn (gamma and bias) through BackPropagation\n",
    "\n",
    "##### you can try both adding Batch Normalization Layer before or after the Activation Function"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "091bf994",
   "metadata": {},
   "source": [
    "`Batch Normalization after Activation Function`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "80e605de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_5 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 784)               3136      \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 271,346\n",
      "Trainable params: 268,978\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "## Adding Batch Normalization Layer before Activation Function\n",
    "## you can Add Batch Normalization Layer before any layer works equally as StandardScaler\n",
    "\n",
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.BatchNormalization()) ## as StandardScaler\n",
    "model_fashion.add(tf.keras.layers.Dense(300, activation='relu', kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.BatchNormalization()) ## added it after Activation Function\n",
    "model_fashion.add(tf.keras.layers.Dense(100, activation='relu', kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.BatchNormalization()) ## added it after Activation Function\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "### note that (gamma and bias) model try to learn them using BackPropagation\n",
    "## note that (mean and std) model calculates them directly using moving average\n",
    "## so we have here for each Batch Normalization (4 parameters: gamma, bias, mean, std)\n",
    "## beacuse model does not learn what mean and std are , directly calculates them\n",
    "## keras set them to (non_trainable) parameters ==> check summary\n",
    "\n",
    "model_fashion.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1248576b",
   "metadata": {},
   "source": [
    "`Batch Normalization before Activation Function`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "59a3578d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_6 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 784)               3136      \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 300)               1200      \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 100)               400       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 271,346\n",
      "Trainable params: 268,978\n",
      "Non-trainable params: 2,368\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "## Adding Batch Normalization Layer before Activation Function\n",
    "## you can Add Batch Normalization Layer before any layer works equally as StandardScaler\n",
    "\n",
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.BatchNormalization()) ## as StandardScaler\n",
    "model_fashion.add(tf.keras.layers.Dense(300, kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.BatchNormalization()) ## added it before Activation Function\n",
    "model_fashion.add(tf.keras.layers.Activation('relu'))\n",
    "model_fashion.add(tf.keras.layers.Dense(100, kernel_initializer='he_normal'))\n",
    "model_fashion.add(tf.keras.layers.BatchNormalization()) ## added it before Activation Function\n",
    "model_fashion.add(tf.keras.layers.Activation('relu'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "### note that (gamma and bias) model try to learn them using BackPropagation\n",
    "## note that (mean and std) model calculates them directly using moving average\n",
    "## so we have here for each Batch Normalization (4 parameters: gamma, bias, mean, std)\n",
    "## beacuse model does not learn what mean and std are , directly calculates them\n",
    "## keras set them to (non_trainable) parameters ==> check summary\n",
    "\n",
    "model_fashion.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2fdfb4fc",
   "metadata": {},
   "source": [
    "### Gradient Cliping\n",
    "#### `it is a technique specially for RNN when gradients are exploding`\n",
    "###### we set that weights should not exceed determined value, this value may by two Methods\n",
    "1. by using (clipvalue ), here you say that value should not exceed it\n",
    "2. by using (clipnorm  ), here you say that norm should not exceed it\n",
    "###### `try both`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bd2785c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "## model\n",
    "model_fashion = tf.keras.models.Sequential()\n",
    "\n",
    "## first flatten the input of each image\n",
    "\n",
    "model_fashion.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_fashion.add(tf.keras.layers.Dense(300, activation='tanh', kernel_initializer='glorot_uniform'))\n",
    "model_fashion.add(tf.keras.layers.Dense(100, activation='tanh', kernel_initializer='glorot_uniform'))\n",
    "model_fashion.add(tf.keras.layers.Dense(10, activation='softmax'))  ## 10 calss with sotmax function\n",
    "\n",
    "optimizer_clipValue = tf.keras.optimizers.SGD(clipvalue=1.0)  ## or try (clipnorm=?)\n",
    "model_fashion.compile(loss='sparse_categorical_crossentropy', \n",
    "                      optimizer=optimizer_clipValue, \n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "early_stopp = tf.keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True)\n",
    "# history = model_fashion.fit(X_train, y_train, \n",
    "#                             validation_data=(X_val, y_val), \n",
    "#                             epochs=30, batch_size=32, callbacks=[early_stopp])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acd7f6ae",
   "metadata": {},
   "source": [
    "## Optimizers\n",
    "\n",
    "`In a previous Notebook, I implemented (GD, SGD, mini_batch GD)`\n",
    "\n",
    "`Here, I will implement (Momentum, NAG, Adagrad, RMSProp, Adam)`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78007790",
   "metadata": {},
   "source": [
    "### Momentum"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "466a852a",
   "metadata": {},
   "source": [
    "#### Dummy Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7ebfe023",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.linspace(0,20,50)\n",
    "y = -1 * X + 2\n",
    "X = X.reshape(-1,1) \n",
    "X =  np.insert(arr=X, obj=0, values=1, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "167f6eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "theta_moment = np.zeros(2)\n",
    "def momentum_based_GD(X, y, theta_moment, alfa, iters, gamma):\n",
    "    \n",
    "    cost_steps = np.zeros(iters)\n",
    "    theta_steps = np.zeros(iters*2).reshape(-1,2)\n",
    "    \n",
    "    vt = 0\n",
    "    for i in range(iters):\n",
    "        h_theta = X @ theta_moment\n",
    "        loss = h_theta - y\n",
    "        cost = np.sum(loss**2)/(2 * len(X))\n",
    "        \n",
    "        gradient = (X.T @ loss) / len(X)\n",
    "        \n",
    "        vt = gamma * vt + alfa * gradient      \n",
    "        theta_moment = theta_moment - vt\n",
    "        \n",
    "        # append in arrays\n",
    "        theta_trmpor = theta_moment\n",
    "        cost_steps[i] = cost\n",
    "        theta_steps[i] = theta_trmpor\n",
    "        \n",
    "        \n",
    "    return theta_moment, theta_steps, cost_steps\n",
    "\n",
    "\n",
    "## call the function\n",
    "## my Results using Momentum\n",
    "theta_moment_final, theta_steps_moment, cost_steps_moment = momentum_based_GD(X, y, \n",
    "                                                                              theta_moment, \n",
    "                                                                              alfa=0.01, \n",
    "                                                                              iters=1000, \n",
    "                                                                              gamma=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5078d966",
   "metadata": {},
   "outputs": [],
   "source": [
    "### but momentum in keras is mouch easier\n",
    "optimizer_moment = tf.keras.optimizers.SGD(learning_rate=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9294f00",
   "metadata": {},
   "source": [
    "### Nestrov Accelerated Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "14f7941b",
   "metadata": {},
   "outputs": [],
   "source": [
    "thetas_NAG = np.zeros(2)\n",
    "def NAG_GD(X, y, thetas_nag, alfa, iters, gamma):\n",
    "         \n",
    "    cost_steps = np.zeros(iters)\n",
    "    theta_steps = np.zeros(iters*2).reshape(-1,2)\n",
    "    \n",
    "    vt = 0\n",
    "    \n",
    "    for i in range(iters):\n",
    "                \n",
    "        # get theta_temp\n",
    "        thetas_temp = thetas_nag - gamma * vt\n",
    "        \n",
    "        # Note : I want to get gradient_temp not gradient_thetas_nag\n",
    "        h_theta_temp = X @ thetas_temp\n",
    "        loss_temp = h_theta_temp - y\n",
    "        cost_temp = np.sum(loss_temp**2) / (2 * len(X))\n",
    "        gradient_temp = (X.T @ loss_temp) / len(X)\n",
    "        \n",
    "        \n",
    "        thetas_nag = thetas_temp - alfa * gradient_temp\n",
    "        vt = gamma * vt + alfa * gradient_temp\n",
    "        \n",
    "        \n",
    "        # append in arrays\n",
    "        theta_tempor = thetas_nag\n",
    "        cost_steps[i] = cost_temp\n",
    "        theta_steps[i] = theta_tempor\n",
    "               \n",
    "    return thetas_nag, theta_steps, cost_steps\n",
    "\n",
    "## here is my reults using NAG\n",
    "# using gamma = 0.9, alfa = 0.01\n",
    "thetas_nag_final, theta_steps_nag, cost_steps_nag = NAG_GD(X, y, thetas_NAG, 0.01, 1000, gamma=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c3d1c853",
   "metadata": {},
   "outputs": [],
   "source": [
    "### but NAG in keras is much easier\n",
    "optimizer_nag = tf.keras.optimizers.SGD(learning_rate=0.001, momentum=0.9, nesterov=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccc94b35",
   "metadata": {},
   "source": [
    "### Adagrad (not preferable in NN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c4c48b09",
   "metadata": {},
   "outputs": [],
   "source": [
    "thetas_adag = np.zeros(2)\n",
    "def adagrad(X, y, theta_input, alfa, epsilon):\n",
    "    \n",
    "    cost_steps = []\n",
    "    theta_steps = []\n",
    "    h_teta_steps = []\n",
    "    \n",
    "    \n",
    "    vt = 0\n",
    "    \n",
    "    # first calculations\n",
    "    h_theta = X @ theta_input     \n",
    "\n",
    "    loss = h_theta - y\n",
    "\n",
    "    cost = np.sum(loss**2) / (2 * len(X))\n",
    "\n",
    "    gradient = (X.T @ loss) / len(X)\n",
    "    \n",
    "    vt = vt + (gradient)**2\n",
    "\n",
    "    theta_input = theta_input - ((alfa * gradient) / (np.sqrt(vt)+epsilon))\n",
    "\n",
    "\n",
    "    cost_steps.append(cost)\n",
    "    theta_steps.append(theta_input)\n",
    "    h_teta_steps.append(h_theta)\n",
    "    \n",
    "   \n",
    "    \n",
    "    while (gradient[0] > 0.01 or gradient[0] > -0.01) and (gradient[1] > 0.01 or gradient[1] > -0.01): \n",
    "        \n",
    "        h_theta = X @ theta_input     \n",
    "        \n",
    "        loss = h_theta - y\n",
    "        \n",
    "        cost = np.sum(loss**2) / (2 * len(X))\n",
    "        \n",
    "        gradient = (X.T @ loss) / len(X)\n",
    "        \n",
    "        vt = vt + (gradient)**2\n",
    "        \n",
    "        theta_input = theta_input - ((alfa * gradient) / (np.sqrt(vt)+epsilon))\n",
    "        \n",
    "        # append in array\n",
    "        cost_steps.append(cost)\n",
    "        theta_steps.append(theta_input)\n",
    "        h_teta_steps.append(h_theta)\n",
    "    \n",
    "    return theta_input, theta_steps, cost_steps, h_teta_steps\n",
    "\n",
    "## my results\n",
    "theta_adag1, theta_steps_adag1, cost_steps_adag1, h_steps_adag1 = adagrad(X, y, \n",
    "                                                                          thetas_adag, 0.01, 1e-8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c005c0d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "## in keras Adgarad is much easier\n",
    "optimizer_adagard = tf.keras.optimizers.Adagrad(learning_rate=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f0e8e59",
   "metadata": {},
   "source": [
    "### RMSProp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b0a04fda",
   "metadata": {},
   "outputs": [],
   "source": [
    "thetas_rms1 = np.zeros(2)\n",
    "def RMSProp(X, y, theta_input, alfa, epsilon, beta):\n",
    "    \n",
    "    cost_steps = []\n",
    "    theta_steps = []\n",
    "    h_theta_rms = []\n",
    "    \n",
    "    vt = 0\n",
    "    \n",
    "    h_theta = X @ theta_input     \n",
    "\n",
    "    loss = h_theta - y\n",
    "\n",
    "    cost = np.sum(loss**2) / (2 * len(X))\n",
    "\n",
    "    gradient = (X.T @ loss) / len(X)\n",
    "\n",
    "    vt = beta * vt + (1 - beta) * gradient**2\n",
    "\n",
    "    theta_input = theta_input - ((alfa * gradient) / (np.sqrt(vt)+epsilon))\n",
    "    \n",
    "    \n",
    "    cost_steps.append(cost)\n",
    "    theta_steps.append(theta_input)\n",
    "    h_theta_rms.append(h_theta)\n",
    "    \n",
    "    while (gradient[0] > 0.01 or gradient[0] > -0.01) and (gradient[1] > 0.01 or gradient[1] > -0.01):\n",
    "        \n",
    "        h_theta = X @ theta_input     \n",
    "        \n",
    "        loss = h_theta - y\n",
    "        \n",
    "        cost = np.sum(loss**2) / (2 * len(X))\n",
    "        \n",
    "        gradient = (X.T @ loss) / len(X)\n",
    "        \n",
    "        vt = beta * vt + (1 - beta) * gradient**2\n",
    "        \n",
    "        theta_input = theta_input - ((alfa * gradient) / (np.sqrt(vt)+epsilon))\n",
    "        \n",
    "        # append in array\n",
    "        cost_steps.append(cost)\n",
    "        theta_steps.append(theta_input)\n",
    "        h_theta_rms.append(h_theta)\n",
    "\n",
    "    return theta_input, theta_steps, cost_steps, h_theta_rms\n",
    "\n",
    "\n",
    "## my results\n",
    "# using alfa = 0.5 and epsilon = 1e-8 and beta = 0.4\n",
    "thetas_rms1, theta_steps_rms1, cost_steps_rms1, h_thetas = RMSProp(X, y, thetas_rms1, 0.5, 1e-8, 0.9)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5232a947",
   "metadata": {},
   "outputs": [],
   "source": [
    "## in keras RMSProp is much easier\n",
    "optimizer_rmsprop = tf.keras.optimizers.RMSprop(learning_rate=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcd66c6b",
   "metadata": {},
   "source": [
    "### Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d28df9c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "thetas_adam1 = np.zeros(2)\n",
    "def Adam(X, y, theta_input, alfa, epsilon, beta1, beta2):\n",
    "    \n",
    "    cost_steps = []\n",
    "    theta_steps = []\n",
    "    h_thetas_steps = []\n",
    "       \n",
    "    vt = 0\n",
    "    mt = 0\n",
    "    i = 0\n",
    "\n",
    "    h_theta = X @ theta_input     \n",
    "\n",
    "    loss = h_theta - y\n",
    "\n",
    "    cost = np.sum(loss**2) / (2 * len(X))\n",
    "\n",
    "    gradient = (X.T @ loss) / len(X)\n",
    "\n",
    "    mt = beta1 * mt + (1 - beta1) * gradient\n",
    "    vt = beta2 * vt + (1 - beta2) * gradient**2\n",
    "\n",
    "    mt_bias = mt / (1 - beta1**(i+1))\n",
    "    vt_bias = vt / (1 - beta2**(i+1))\n",
    "\n",
    "    theta_input = theta_input - ((alfa * mt_bias) / (np.sqrt(vt_bias)+epsilon))\n",
    "    \n",
    "    \n",
    "    cost_steps.append(cost)\n",
    "    theta_steps.append(theta_input)\n",
    "    h_thetas_steps.append(h_theta)\n",
    "    \n",
    "   \n",
    "    while (gradient[0] > 0.01 or gradient[0] > -0.01) and (gradient[1] > 0.01 or gradient[1] > -0.01):\n",
    "        \n",
    "        i+=1\n",
    "        \n",
    "        h_theta = X @ theta_input     \n",
    "        \n",
    "        loss = h_theta - y\n",
    "        \n",
    "        cost = np.sum(loss**2) / (2 * len(X))\n",
    "        \n",
    "        gradient = (X.T @ loss) / len(X)\n",
    "        \n",
    "        mt = beta1 * mt + (1 - beta1) * gradient\n",
    "        vt = beta2 * vt + (1 - beta2) * gradient**2\n",
    "        \n",
    "        mt_bias = mt / (1 - beta1**(i+1))\n",
    "        vt_bias = vt / (1 - beta2**(i+1))\n",
    "        \n",
    "        theta_input = theta_input - ((alfa * mt_bias) / (np.sqrt(vt_bias)+epsilon))\n",
    "        \n",
    "        # append in array\n",
    "        cost_steps.append(cost)\n",
    "        theta_steps.append(theta_input)\n",
    "        h_thetas_steps.append(h_theta)\n",
    "\n",
    "    \n",
    "    return theta_input, theta_steps, cost_steps, h_thetas_steps\n",
    "\n",
    "## my reults using Adam\n",
    "thetas_adam1, thetas_steps_adam1, cost_adam1, hypo_adam = Adam(X, y, thetas_adam1, 0.8, 1e-8, 0.2, 0.2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4c5e27a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "## adam and Naam in keras\n",
    "optmizer_adam = tf.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)\n",
    "optmizer_Nadam = tf.keras.optimizers.Nadam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d339c41",
   "metadata": {},
   "source": [
    "-----------------"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3284ebb",
   "metadata": {},
   "source": [
    "### Learning Rate Scheduling\n",
    "`Instead of making LR constant we can make it high at start where cost is high and then decrease it`\n",
    "\n",
    "**Scheduling Methods**\n",
    "1. Power Scheduling\n",
    "2. Exponential Scheduling\n",
    "3. Piecewise Scheduling\n",
    "4. Performance Scheduling\n",
    "5. 1 Cycle Scheduling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "614648cc",
   "metadata": {},
   "source": [
    "#### `Power Scheduling`\n",
    "\n",
    "$$Lr^{t} = Lr_0 / (1 + t/s)^{c}$$\n",
    "`Keras uses c=1, decay=1/s`\n",
    "\n",
    "#### you can set that by parameter (decay, inverse of s) in the optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ef9519d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "### call the optimizer\n",
    "#### you can set that by parameter (decay, inverse of s) in the optimizer\n",
    "optimizer_power_LR = tf.keras.optimizers.SGD(learning_rate=0.01, decay=1e-4)\n",
    "\n",
    "# history = model_fashion.fit(X_train, y_train, \n",
    "#                             validation_data=(X_val, y_val), epochs=10,\n",
    "#                             batch_size=32, callbacks=[early_stopp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b679cd84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEaCAYAAADZvco2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwkklEQVR4nO3dd5hU5dnH8e+9DZa6lKXs0otUUUCKogKWiBWMJfrGmmI0mJim0Zjk1cRY8xo1URGVRGMLsRILSEQUUATFAojAitKrArLSFvZ+/zhn12HZcoDZmVn297muuXbOOc8zc8+Kc+9Tj7k7IiIi8ZCW7ABEROTgoaQiIiJxo6QiIiJxo6QiIiJxo6QiIiJxo6QiIiJxo6QikmLM7BIzK6ym155nZjfsY53PzexXFR2LxFJSkZRkZv8wMw8fRWa2xMz+bGb1kx1bVcyso5k9ZmYrzGyHma0ys5fMrG+yY4uTAcB9yQ5CUlNGsgMQqcR/gQuBTOAY4CGgPnBFMoMqYWaZ7l5U9hwwGfgUOBdYCeQDJwJNEx5kNXD39cmOQVKXWiqSyna4+xp3X+7uTwCPA6MAzKyOmd1lZmvNbLuZzTSzo0sqmtk7ZvbrmOPHw1ZPq/C4npntNLMh4bGZ2TVm9qmZbTOzuWZ2QUz9DmH9881sipltA35UTsy9gM7AaHd/y92Xhj9vdPfXYl6vkZndb2arw/gXmNl3Yl/IzI4Pu6u+NrPXzaxjmeunm9l7Yf3PzOxPZpYVc72Fmb0Qfp6lZva9ssGGn+nsMucq7d4qpzvMzewyM/t3GOuS2N9dWGaQmc0JY33fzE4J6w2r6H2kZlJSkZpkG0GrBeB24DvA94C+wFxgopm1Dq9PBYbH1B0KbACGhcdDgCJgVnh8E/B9YDTQE7gFeMDMTi0Twy0EXT89gefLiXE9UAycZWbl9gSYmQGvhDFdGr7WL4CdMcXqANeFn+9IIAcYE/MaJxEk2b8RJLLvAWcDN8e8xj+ALsAJBMn4IqBDeTHFwe+BF4DDgH8B48ysfRhrA+BF4BOgP3ANcEc1xSHJ5u566JFyD4IvxBdjjgcSJIV/EXSB7QQuirmeTtDldFN4fDJQSNDF2xXYAvwJeCC8/idgcvi8PkHCOqZMDHcBL4fPOwAO/DJC7KOBr8P3fwP4I9Ar5vqJBImnRwX1Lwnfq1vMue+GnzktPH4T+F2ZeqPC9zTgkPA1hsRcbw/sBm6IOefA2WVe53PgV/tw7MAtMccZwFbggvD4R8CXQHZMmf8J6w1L9r81PeL7UEtFUtkIMys0s+3A2wRfpD8h6F7KBGaUFHT33WGZnuGpaQR/7Q8gaJ1MIxijGRZeH0bQmiGsU5egpVNY8iAYu+lcJqZ3qwra3e8FWhF8cU4HRgIfmNmFYZG+wGp3X1DJy+xw94Uxx6vCz5wTHvcHri8T7xMECbIV0IMgcZW0xHD3peHrVIePYt5nF0GLrUV4qjswz923xZR/p5rikCTTQL2ksjeBywi6qVZ5OCge08VV3hbbwZ/O7oVmNoegC6wX8DpB0mlvZl0Jks01YZ2SP65OB5aVeb2iMsdfRwnc3bcAE4AJZvZbYBJBi+WfBC2Jquwq+5JlYk0DbgT+XU7d9RHfo+R1y5bNLK9gFcr+npxvYjXK/28lByElFUllW929oJzzBQRdQUcDSwDMLJ1g7OGJmHJTCZJKD+Aud99uZu8A17PneMrHwA6gvbtPifeHcHc3s0+AfuGpOUBrM+tRRWulMnOA7hX8fjCzBQRf6gOAt8Jz7YC8MkXXA61j6rWMPY6TBcBFZpYd01oZGOf3kBShpCI1jrt/bWb3A7ea2QbgM+DnQEv2XD8xFfglQetiTsy564HXS1o+7r7FzP4M/DkcRH8TaAAMBordfWzU2MzscIIWxD8JktVOggH57wFPhsVeI+j+ecbMfg4sIhhQr+/uz0d8qz8AL5rZUmA8QcumNzDQ3a9x94VmNpFgssFlBGNGd4Y/Y00BRpvZWwTjLTcD26N+3ogeJ5gI8aCZ3UyQ2H4TXlML5iCjMRWpqX5N8GX6d+ADoA8wwt1Xx5SZRvClNS0cc4GgGyydb8ZTSvwOuAH4FTCfYK3JWQQJa1+sIGg9/R6YGcb2S+DPBONBuHsxwUSCGcBjBH/J3w1k7f1y5XP3ScCpBC2xWeHjWvbsvrskjH8K8B+CVtznZV7ql2G8U4GnCdYCrYsaR8RYCwm6FnsB7xPM/LohvBzvBCZJZu76Q0FEEsvMRgLPAS3cfUOy45H4UfeXiFQ7M7uYoEW0nKCb7i7gP0ooB5+EdX+Z2QgzW2hmBWZ2bTnXzczuCa9/ZGb9Yq6NM7N1ZjavTJ2mZjbZzBaHP5sk4rOIyD5rSTDOtBC4l2Dx5wWV1pAaKSHdX+HMnEUEi75WALOB893945gypxD0OZ8CDALudvdB4bVjCRZ1PeruvWPq3A586e63homqibuXbs0hIiKJlaiWykCgwN2XuPtO4CmCBWGxRhIkDXf3mUBOyXoEd3+TYEVuWSOBR8LnjxDuCyUiIsmRqDGVfIK+1BIrCFojVZXJB1ZTsZYls33cfbWZtSivUDil8jKAtOxG/TMaf1OsQ6PkT4ArLi4mLS35ccRKxZggNeNSTNEopuhSMa5FixZtcPfcqsolKqmUt7q3bL9blDL7JVxnMBagTuuu3vriuwDIz6nLjGuPj8dbHJCpU6cybNiwZIexh1SMCVIzLsUUjWKKLhXjCtdEVSlRqXAF0DbmuA1770EUpUxZa0u6yMKf+zS/vnXjuuzYtbvqgiIiEkmikspsoKsFd8TLAs4j2Bcp1gSCrRzMzAYDm8ssZCvPBODi8PnFBFtvVyk/py4jerXi3aWbuOChd/jy651VVxIRkSolJKmEu5ZeSbCp3gJgvLvPN7PLzezysNjLBPPYC4AHgR+X1DezJwk2A+xmwS1avx9euhU40cwWE8wsu7WqWDo0SmPGtccz5sL+3HN+Xz5csZlR986gYN2WOH1aEZHaK2GLH939ZYLEEXtuTMxzJ7gPRXl1z6/g/BfAfg+KnHFYHm2aZHPZo+9y5n1vcd93+3FM1yrHoUREpAKpNb0gCfq1a8Lzo4eQn5PNJX+fzT9nRhqLEhGRctT6pALQpkk9nr7iKIYeksvvnp/HDRPms2t3cbLDEhGpcZRUQg3qZPDgRUfw/aM78o+3PucHj77Llu1l7zskIiKVUVKJkZ5m/O60nvzpzN5MW7yBs+9/m+Vfbk12WCIiNYaSSjm+O6g9j1w6kNWbt3HmfTN4b+nGZIckIlIjKKlU4OiuzXn2x0OoXyeD8x+cyQsfrEx2SCIiKU9JpRJdWjTg+R8P4fC2OVz11AfcOXkRuqmZiEjFlFSq0KR+Fo99fxDn9G/DPa8t5idPvs/2Im3tIiJSHt35MYKsjDRuP7sPnVs04LaJn7Bi4zbGXtSfFg3rJjs0EZGUopZKRGbG5UM7c/93+7NwzRbOvPctFqz+KtlhiYikFCWVfTSidyv+ffmR7Cou5uz732LKJ2uTHZKISMpQUtkPvfMb88Loo+mYW58fPPIuD01bogF8ERGUVPZbq8Z1Gf+jI/lWz1bc9NICfvPcPIq0tYuI1HJKKgegXlYG9323Hz8e1pknZy3jkr/PYvNWbe0iIrWXksoBSkszrhnRnT+fcxizPvuSM++fwecbvk52WCIiSaGkEidn92/D4z8YzMavdzLqvhnMXPJFskMSEUk4JZU4GtixKc+PHkKz+llc+PA7jH93ebJDEhFJKCWVOGvfrD7P/ngIgzs145qnP+KWVxZQXKyZYSJSOyipVIPG2ZmMu2QA3x3UjgfeWMLlj73H1p27kh2WiEi1U1KpJpnpadw0qjf/e3pP/rtgLeeMeZs1m7cnOywRkWqlpFKNzIxLh3Tk4YsHsPSLrYy8dzpzV2xOdlgiItVGSSUBhndvwdNXHElGWhrnPPAWE+etTnZIIiLVQkklQbq3asTzo4fQo3UjLn9sDve+XqCtXUTkoKOt7xMot2EdnvzhYK55+iPumLSQqZ+sY+WmbazavJ38mVO4+qRujOqbn+wwRUT2m5JKgtXNTOfu8w5n565iJs5fU3p+5aZtXPfsXAAlFhGpsdT9lQRmxtyVew/YbyvazR2TFiYhIhGR+FBSSZJVm7bt03kRkZpASSVJ8nKyK7z2xDvLtApfRGokJZUkufqkbmRnpu9xrk5GGp2a1+c3z83lO2PfpmDdliRFJyKyf5RUkmRU33xu+fah5IctlvycbG47qw///eVQbj+rD4vWFnLy3dO4c/IithftTnK0IiLRaPZXEo3qm8+ovvlMnTqVYcOGlZ4/d0BbjuvRgpte/Jh7XlvMix+t4uYzD2Vwp2bJC1ZEJAK1VFJU8wZ1uOu8vjzyvYEU7S7mvLEzuebpD9m0dWeyQxMRqZCSSoobekgur/5sKD8a2oln5qzkhDvf4IUPVmo1voikJCWVGiA7K53rTu7Bf648mvycbK566gMu/vtsln+5NdmhiYjsQUmlBumZ14hnfzyE/z29J+99/iUn/uUNHnjjU3btLk52aCIiQAKTipmNMLOFZlZgZteWc93M7J7w+kdm1q+qumZ2uJnNNLMPzOxdMxuYqM+TLOlpwXb6k38xlKO75HLLK59wxt9m8OHyTckOTUQkMUnFzNKBe4GTgZ7A+WbWs0yxk4Gu4eMy4P4IdW8HbnT3w4Hfh8e1Ql5ONg9e1J8xF/Tji693cOZ9M7hhwnwKd+gOkyKSPIlqqQwECtx9ibvvBJ4CRpYpMxJ41AMzgRwza11FXQcahc8bA6uq+4OkEjNjRO/WTP7FUL47qD2PvP05J975BpM/Xpvs0ESklrJEzCIys7OBEe7+g/D4QmCQu18ZU+ZF4FZ3nx4evwb8GuhQUV0z6wFMAowgQR7l7kvLef/LCFo/5Obm9h8/fny1fdb9UVhYSIMGDQ74dQo27uYf83ewotA5omU63+2RRZO6+/d3Q7xiirdUjEsxRaOYokvFuIYPH/6eux9RVblELX60cs6VzWYVlams7hXAz939GTM7F3gYOGGvwu5jgbEA3bp189iFhqmg7OLH/TUMuOj0Yh6ctoS7X1vMwreLuObk7nx3YDvS0sr7NVZ/TPGWinEppmgUU3SpGlcUier+WgG0jTluw95dVRWVqazuxcCz4fN/E3SV1WpZGWmMHt6FV392LH3aNuZ3z8/jnAfeZuEa7SMmItUvUUllNtDVzDqaWRZwHjChTJkJwEXhLLDBwGZ3X11F3VXA0PD5ccDi6v4gNUWH5vV57PuD+L9zDmPJ+kJOvWcad0z6RPuIiUi1Skj3l7vvMrMrCcY/0oFx7j7fzC4Pr48BXgZOAQqArcClldUNX/qHwN1mlgFsJxw3kYCZcVb/Ngzv3oKbXvqYe1//lJc+Ws3NZx7KUV2aJzs8ETkIJWxDSXd/mSBxxJ4bE/PcgdFR64bnpwP94xvpwadp/SzuPPdwvt23Ddc/P5f/eegdzurXhutP7UHT+lnJDk9EDiJaUV+LHN21OZN+diyjh3fmhQ9Wcvz/TeXZOSu0j5iIxI2SSi1TNzOdq0/qzos/PZoOzevzi/EfcuHDs1j6xdfJDk1EDgK6n0ot1b1VI565/Cgef2cpt01cyLf+8iY/Pb4rrRrV4c7Ji1m5aRv5M6dw9UndGNU3P9nhikgNoaRSi6WlGRce2YETe7bihgnzuWPSQoxvFgGt3LSN656dC6DEIiKRqPtLaNW4LmMu7E/T+ll7rUjdVrSbOyYtTEpcIlLzKKlIqY1fl39XyVWbtiU4EhGpqZRUpFReTna55x24/rm5rNiom4KJSOWUVKTU1Sd1IzszfY9zdTLSOLJzU8a/u5xhd0zlmqc/1EwxEamQBuqlVMlg/B2TFgazv3KyS2d/rdq0jQfe+JQnZy/nmTkrGXlYHqOP60Ln3NTaSVVEkktJRfYwqm8+o/rm77VLal5ONjeO7M3o4V0Y++YSHntnKc99sJLT+uRx5fAudGvVMHlBi0jKUPeX7JMWjery29N6Mv3Xx/GjYzszZcFaTrrrTS7/53vMX7U52eGJSJKppSL7pXmDOlx7cnd+dGwnxs34jH/M+JyJ89dwQo8W/OS4rhzWNifZIYpIEqilIgekSf0sfvmtbky/9jh+ceIhzP58IyPvncHF42bx3tIvkx2eiCSYkorERePsTH56fFem/3o414zoxtyVmznr/rf5nwdnMnPJF8kOT0QSRElF4qph3Ux+PKwL0389nN+e2oNFaws5b+xMzh3zNtMWr9eOyCIHOSUVqRb1sjL4wTGdmP7r4dxwek+WfbmVCx+exZn3vcXrn6xTchE5SCmpSLWqm5nOJUM68sY1w7hpVG/Wb9nBpf+YzRl/m8Gk+WsoLlZyETmYKKlIQtTJSOeCwe2ZevUwbj+rD5u3FfGjf77HKfdM46WPViu5iBwklFQkoTLT0zh3QFum/HIod557GDt3FzP6iTl86643eeGDlexWchGp0fYpqZhZmpm1rq5gpPbISE/j2/3aMPnnQ/nr+X1JM7jqqQ844c43+Pe7yynaXZzsEEVkP0RKKmaWY2ZPANuBgvDcGWZ2U3UGJwe/9DTj9MPymHjVsYy5oB/Zmelc/fRHHPd/U3ly1jJ27gqSy/Pvr2TIrVO4ZOLXDLl1Cs+/vzLJkYtIeaKuqB8DbATaAx+H594G/g/4bTXEJbVMWpoxondrTurVitcWrOOvUxZz3bNz+etrizmqczNenLua7UVBgtEdKUVSV9SkcjyQ5+5FZuYA7r7ezFpUX2hSG5kZJ/RsyfE9WvDm4g3c89pinp6zd6uk5I6USioiqSXqmMpmoHnsCTNrB6yOe0QiBMll6CG5PH35kRWW0R0pRVJP1KTyEPCMmQ0H0szsSOARgm4xkWpjZuRXcEfK9DTjoWlL+KJwR4KjEpGKRE0qtwHjgXuBTGAc8AJwdzXFJVKqvDtSZqYb+Tl1uemlBQy+5TVGPz6HNxet13oXkSSLOqbS0t3vAu6KPWlmrYA1cY5JZA+V3ZFy0dot/Gv2cp6ds4KX5q4mPyebc49oyzlHtCGvghaOiFSfqEllEdConPMfA03jF45I+Sq6I+UhLRvyu9N6cs2Ibkz+eC3/mr2cv/x3EXe/toihh+TynQHtOL5HCzLTtc5XJBGiJhXb64RZI0Ar1CQl1MlI57Q+eZzWJ4/lX25l/LvLGf/uci5/7D2aN8jirP5t+M4RbemU2yDZoYoc1CpNKma2HHAg28yWlbncDHiyugIT2V9tm9bjl9/qxlXHd+XNxet5atZyHpr2GQ+8sYSBHZty3oC2nNy7NdlZ6VW/mIjsk6paKhcQtFJeBi6MOe/AWndfWF2BiRyojPQ0juvekuO6t2Tdlu08895K/jV7Gb8Y/yH/O2E+ow7P5zsD2tI7v3GyQxU5aFSaVNz9DQAza+7uWxMTkkj8tWhYlyuGdebyoZ1457Mv+dfsoHvsnzOX0ju/EecNaMcZh+fRqG5mskMVqdEijam4+1YzOxw4hmARpMVc+331hCYSf2bG4E7NGNypGTec3osXPlzJk7OW89vn53HTSx9z6qF5nDewLUe0b4LZXkOJIlKFSEnFzC4D/gK8CpwMvAJ8i2CtikiN1LheJhcd2YELB7dn7srNPDV7ORM+WMUzc1bQKbc+5w1oy7f7taF5gzrJDlWkxog6z/IaYIS7nwlsC3+eDRRFfSMzG2FmC82swMyuLee6mdk94fWPzKxflLpm9pPw2nwzuz1qPCIlzIw+bXK4+cxDmXX98dxxdh+a1svi5pc/YfDNr3HFY+8xdeE63etFJIKoU4pbuPu08HmxmaW5+ytm9niUymaWTrAa/0RgBTDbzCa4+8cxxU4GuoaPQcD9wKDK6obbxowE+rj7Dm1wKQeqXlYG5xzRlnOOaEvBumBh5TNzVvLKvDXkNa4bXmtDmyb1eP79ld8syJw5pXRBpkhtFjWprDCzDu7+OcFCyJFmtgHYGbH+QKDA3ZcAmNlTBMkgNqmMBB51dwdmhvdwaQ10qKTuFcCt7r4DwN3XRYxHpEpdWjTk+lN7cvVJ3fnvgrU8NXs590xZzD1TFtOtZUOWrC9k5+6g9aLt+EUCUZPK7UAP4HPgD8DTQBZwVcT6+cDymOMVBK2RqsrkV1H3EOAYM/sTwQ3EfuXus8u+eTgmdBlAbm4uU6dOjRh2YhQWFiqmiJIVVz3ge53gjNbZTFuxi/98umWvlb/binbzxxc+JGfz4oTHV1Yq/vdTTNGlalxRRJ399Y+Y56+YWRMgy90LI75PedNoynZQV1SmsroZQBNgMDAAGG9mncLWzjeF3ccCYwG6devmsdt8pIKyW4+kglSMCVIjrrOBCde+VO61L7Y7RS16cEzX5tTNTN7iylT4PZWlmKJL1biiiNpS2YO77zSzk8zsd+4+MEKVFUDbmOM2wKqIZbIqqbsCeDZMIrPMrJhgyvP6yB9GZD/k5WSzspz7uRjww0ffpV5WOsO7teCk3q0Y3i2Xhlr/IrVEVdu0tAT+DBwOLAZ+SvClfh/QArgn4vvMBrqaWUdgJXAe8D9lykwArgzHTAYBm919tZmtr6Tu88BxwFQzO4QgAW2IGJPIfrv6pG5c9+xcthXtLj2XnZnOH0f2okWjukyav4ZJ89fy0tzVZKWnMaRLM0b0bsUJPVrSTFOU5SBWVUvlPqBB+PPbBF/8TYA/AY+4e6Qpxe6+y8yuBCYB6cA4d59vZpeH18cQbAVzClAAbAUuraxu+NLjgHFmNo9g0sDFZbu+RKpDZdvxAxx7SC5/GNmb95dtZOK8NUycv4bXn5lLms1lQIemjOjdipN6tdL2/HLQqSqpHAN0cfevzGw8QbfSoTFf6pG5+8sEiSP23JiY5w6Mjlo3PL+TYH8ykYSraDv+EulpxhEdmnJEh6Zcf2oPPl79FZPCBHPjfz7mxv98zGFtGnNSmGA6awdlOQhUlVTquvtXAO7+hZlt3p+EIlLbmRm98hrTK68xv/hWN5asL2TS/LVMnL+G2ycu5PaJC+naokFpC6ZXXiNtEyM1UlVJJdPMLuWbGViZZva92ALuPq5aIhM5iHXKbcAVwxpwxbDOrNq0jVfDMZh7Xy/gr1MKyM/JZkTvVozo3Yp+7ZqQnqYEIzVDVUnlHeCimOPZ7L0FvpKKyAHIy8nmkiEduWRIR74o3MFrC9Yxcf4a/vn2Uh6e/hnNG2RxYs8gwRzZqRlZGbqLpaSuqra+H5agOEQEaNagDucOaMu5A9qyZXsRUxeuZ+L8NbzwwUqenLWMhnUzOKFHS07q1ZJjD8mlXtZ+rQoQqTb6FymSohrWzeT0w/I4/bA8thftZvriDUyav4bJC9by3PsrqZuZxtBDcjmpVyuO796SxvUytR+ZJJ2SikgNUDcznRN6tuSEni3ZtbuYWZ99ycT5a0rXw2SkGZ1z67Nkw9cUaT8ySSIlFZEaJiM9jaO6NOeoLs254fRefLhiExPnr+GhaZ/ttT3/tqLd3D7xEyUVSRiN+InUYGlpRt92Tbju5B4UV3C/l1Wbt3PRuFmMffNT5q/aXGE5kXiIeufHThVc2gGsdveyG7aKSIJVtB9Z/TrprNq0jZtf/gSApvWzOKpzM47u0pwhXZrTtmm9RIcqB7Go3V8FfLMzsLHnDsPFZjYB+LG7r41ncCISXUX7kf1p1KGM6pvPms3bmVGwgRkFG5hesIEXP1oNQPtm9RjSpTnHdGnOkZ2bkVMvK1kfQQ4CUZPKD4GhwI0E9zZpB/wOeAt4A7iN4O6MZ1dDjCISQVX7kbVqXJez+rfhrP5tcHcK1hUyPUwyEz5YxRPvLMMMDs1vzJAuzTm6S3P6t2+S1C38peaJmlRuJNgDbHt4XGBmVwCL3P0BM7uEYBdjEUmiqvYjK2FmdG3ZkK4tG3LpkI4U7S7moxWbmL74C2YUbODBN5dw/9RPqZORxoAOTUuTTM+8RlrdL5WKmlTSCG7r+0nMuXYEuwYDFO7Da4lIislMT6N/+6b0b9+Uq07oSuGOXcz+7EumLQ5aMrdN/ITbgJx6mRzVuVlpkmnfrH6yQ5cUEzUR3AVMMbO/E3R/tSHYmv6u8PqpwNvxDk5EkqNBnQyGd2/B8O4tAFi3ZTtvFXxR2l328tw1ALRtml064H9U5+Y0ra/xmNou6u2Ebzezj4BzgH7AauD77j4xvP48wQ2zROQg1KJh3dKuNXdnyYavgwH/xcGA/5OzlgPQK69RaZIZ0KEp2VnpWuVfy0TusgoTyMRqjEVEagAzo3NuAzrnNuCiIzuwa3cxc1duLp1VNm7GZzzw5hKy0tNo1zSbz7/Yyq5irfKvLaKuU8kCLiG4rfAedxJy94vKqSIitURGehp92zWhb7smXHlcV7bu3MXszzcyo2AD46Z/VppQSmwr2s0fX/yY4d1b0Dg7M0lRS3WJ2lJ5BDgM+A+gtSgiUqF6WRkMPSSXoYfk8uCbS8ot88XXOzn8D6/StUUD+rVrQr/2TejXrgmdc+vr5mQ1XNSkMgLo6O6bqjEWETnIVLTKv3mDLC4+sgPvLdvIK/PW8NTsYEwmp14mfdvm0D9MMoe1zaF+HU0srUmi/tdaBtSpzkBE5OBT0Sr/357as3RMpbjYWbKhkDlLN/He0o3MWbaR1xeuByDNoHurRkGSaZ9D/3ZNads0W62ZFBY1qTwKvGBmd1Om+8vdp8Q9KhE5KFS1yh+CTTG7tGhIlxYNOXdAWwA2by3i/eUbmbN0I3OWbeLZOSv458ylQNDKKeky69++CYfmN9aq/xQSNalcGf68ucx5ByrabFJEJPIq/1iN62UyrFsLhnUL1snsLnYWrd0StGTC1syrHwd/32amGz3zGtO/Xdiaad+E1o2zq+vjSBWirlPpWN2BiIhUJD3N6NG6ET1aN+KCwe0B2FC4g/eXfdNl9sSspYyb8RkArRvXLR3879++CT1bNyIr45s7fWjtTPXRCJiI1EjNG9ThxJ4tObFnSwCKdhezYPVXYZLZxJylG3kp3Im5TkYafdo0pl+7JhTtLuaJd5axfVdwxw6tnYmvCpOKmS1w9x7h8+Xsud19KXdvV02xiYhElpmeRp82OfRpk8OlQ4JzazZvZ86yoMvsvWUb+fuMz9m5e+/bP20r2s1tukNmXFTWUvlhzPMLqjsQEZF4a9W4Lqcc2ppTDm0NwPai3fT43cRy/0JevXk7R982hV55jeid15he+Y3oldeYFg3raLbZPqgwqbj79JjnbyQmHBGR6lM3M73CtTON6mZwWNscPl71FZPmfzPJtXmDOvTKaxQ+GtM7vxHtmtZToqmAtmkRkVqlorUzfxjZu7T7a8v2Ihas3sL8VZuZv+or5oV7m5VsOdOwTgY9Slo0eY3old+ILrkNyEhPK/c9axNt0yIitUqUtTMN62YysGNTBnZsWnpue9FuFq8tZN6qzaXJ5olZS9leFIzRZGWk0aNVQ3qWJJq8YLZabVtDo21aRKTW2Z+1M3Uz0zm0TWMObdO49NzuYmfJ+kLmr/qqNNG89NEqnpy1DAimQnfOrU+v0kTTmJ55jSrcSPNgmOqsbVpERPZTeto3t2Uu+fJ3d1Zs3LZHonnr0w089/7K0nptm2Z/03UW/nzr0y/26JarqVOdtU2LiEgcmRltm9ajbdN6jOjdqvT8+i07SpPMx2HCeWXemtLraQZl7hLAtqLd3DFp4UGZVLRNi4jIAchtWGePrWcAvtpexIJVXzF/1Vf84cWPy623ctM2fvTPd+naoiFdWzbgkJYN6di8fsqO1VSZVMwsDfgBMN3dd1R/SCIitUOjupkM6tSMQZ2a8fD0z8qd6lw3M43F6wr574J17A6bMmkGHZrVp2vLBimXbKpMKu5ebGbPu3vDRAQkIlIbVTTV+ZZvH8qovvns2LWbzzZ8zaK1hRSs3cKitYUsWrcl5ZJN1O6vN81ssLvP3N83MrMRwN1AOvCQu99a5rqF108BtgKXuPuciHV/BdwB5Lr7hv2NUUQkWaqa6lwnI53urRrRvVWjPepVd7IpmZGW1apL/yifI2pSWQq8YmYvAHvsA+buv6+qspmlA/cCJwIrgNlmNsHdYzsRTwa6ho9BwP3AoKrqmlnb8NqyiJ9FRCQl7c9U5+pMNs+/v3Kv1lNVoiaVbOD58HmbyK/+jYFAgbsvATCzp4CRQGxSGQk86u4OzDSzHDNrDXSoou5fgGuAF/YjLhGRg1JVyWbx2kIWh8lmcQXJZtXmbaWLO6OKej+VS/fpVfeWT9DCKbGCoDVSVZn8yuqa2RnASnf/sLJ9eMzsMuAygNzcXKZOnbpfH6K6FBYWKqaIUjEuxRSNYoouEXE1BPplQb+2QFsoKs5mzdfOqsJiVhYWs7JwO0v2MaHAPt5PxcwaAs2B0m/wkhZEVVXLOVd2o9CKypR73szqAdcD36rqzd19LDAWoFu3bh61WZko+9LUTZRUjAlSMy7FFI1iii5V4hpy65RyZ6RVJtLuZ2bW08zeBzYDBeFjcfiIYgXQNua4DbAqYpmKzncGOgIfmtnn4fk5ZtYKERE5YFef1I3sfZw1FnVLzfuA14GmwFdAE+AB4OKI9WcDXc2sY7jj8XnAhDJlJgAXWWAwsNndV1dU193nunsLd+/g7h0Ikk8/d1+DiIgcsFF987nl24eSn5MduU7U7q/DgBPdvcjMzN03m9nVwDzgsaoqu/suM7sSmEQwLXicu883s8vD62OAlwmmExcQTCm+tLK6kT+hiIjst5IZaXZdwXtRykdNKtuBTKAI2GBm7YCNQLOogbn7ywSJI/bcmJjnDoyOWrecMh2ixiIiItUjavfXNODc8PnTwCvAG4A2kxQRkVJRpxSfG3P4G4Jur4YEuxeLiIgA+z6lOA1o6e5VjqOIiEjtE3VKcRMze4JgbKUgPHeGmd1UncGJiEjNEnVM5X6CNSrtgZ3hubeB71RHUCIiUjNF7f46HsgLpxQ7gLuvN7MWVdQTEZFaJGpLZTPB9iylwmnFq+MekYiI1FhRk8pDwDNmNhxIM7MjgUcIVtWLiIgA0bu/biMYpL+XYBHkOOABd7+rmuISEZEaKFJLxQN3uXtPd6/v7j2Av5rZH6o5PhERqUGidn+VJ4Ng63kRERHgwJIKlH+vExERqaUONKmUvdGWiIjUYpUO1JvZcZVczopzLCIiUsNVNfvr4SquL4tXICIiUvNVmlTcvWOiAhERkZrvQMdURERESimpiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CQsqZjZCDNbaGYFZnZtOdfNzO4Jr39kZv2qqmtmd5jZJ2H558wsJ0EfR0REypGQpGJm6cC9wMlAT+B8M+tZptjJQNfwcRlwf4S6k4He7t4HWARcV80fRUREKpGolspAoMDdl7j7TuApYGSZMiOBRz0wE8gxs9aV1XX3V919V1h/JtAmER9GRETKl5Gg98kHlsccrwAGRSiTH7EuwPeAf5X35mZ2GUHrh9zcXKZOnboPoVe/wsJCxRRRKsalmKJRTNGlalxRJCqpWDnnPGKZKuua2fXALuDx8t7c3ccCYwG6devmw4YNqyLcxJo6dSqKKZpUjEsxRaOYokvVuKJIVFJZAbSNOW4DrIpYJquyumZ2MXAacLy7l01UIiKSQIkaU5kNdDWzjmaWBZwHTChTZgJwUTgLbDCw2d1XV1bXzEYAvwbOcPetCfosIiJSgYS0VNx9l5ldCUwC0oFx7j7fzC4Pr48BXgZOAQqArcClldUNX/pvQB1gspkBzHT3yxPxmUREZG+J6v7C3V8mSByx58bEPHdgdNS64fkucQ5TREQOgFbUi4hI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CipiIhI3CQsqZjZCDNbaGYFZnZtOdfNzO4Jr39kZv2qqmtmTc1sspktDn82SdTnERGRvSUkqZhZOnAvcDLQEzjfzHqWKXYy0DV8XAbcH6HutcBr7t4VeC08FhGRJElUS2UgUODuS9x9J/AUMLJMmZHAox6YCeSYWesq6o4EHgmfPwKMqubPISIilchI0PvkA8tjjlcAgyKUya+ibkt3Xw3g7qvNrEV5b25mlxG0fgB2mNm8/fkQ1ag5sCHZQZSRijFBasalmKJRTNGlYlzdohRKVFKxcs55xDJR6lbK3ccCYwHM7F13P2Jf6lc3xRRdKsalmKJRTNGlYlxm9m6Uconq/loBtI05bgOsilimsrprwy4ywp/r4hiziIjso0QlldlAVzPraGZZwHnAhDJlJgAXhbPABgObw66tyupOAC4On18MvFDdH0RERCqWkO4vd99lZlcCk4B0YJy7zzezy8PrY4CXgVOAAmArcGlldcOXvhUYb2bfB5YB50QIZ2z8PlncKKboUjEuxRSNYoouFeOKFJO579PwhIiISIW0ol5EROJGSUVEROKm1iSVqraJSQYzG2dm61Jp3YyZtTWz181sgZnNN7OrUiCmumY2y8w+DGO6MdkxlTCzdDN738xeTHYsJczsczOba2YfRJ0GWt3MLMfMnjazT8J/W0cmOZ5u4e+n5PGVmf0smTGFcf08/Dc+z8yeNLO6KRDTVWE886P8jmrFmEq41csi4ESCKcqzgfPd/eMkx3UsUEiwk0DvZMZSIpya3drd55hZQ+A9YFQyf1dmZkB9dy80s0xgOnBVuPNCUpnZL4AjgEbuflqy44EgqQBHuHvKLJ4zs0eAae7+UDiLs567b0pyWEDp98NKYJC7L01iHPkE/7Z7uvs2MxsPvOzu/0hiTL0JdjEZCOwEJgJXuPviiurUlpZKlG1iEs7d3wS+THYcsdx9tbvPCZ9vARYQ7GqQzJjc3QvDw8zwkfS/hsysDXAq8FCyY0llZtYIOBZ4GMDdd6ZKQgkdD3yazIQSIwPINrMMoB57r+dLtB7ATHff6u67gDeAMyurUFuSSkVbwEglzKwD0Bd4J8mhlHQzfUCwwHWyuyc9JuAu4BqgOMlxlOXAq2b2XrhFUbJ1AtYDfw+7Ch8ys/rJDirGecCTyQ7C3VcCfyZYHrGaYK3eq8mNinnAsWbWzMzqESz7aFtZhdqSVA54q5faxswaAM8AP3P3r5Idj7vvdvfDCXZUGBg2y5PGzE4D1rn7e8mMowJD3L0fwc7eo8Nu1mTKAPoB97t7X+BrUmRH8bAr7gzg3ykQSxOCHpSOQB5Q38wuSGZM7r4AuA2YTND19SGwq7I6tSWpRNkmRkLhuMUzwOPu/myy44kVdptMBUYkNxKGAGeE4xdPAceZ2WPJDSng7qvCn+uA5wi6f5NpBbAipnX5NEGSSQUnA3PcfW2yAwFOAD5z9/XuXgQ8CxyV5Jhw94fdvZ+7H0vQXV/heArUnqQSZZsYoXRQ/GFggbvfmex4AMws18xywufZBP/zfZLMmNz9Ondv4+4dCP49TXH3pP5VCWBm9cMJFoRdTN8i6MJIGndfAyw3s5Jdbo8HkjpJJsb5pEDXV2gZMNjM6oX/Hx5PMKaZVBbu/m5m7YBvU8XvK1G7FCdVFVu9JI2ZPQkMA5qb2Qrgf9394eRGxRDgQmBuOIYB8Bt3fzl5IdEaeCScpZMGjHf3lJnCm2JaAs8F30lkAE+4+8TkhgTAT4DHwz/qlhBuw5RM4RjBicCPkh0LgLu/Y2ZPA3MIupjeJzW2a3nGzJoBRcBod99YWeFaMaVYREQSo7Z0f4mISAIoqYiISNwoqYiISNwoqYiISNwoqYiISNwoqYjUQGbmZtYl2XGIlKWkIhIH4Xbz28ysMObxt2THJZJotWLxo0iCnO7u/012ECLJpJaKSDUys0vMbIaZ/dXMNoc3qTo+5nqemU0wsy/DG8j9MOZaupn9xsw+NbMt4a7DsXvYnWBmi81so5ndG27tIZJUaqmIVL9BBJsoNifYO+lZM+vo7l8S7KM0n2BX2u7AZDNb4u6vAb8g2JvqFIKbzPUBtsa87mnAAKARwc3U/kOwk6xI0mibFpE4CHcrbs6e24JfTbBf0s1Avof/s5nZLOCvBLstfw7khDdEw8xuIbjz5iVmthC4xt1fKOf9HDjG3aeHx+MJdtu9tVo+oEhE6v4SiZ9R7p4T83gwPL/S9/zrbSlByyQP+LIkocRcK7mBXFvg00reb03M861AgwMLX+TAKamIVL/8MuMd7Qju57MKaFqyVX3MtZXh8+VA58SEKBIfSioi1a8F8FMzyzSzcwju+/2yuy8H3gJuMbO6ZtYH+D7weFjvIeCPZtbVAn3CLchFUpYG6kXi5z9mtjvmeDLwAvAO0BXYAKwFznb3L8Iy5wNjCFotGwnuqTM5vHYnUAd4lWC85hPgzOr+ECIHQgP1ItXIzC4BfuDuRyc7FpFEUPeXiIjEjZKKiIjEjbq/REQkbtRSERGRuFFSERGRuFFSERGRuFFSERGRuFFSERGRuPl/yvu9VhEx35MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### Viusalizing what learning_rate values\n",
    "\n",
    "### you can test what power method does in learning rate\n",
    "n_epochs = 10 ## in the above i try 10 epochs\n",
    "learning_rate = 0.01\n",
    "decay = 1e-4\n",
    "batch_size = 32\n",
    "n_steps_per_epoch = np.ceil(len(X_train) // batch_size)\n",
    "epochs = np.arange(n_epochs)\n",
    "## each step learning rate is updated\n",
    "lrs = learning_rate / (1 + decay * epochs * n_steps_per_epoch)\n",
    "\n",
    "plt.plot(epochs, lrs,  \"o-\")\n",
    "plt.axis([0, n_epochs - 1, 0, 0.01])\n",
    "plt.xlabel(\"Epoch\",  fontsize=12)\n",
    "plt.ylabel(\"Learning Rate\", fontsize=12)\n",
    "plt.title(\"Power Scheduling\", fontsize=14)\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66e2df13",
   "metadata": {},
   "source": [
    "#### `Exponential Scheduling`\n",
    "\n",
    "$$Lr^{t} = Lr_0 * 0.1^{t/s}$$\n",
    "\n",
    "#### you can emplement you custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c1cc3652",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "1688/1688 [==============================] - 12s 7ms/step - loss: 0.8170 - accuracy: 0.6998 - val_loss: 0.5928 - val_accuracy: 0.7867\n",
      "Epoch 2/4\n",
      "1688/1688 [==============================] - 10s 6ms/step - loss: 0.6095 - accuracy: 0.7838 - val_loss: 0.5943 - val_accuracy: 0.7917\n",
      "Epoch 3/4\n",
      "1688/1688 [==============================] - 11s 6ms/step - loss: 0.5625 - accuracy: 0.8021 - val_loss: 0.6463 - val_accuracy: 0.7693\n",
      "Epoch 4/4\n",
      "1688/1688 [==============================] - 11s 6ms/step - loss: 0.5330 - accuracy: 0.8126 - val_loss: 0.4883 - val_accuracy: 0.8353\n"
     ]
    }
   ],
   "source": [
    "## le that initail LR = 0.01\n",
    "## and i will reduce learing rate each 20 step\n",
    "\n",
    "def exponential_LR_fn(epoch):\n",
    "    ''' this function is to get the learning rate at each epoch\n",
    "    * initial learning rate = 0.01\n",
    "    * reducing the learning rate each 20 step, each 20 batch_size\n",
    "    \n",
    "    '''\n",
    "    lr_epoch = 0.01 * 0.1**(epoch/20)\n",
    "    \n",
    "    return lr_epoch\n",
    "\n",
    "## you can make LearningRateSchedular, and pass this to callbaks when fitting the Model\n",
    "\n",
    "model_fashion.compile(loss='sparse_categorical_crossentropy', \n",
    "                      optimizer='rmsprop', metrics=['accuracy'])\n",
    "\n",
    "## pass this function to LearningRateScheduler\n",
    "lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_LR_fn)\n",
    "\n",
    "## pass above  'lr_scheduler' to callabaks in fit\n",
    "## ==> read hands-on-ML for this Note\n",
    "history = model_fashion.fit(X_train, y_train, \n",
    "                            validation_data=(X_val, y_val), epochs=4, \n",
    "                            batch_size=32, callbacks=[early_stopp, lr_scheduler])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19536c82",
   "metadata": {},
   "source": [
    "#### you can make schedular for exponential instead of your function\n",
    "\n",
    "`ExponentialDecay` return the below line\n",
    "```python\n",
    "return initial_learning_rate * decay_rate ^ (step / decay_steps)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "43da4498",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.4920 - accuracy: 0.8243 - val_loss: 0.4298 - val_accuracy: 0.8457\n",
      "Epoch 2/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3936 - accuracy: 0.8569 - val_loss: 0.3809 - val_accuracy: 0.8598\n",
      "Epoch 3/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3580 - accuracy: 0.8698 - val_loss: 0.3594 - val_accuracy: 0.8660\n",
      "Epoch 4/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3338 - accuracy: 0.8784 - val_loss: 0.3296 - val_accuracy: 0.8813\n",
      "Epoch 5/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.3137 - accuracy: 0.8855 - val_loss: 0.3201 - val_accuracy: 0.8778\n",
      "Epoch 6/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2979 - accuracy: 0.8908 - val_loss: 0.3149 - val_accuracy: 0.8763\n",
      "Epoch 7/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2848 - accuracy: 0.8949 - val_loss: 0.3249 - val_accuracy: 0.8782\n",
      "Epoch 8/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2737 - accuracy: 0.8997 - val_loss: 0.3032 - val_accuracy: 0.8883\n",
      "Epoch 9/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2626 - accuracy: 0.9033 - val_loss: 0.3004 - val_accuracy: 0.8877\n",
      "Epoch 10/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2547 - accuracy: 0.9065 - val_loss: 0.2930 - val_accuracy: 0.8928\n",
      "Epoch 11/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2455 - accuracy: 0.9098 - val_loss: 0.2951 - val_accuracy: 0.8940\n",
      "Epoch 12/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2383 - accuracy: 0.9121 - val_loss: 0.2802 - val_accuracy: 0.8968\n",
      "Epoch 13/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2320 - accuracy: 0.9146 - val_loss: 0.2743 - val_accuracy: 0.8970\n",
      "Epoch 14/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2251 - accuracy: 0.9167 - val_loss: 0.2717 - val_accuracy: 0.8980\n",
      "Epoch 15/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2198 - accuracy: 0.9202 - val_loss: 0.2699 - val_accuracy: 0.9007\n",
      "Epoch 16/20\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.2138 - accuracy: 0.9215 - val_loss: 0.2770 - val_accuracy: 0.8995\n",
      "Epoch 17/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2103 - accuracy: 0.9229 - val_loss: 0.2760 - val_accuracy: 0.8987\n",
      "Epoch 18/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2054 - accuracy: 0.9243 - val_loss: 0.2725 - val_accuracy: 0.9028\n",
      "Epoch 19/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2013 - accuracy: 0.9272 - val_loss: 0.2735 - val_accuracy: 0.9003\n",
      "Epoch 20/20\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1980 - accuracy: 0.9284 - val_loss: 0.2728 - val_accuracy: 0.8997\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "                tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
    "                tf.keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "                tf.keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "                tf.keras.layers.Dense(10, activation=\"softmax\")\n",
    "                            ])\n",
    "\n",
    "## you can try these criteria\n",
    "## => batch size=32 ==> (len(X_train) // 32)=no of batches in 1 epoch\n",
    "## for 20 epoch ===> we get the number of batches in 20 epoch\n",
    "\n",
    "n_epochs = 20\n",
    "s = n_epochs * len(X_train) // 32  \n",
    "lr_schedular = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01, \n",
    "                                                              decay_steps=s, decay_rate=0.1)\n",
    "\n",
    "optimizer_with_lr = tf.keras.optimizers.SGD(learning_rate=lr_schedular, momentum=0.9, nesterov=True)\n",
    "model.compile(loss='sparse_categorical_crossentropy', \n",
    "                      metrics=['accuracy'], \n",
    "                      optimizer=optimizer_with_lr)\n",
    "\n",
    "early_stopp = tf.keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True)\n",
    "history = model.fit(X_train, y_train, \n",
    "                            epochs=n_epochs,\n",
    "                            validation_data=(X_val, y_val), \n",
    "                            batch_size=32, callbacks=[early_stopp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "683c2330",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAEWCAYAAADB1OnUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAApGUlEQVR4nO3de5hkVXnv8e/PGS4qg4iMyNVBHTSoRHECeInRRJQBdcgxF4hGRCOCksS748EoJppgjvHCkSOBhAiCgiaRoKBINODRI+AgiBBFBkQYGGG431Ru7/lj75ai6K6uGbp6d09/P8+zn9qXtXa9e9Xu6rdWrb0rVYUkSZKk7jyi6wAkSZKkuc6kXJIkSeqYSbkkSZLUMZNySZIkqWMm5ZIkSVLHTMolSZKkjpmUS+upJL+d5NKu45ipklyZ5CXt/P9M8k89234/ydVJ7kjy7CRPTXJBktuT/EV3Ua8fknw1yf5dxzEbJXl1kq8PWfawJCdM8fO/KMmqKdxfJXnKVJeVZiOTcmkEehO+rlTV/62qp3YZw5ip/kc+zv7PSvJn61q/qv62qnrrfxQ4pKo2qaoLgHcDZ1XVgqo64uHG22+Y86X94PDT9oPCqiQn92x7WMc/3apqaVUdN9X7bc+z+9s26p2eO9XPNR2SLGoT0flj66rqxKp66RTtf8JzStL0mz95EUkzUZJ5VXXfDIgjQKrq/q5jmUJPBC7pWz5posKjfi3aXuU/BV5SVZcneQLwylE93yx3bVVt23UQM91cOqeSzK+qe7uOQ5qMPeXSNEryiCTLk1ye5MYkX0iyec/2Lyb5eZJbk3wrydN7tn0myaeTnJ7kTuDFbQ/rO5Nc1NY5OcnGbfkH9U4PKttuf3eS1UmuTfJng74qbntmP5zkO8BdwJOSHJDkR+0QjyuSvKkt+2jgq8DWPT2XW0/WFn3P99gkX0myJsnN7fy27bYPA78NfKrd96cm2MefJvlZ+1yH9m07LMkJSTZKcgcwD/hBG9s3gRf37H/HCV6LrZP8WxvjT9MzzKXd/xeSHN+2zyVJlrTbPgtsD3y53f+7xwn/t4AzqupygKr6eVUdPej4kzwtyZlJbkpyaZI/6onnM0mOarffnuTsJE9styXJx5Nc354nFyV5xgRt+qAe/vQMl0iycdumNya5Jcn3kmzZbvt1z36S1yX5dpKPtq/tT5Ms7dnnDmn+Fm5P8p9Jjsw6DMlIsnma3uBXtMubJFmZ5LWTtUm7/XntMdzaPj6vZ9tZSf4myXfaul9PskXP9t2T/L+2HX6Q5EVD1v1W+3hL+9o+d6y9eup/Ms1Qq9uSnJ/kt4dskgnPqZ72+pc07wc3Jzmlrz3f0Z4jq5Mc0LN+o/a1vCrJdW2bPrJn+7vywPvM6/v2+aBvfPqPta/s3mmGlN3WHv9hPdvGvmF4Q5KrgG8O2SZSt6rKyclpiifgSpoeqP71bwXOAbYFNgL+Efh8z/bXAwvabZ8ALuzZ9hngVuD5NB+oN26f5zxga2Bz4EfAQW35FwGr+mKaqOyewM+BpwOPAj4LFPCUCY7vLOCqtvx8YANgb+DJQIDfoUnWdxkvlmHaoq/s44BXtbEtAL4InNIXz58NeD12Au4AXtg+18eAe8deI+Aw4ISe8g869v79j/NaPAo4H3g/sCHwJOAK4GU9+/8lsBdNwv93wDmTnS89218D3AS8C1gCzBvn9eiN79HA1cAB7euzC3AD8PSe+G/vaY9PAt9ut72sPZbN2tfyN4CthjnPe9sReBPw5bZt5gHPATbtjxd4HXAP8Ma23MHAtTTfvgB8l2Y40YbAC4Dbel+rvnheRN951rf9pTTn+eOBY4B/7XtNJ2qTzYGbaXqW5wP7tcuP6zmey4EdgUe2y4e327YBbmxf+0cAe7TLC4eou4jmXJzfE+frxuLqOTce18b1jvb4Nh7vvF7Lc+o04GTgsTR/37/T08b3An/drt+L5m/9se32TwCntm22oD0H/q7nfeY64Bk05+jn6Plb46Hncf+x9pZ9EfDMtk13bve7T1+7Hd8+zyOn6r3dyWmUkz3l0vR6E3BoVa2qql/R/NP8g7RjRqvq2Kq6vWfbbyZ5TE/9/6iq71TV/VX1y3bdEVV1bVXdRPMP8FkDnn+isn8E/EtVXVJVdwEfHOJYPtOWv7eq7qmq06rq8mqcDXydpgd3ndqiV1XdWFX/VlV3VdXtwIdpEv9h/QHwlar6VvtcfwU83OE2v34taJKDhVX111V1d1VdQZP07dtT/ttVdXo1w1w+C/zmsE9UVScAf06TMJ8NXJ9k+YAqLweurKp/aV+f7wP/RtMOY07raY9Dgecm2Y4mQV4API0mMf5RVa0eNtYe99Aki0+pqvuq6vyqum2Csj+rqmPatjkO2ArYMsn2ND2672/b9ds0Cd8gW7c90r3TowGq6us0H+i+QfMh8k19dSdqk72By6rqs217fh74MfCKnrr/UlU/qapfAF/ggb+t1wCnt6/9/VV1JrCCJpmdrO6kquqE9u/j3qr6B5oPFJNeSzLonEqyFbCU5kP7ze3f99k91e8B/rpdfzrNB96nJgnNh6u3VdVN7d/q3/LA38HY+8zFVXUnzd/8Oqmqs6rqh22bXgR8noe+JxxWVXe27SrNeI4pl6bXE4EvJelNCO+jSUB+TpNs/iGwkAeSxi1oemWh6f3s9/Oe+btoesInMlHZrWkShTHjPU+/B5Vphxx8gKbHb6z3+IcD6k/YFsA1fft+FPBxmp62x7arF2T4sdxb98ZbVXcmuXGIeoP0Hv8TaZPBnnXzgP/bs9zf9htnLca6VtWJwIlJNgD2aecvqKozxin+RGC3vnjm03wYeEj8VXVHkpuAravqm2mGwBwJbJ/kS8A7ByTUE/kssB1wUpLNgBNoPoTdM07ZX7dNVd3V5HZsQnPu39R+UOyNe7sBzzvZmPKjgUOAv62q/nNg3DZpp5/1lf0ZTS/4Q46B5vXdpJ1/IvCHY8NmWhsA/zVE3UkleQfwZ22MBWxK026TmuicovkW4KaqunmCqjf2nbdjMS+k/daofQ2h+bZlXju/Nc23MGP623RoSXYDDqfpdd+Q5sPIF/uKDfM+Js0Y9pRL0+tqYGlVbdYzbVxV1wB/AiwDXgI8huYrWGj+qY2pEcW1mmYYyZhBSc9DYkmyEU1P7EeBLatqM+B0Hoh9vLgHtUW/d9D0/u1WVZvSDDFgkv33Wt17TG2S/7hJ6kym9zmvBn7adywLqmqviSoP2Nfggk3v5BeBi2gSkvHqXw2c3RfPJlV1cE+Z3vbYhGa4wbXtcxxRVc+hGZ60I80Qh/HcSZOEjXlCX5wfrKqdgOfR9N6/dtjjbK0GNm9fr4fEvbaSzKMZJnU8cHAees3ERG1yLU1y3Wt7+j48TuBq4LN9r8Wjq+rwIeoOPC/a8ePvoemBfmz7d3crD37PmPxJHnpOXU3T7putzX5ohkj9gmaY1NixPqaqxj5kPOjvkKYNe014Po3jczTfmmxXVY8BjuKhxz2q90tpJEzKpdHZIM3FbmPTfJp/HB/OAxfVLUyyrC2/APgVzXjTR9F87TtdvgAckOQ32gTo/WtZf6ynag1wb9tr3nvbtuuAx/UNxRnUFv0W0PyzvyXNxaAf6Nt+Hc047on8K/DyJC9IsiHNeNipfP87D7gtyXuSPDLJvCTPSPJbQ9YfGH97wdveSRakuUB2KU3CfO4E9b8C7Jjm4tYN2um3kvxGT5m9etrjb4Bzq+rqttxube/pnTRj4Sf6NuJCYN92/0voGR6T5MVJntkmwrfRDHlYqzvUVNXPaL7BOSzJhmlubfiKSaoN8j/bx9fTfIA8vo1vzLhtQvMBc8ckf5JkfpI/prlO4StDPOcJwCuSvKw9LzZOcxH2MHeIWUPzjdlE58YCmvHda4D5Sd5P01M+qUHnVDtc6avA/0lzkfUGSV44eI/QDuU6Bvh4kse3z7NNkpe1Rb4AvC7JTu37TP/f8YXA/0jyqPYD0xsGPN0Cmt78XybZlaZTQ5rVTMql0TmdJpEcmw6juXjsVODrSW6nudBxt7b88TRf514D/He7bVpU1VeBI2i+Ul9Jc3EdNB8Shql/O/AXNP90b6b5B3lqz/Yf04z5vKId47s1g9ui3ydoLoK7oS33tb7tn6QZj35zkofcR7yqLgHeQtO7trqNccrum94OoXkFzVjgn7Zx/hPNNx7D+DvgfW3bvHOc7bfRJJRXAbcAfw8c3I6xhr7jb1+Pl9KM5b2WZnjER2g+OI35HE1SdBPNRZivbtdvSpNY3UxzPt5Ik8CO569oLu69meY6hM/1bHsCzYeh22guKj6bJkFdW68GntvG8SGaiw8HnZe9d/kZm16V5DnA24HXtq/XR2h6UnvH5o/bJu0wl5fTfGNzI819619eVTdMFnyb1C+jef3W0PRCv4sh/v+2w3Y+DHynPTd27ytyBk3y/BOa1+qXDD9kY7Jz6k9pPkj9GLie5sLsYbyH5j3knCS3Af9JO8a9fZ/5BM3dUFby0LuifBy4m+ZD5nHAiQOe583AX7fvHe+nee+RZrWxq9sl6dfaHtWLgY2GHfOs2SPJZ2juUvK+rmNZW2l+4ObHVdXfy/pw9/sZZmmbSFo/2FMuCfj1T8tvmOSxNL2IXzYhV9fa4TRPbodY7EnT63xKx2FJ0pQzKZc05k00X69fTjP29+DBxaVp8QSa+1ffQTPE6uCquqDTiCRpBBy+IkmSJHXMnnJJkiSpY3Pix4O22GKLWrRoUddhSJIkaT13/vnn31BVC9e23pxIyhctWsSKFSsmLyhJkiQ9DEnW6ddqHb4iSZIkdcykXJIkSeqYSbkkSZLUMZNySZIkqWMm5ZIkSVLHRpqUJ9kzyaVJViZZPs72JDmi3X5Rkl16th2b5PokF/fV2TzJmUkuax8fO8pjWBenXHANzz/8m+yw/DSef/g3OeWCa7oOSZIkSTPYyJLyJPOAI4GlwE7Afkl26iu2FFjcTgcCn+7Z9hlgz3F2vRz4RlUtBr7RLs8Yp1xwDW8/+UKuueUXFHDNLb/g7SdfaGIuSZKkCY2yp3xXYGVVXVFVdwMnAcv6yiwDjq/GOcBmSbYCqKpvATeNs99lwHHt/HHAPqMIfl29998v4v6+dfe36yVJkqTxjDIp3wa4umd5Vbtubcv027KqVgO0j48fr1CSA5OsSLJizZo1axX4w/GLe/pT8sHrJUmSpFEm5RlnXa1DmXVSVUdX1ZKqWrJw4Vr/0ulIOIRFkiRJ4xllUr4K2K5neVvg2nUo0++6sSEu7eP1DzPOaeMQFkmSJI1nlEn594DFSXZIsiGwL3BqX5lTgde2d2HZHbh1bGjKAKcC+7fz+wP/MZVBj5JDWCRJkjSekSXlVXUvcAhwBvAj4AtVdUmSg5Ic1BY7HbgCWAkcA7x5rH6SzwPfBZ6aZFWSN7SbDgf2SHIZsEe7PGNsNN9bv0uSJGntzB/lzqvqdJrEu3fdUT3zBbxlgrr7TbD+RuD3pjDMKfWRV+3MW0++sOswJEmSNIvYrTvF9nn2ZDePkSRJkh7MpFySJEnqmEm5JEmS1DGTckmSJKljJuWSJElSx0zKJUmSpI6ZlEuSJEkdMymXJEmSOmZSLkmSJHXMpFySJEnqmEm5JEmS1DGTckmSJKljJuWSJElSx0zKJUmSpI6ZlEuSJEkdMymXJEmSOmZSLkmSJHXMpHyanXLBNV2HIEmSpBnGpHyaveuLF3YdgiRJkmYYk/Jpds/9XUcgSZKkmcakfATSdQCSJEmaVUzKR+DVu2/fdQiSJEmaRUzKR+BD+zyz6xAkSZI0i5iUS5IkSR0zKZckSZI6ZlIuSZIkdcykXJIkSeqYSbkkSZLUMZNySZIkqWMm5ZIkSVLHRpqUJ9kzyaVJViZZPs72JDmi3X5Rkl0mq5vkWUnOSXJhkhVJdh3lMUiSJEmjNrKkPMk84EhgKbATsF+SnfqKLQUWt9OBwKeHqPv3wAer6lnA+9tlSZIkadYaZU/5rsDKqrqiqu4GTgKW9ZVZBhxfjXOAzZJsNUndAjZt5x8DXDvCY5AkSZJGbv4I970NcHXP8ipgtyHKbDNJ3bcCZyT5KM2HiueN9+RJDqTpfWf77bdfpwOQJEmSpsMoe8ozzroassygugcDb6uq7YC3Af883pNX1dFVtaSqlixcuHDIkCVJkqTpN8qkfBWwXc/ytjx0qMlEZQbV3R/493b+izRDXSRJkqRZa5RJ+feAxUl2SLIhsC9wal+ZU4HXtndh2R24tapWT1L3WuB32vnfBS4b4TFIkiRJIzeyMeVVdW+SQ4AzgHnAsVV1SZKD2u1HAacDewErgbuAAwbVbXf9RuCTSeYDv6QdNz6bvO+UH/KhfZ7ZdRiSJEmaIVLVP8x7/bNkyZJasWLFtD7nouWnDdx+5eF7T1MkkiRJmi5Jzq+qJWtbz1/0lCRJkjpmUj4iG823aSVJkjQcM8cR+cirdu46BEmSJM0SJuUjss+zt+k6BEmSJM0SJuWSJElSx0zKJUmSpI6ZlEuSJEkdMymXJEmSOmZSLkmSJHXMpFySJEnqmEm5JEmS1DGTckmSJKljJuWSJElSx0zKJUmSpI6ZlEuSJEkdMynvyCkXXNN1CJIkSZohTMo78q4vXth1CJIkSZohTMo7cs/9XUcgSZKkmcKkXJIkSeqYSfkIPf/Jm3cdgiRJkmYBk/IROvGNz+06BEmSJM0CJuWSJElSx4ZOypO8IMkB7fzCJDuMLixJkiRp7hgqKU/yAeA9wHvbVRsAJ4wqKEmSJGkuGban/PeBVwJ3AlTVtcCCUQUlSZIkzSXDJuV3V1UBBZDk0aMLSZIkSZpbhk3Kv5DkH4HNkrwR+E/gn0YXliRJkjR3zB+mUFV9NMkewG3AU4H3V9WZI41MkiRJmiOGSsqTfKSq3gOcOc46SZIkSQ/DsMNX9hhn3dKpDESSJEmaqwb2lCc5GHgz8KQkF/VsWgB8Z5SBSZIkSXPFZD3lnwNeAZzaPo5Nz6mq10y28yR7Jrk0ycoky8fZniRHtNsvSrLLMHWT/Hm77ZIkfz/Ecc5I7zvlh12HIEmSpBlgYFJeVbdW1ZVVtV9V/Qz4Bc1tETdJsv2guknmAUfSDHPZCdgvyU59xZYCi9vpQODTk9VN8mJgGbBzVT0d+OhaHO+McsI5V3UdgiRJkmaAYX/R8xVJLgN+CpwNXAl8dZJquwIrq+qKqrobOIkmme61DDi+GufQ3HJxq0nqHgwcXlW/Aqiq64c5BkmSJGmmGvZCzw8BuwM/qaodgN9j8jHl2wBX9yyvatcNU2ZQ3R2B305ybpKzk/zWeE+e5MAkK5KsWLNmzSShjs78R6Sz55YkSdLsMGxSfk9V3Qg8Iskjquq/gGdNUme8bLSGLDOo7nzgsTQfEt5F88NGDylfVUdX1ZKqWrJw4cJJQh2dj/7hb3b23JIkSZodhk3Kb0myCfAt4MQknwTunaTOKmC7nuVtgWuHLDOo7irg39shL+cB9wNbDHkc026fZ/d/OSBJkiQ92LBJ+TKaizzfBnwNuBx4+SR1vgcsTrJDkg2BfWnu4tLrVOC17V1YdgdurarVk9Q9BfhdgCQ7AhsCNwx5HJIkSdKMM9QvelbVnT2LxyV5GvAR4I0D6tyb5BDgDGAecGxVXZLkoHb7UcDpwF7ASuAu4IBBddtdHwscm+Ri4G5g/6rqHxYjSZIkzRqT/XjQzjS3HNyapof6fwP/B9gN+IfJdl5Vp9Mk3r3rjuqZL+Atw9Zt198NTHqPdEmSJGm2mGz4yjE0PyD0KmAN8H3gCuApVfXxEccmSZIkzQmTDV/ZqKo+085fmuSdwPKqum+0YUmSJElzx2RJ+cZJns0Dtyi8A9h57BaEVfX9UQYnSZIkzQWTJeWrgY/1LP+8Z7lo74IiSZIkad0NTMqr6sXTFYgkSZI0Vw17n/JfS3L0KAKZq/b42FldhyBJkqSOrXVSDiyZ8ijmsMuuv3PyQpIkSVqvrUtSfv2URyFJkiTNYWudlFfVnqMIZH22+PGP7joESZIkzWCT3X0FgCRfprnbSq9bgRXAP1bVL6c6sPXJmW9/EYuWn9Z1GJIkSZqhhu0pv4LmHuXHtNNtwHXAju2yJEmSpHU0VE858OyqemHP8peTfKuqXpjkklEEJkmSJM0Vw/aUL0yy/dhCO79Fu3j3lEclSZIkzSHD9pS/A/h2ksuBADsAb07yaOC4UQUnSZIkzQVDJeVVdXqSxcDTaJLyH/dc3PmJEcUmSZIkzQnD9pQDPAdY1NbZOQlVdfxIopIkSZLmkGFvifhZ4MnAhcB97eoCTMolSZKkh2nYnvIlwE5V1X+vck2B3T58JuceukfXYUiSJKkjw9595WLgCaMMZC677nZvYCNJkjSXDdtTvgXw30nOA341trKqXjmSqCRJkqQ5ZNik/LBRBjEXbLlgQ3vEJUmSNK5hb4l49qgDWd+de+geLFp+WtdhSJIkaQYamJQn+XZVvSDJ7TR3W/n1JqCqatORRidJkiTNAQOT8qp6Qfu4YHrCkSRJkuaeoX88KMk8YMveOlV11SiCkiRJkuaSYX886M+BDwDXAfe3qwvYeURxSZIkSXPGsD3lfwk8tapuHGUwkiRJ0lw07I8HXQ3cOspA5rpTLrim6xAkSZLUkWF7yq8AzkpyGg/+8aCPjSSqOeitJ1/IPs/epuswJEmS1IFhk/Kr2mnDdpIkSZI0RSZNytu7riyuqtdMQzySJEnSnDPpmPKqug9YmGSte8iT7Jnk0iQrkywfZ3uSHNFuvyjJLmtR951JKskWaxtXV7Zc4JcMkiRJeqhhL/S8EvhOkr9K8vaxaVCFtof9SGApsBOwX5Kd+ootBRa304HAp4epm2Q7YA+aITWzxrmH7tF1CJIkSZqBhk3KrwW+0pZf0DMNsiuwsqquqKq7gZOAZX1llgHHV+McYLMkWw1R9+PAu2nulS5JkiTNakNd6FlVH1yHfW9DcyvFMauA3YYos82gukleCVxTVT9IMuGTJzmQpved7bfffh3ClyRJkqbHsL/ouZCmZ/rpwMZj66vqdwdVG2ddf8/2RGXGXZ/kUcChwEsHBtzEdjRwNMCSJUvsUZckSdKMNezwlROBHwM7AB+kGWP+vUnqrAK261nelmYYzDBlJlr/5DaGHyS5sl3//SRPGPI4JEmSpBln2KT8cVX1z8A9VXV2Vb0e2H2SOt8DFifZob1zy77AqX1lTgVe296FZXfg1qpaPVHdqvphVT2+qhZV1SKa5H2Xqvr5kMcxo+3xsbO6DkGSJEkdGPbHg+5pH1cn2Zum13rbQRWq6t4khwBnAPOAY6vqkiQHtduPAk4H9gJWAncBBwyqu1ZHNgtddv2dXYcgSZKkDgyblH8oyWOAdwD/G9gUeNtklarqdJrEu3fdUT3zBbxl2LrjlFk0WQySJEnSTDfs3Ve+0s7eCrx4dOFIkiRJc89QY8qT7JjkG0kubpd3TvK+0Ya2fnrN7t6eUZIkSQ827IWexwDvpR1bXlUX0Vx8qbX0oX2e2XUIkiRJmmGGTcofVVXn9a27d6qDkSRJkuaiYZPyG5I8mfbHf5L8AbB6ZFFJkiRJc8iwSflbgH8EnpbkGuCtwEGjCmouO+WCa7oOQZIkSdNsqKS8qq6oqpcAC4GnVdULgN8faWRz1FtPvrDrECRJkjTNhu0pB6Cq7qyq29vFt48gHkmSJGnOWaukvE+mLApJkiRpDns4SXlNWRRzzKYbzes6BEmSJM0gA5PyJLcnuW2c6XZg62mKcb1z0Qf37DoESZIkzSDzB22sqgXTFYgkSZI0Vz2c4SuSJEmSpoBJ+Qy08we+1nUIkiRJmkYm5TPQbb+6r+sQJEmSNI1MyiVJkqSOmZRLkiRJHTMp78hrdt++6xAkSZI0Q5iUd+RD+zyz6xAkSZI0Q5iUS5IkSR0zKZ+hdlh+WtchSJIkaZqYlM9Q1XUAkiRJmjYm5ZIkSVLHTMo7ND9dRyBJkqSZwKS8Qyv/bu+uQ5AkSdIMYFI+g73vlB92HYIkSZKmgUn5DHbCOVd1HYIkSZKmgUm5JEmS1DGTckmSJKljJuUd8w4skiRJGmlSnmTPJJcmWZlk+Tjbk+SIdvtFSXaZrG6S/5Xkx235LyXZbJTHMGregUWSJEkjS8qTzAOOBJYCOwH7Jdmpr9hSYHE7HQh8eoi6ZwLPqKqdgZ8A7x3VMcwEi5af1nUIkiRJGrFR9pTvCqysqiuq6m7gJGBZX5llwPHVOAfYLMlWg+pW1der6t62/jnAtiM8BkmSJGnkRpmUbwNc3bO8ql03TJlh6gK8HvjqeE+e5MAkK5KsWLNmzVqGLkmSJE2fUSbl413CWEOWmbRukkOBe4ETx3vyqjq6qpZU1ZKFCxcOEa4kSZLUjVEm5auA7XqWtwWuHbLMwLpJ9gdeDry6qvoT/VnnysMHX+y524fPnKZIJEmS1IVRJuXfAxYn2SHJhsC+wKl9ZU4FXtvehWV34NaqWj2obpI9gfcAr6yqu0YY/4xx3e13dx2CJEmSRmj+qHZcVfcmOQQ4A5gHHFtVlyQ5qN1+FHA6sBewErgLOGBQ3XbXnwI2As5MAnBOVR00quOQJEmSRm1kSTlAVZ1Ok3j3rjuqZ76Atwxbt13/lCkOU5IkSeqUv+g5Q0z2y56OK5ckSVp/mZTPEJP9sqfjyiVJktZfJuWSJElSx0zKJUmSpI6ZlM8gkwwrZ9Hy06YlDkmSJE0vk/IZ5KeT/IiQJEmS1k8m5ZIkSVLHTMpnmR0cwiJJkrTeMSmfYbZcsOHA7TVNcUiSJGn6mJTPMOceukfXIUiSJGmamZTPQt6FRZIkaf1iUj4DTXZrREmSJK1fTMpnoGFujfjqY747DZFIkiRpOpiUz1LfufymrkOQJEnSFDEpn6E2nucgFkmSpLnCpHyG+vGH95q0jBd8SpIkrR9MyiVJkqSOmZTPYFcOccGnveWSJEmzn0n5emDnD3yt6xAkSZL0MJiUz3DPf/Lmk5a57Vf3TUMkkiRJGhWT8hnuxDc+d6hyDmORJEmavUzKZ4HX7L79UOVMzCVJkmYnk/JZ4EP7PHPosu875YcjjESSJEmjYFI+SwxzJxaAE865asSRSJIkaaqZlK+HFi0/zaEskiRJs4hJ+SwybG/5GBNzSZKk2cGkfJZZl8Tc5FySJGlmMymfhdY2MQeTc0mSpJlsftcBaN28Zvft1+mizt7EfF2Se0mSJE09k/JZauw2iQ/nbiv9Pecm6ZIkSd1IVXUdw8gtWbKkVqxY0XUYIzPdw1JM3iVJksaX5PyqWrLW9UaZlCfZE/gkMA/4p6o6vG972u17AXcBr6uq7w+qm2Rz4GRgEXAl8EdVdfOgONb3pBy804okSdJ4prszcV2T8pFd6JlkHnAksBTYCdgvyU59xZYCi9vpQODTQ9RdDnyjqhYD32iX5zx7ryVJkh5qtnRcjvLuK7sCK6vqiqq6GzgJWNZXZhlwfDXOATZLstUkdZcBx7XzxwH7jPAYZpUrD9/b5FySJGkWGmVSvg1wdc/yqnbdMGUG1d2yqlYDtI+PH+/JkxyYZEWSFWvWrFnng5iNTM4lSZJml1HefSXjrOsfwD5RmWHqDlRVRwNHQzOmfG3qri96E/PZ8tWNJEnSXDTKpHwVsF3P8rbAtUOW2XBA3euSbFVVq9uhLtdPadTrqf6ec5N0SZKkmWOUSfn3gMVJdgCuAfYF/qSvzKnAIUlOAnYDbm2T7TUD6p4K7A8c3j7+xwiPYb017PAWk3dJkjSbzZYhvSNLyqvq3iSHAGfQ3Nbw2Kq6JMlB7fajgNNpboe4kuaWiAcMqtvu+nDgC0neAFwF/OGojkGz50SWJEmazfzxIEmSJGmKzLj7lEuSJEkajkm5JEmS1DGTckmSJKljJuWSJElSx+bEhZ7tLRZ/1sFTbwHc0MHzzkW29fSwnaeH7Tw9bOfpYTtPD9t5+kzW1k+sqoVru9M5kZR3JcmKdbn6VmvPtp4etvP0sJ2nh+08PWzn6WE7T59RtbXDVyRJkqSOmZRLkiRJHTMpH62juw5gDrGtp4ftPD1s5+lhO08P23l62M7TZyRt7ZhySZIkqWP2lEuSJEkdMymXJEmSOmZSPiJJ9kxyaZKVSZZ3Hc9slOTKJD9McmGSFe26zZOcmeSy9vGxPeXf27b3pUle1rP+Oe1+ViY5Ikm6OJ6ZIsmxSa5PcnHPuilr1yQbJTm5XX9ukkXTeoAzxATtfFiSa9pz+sIke/Vss53XQZLtkvxXkh8luSTJX7brPaen0IB29pyeQkk2TnJekh+07fzBdr3n8xQa0M7dns9V5TTFEzAPuBx4ErAh8ANgp67jmm0TcCWwRd+6vweWt/PLgY+08zu17bwRsEPb/vPabecBzwUCfBVY2vWxddyuLwR2AS4eRbsCbwaOauf3BU7u+phnUDsfBrxznLK287q381bALu38AuAnbXt6Tk9PO3tOT207B9iknd8AOBfY3fN52tq50/PZnvLR2BVYWVVXVNXdwEnAso5jWl8sA45r548D9ulZf1JV/aqqfgqsBHZNshWwaVV9t5q/jON76sxJVfUt4Ka+1VPZrr37+lfg98Z6DuaSCdp5IrbzOqqq1VX1/Xb+duBHwDZ4Tk+pAe08Edt5HVTjjnZxg3YqPJ+n1IB2nsi0tLNJ+WhsA1zds7yKwW9eGl8BX09yfpID23VbVtVqaP5JAI9v10/U5tu08/3r9WBT2a6/rlNV9wK3Ao8bWeSzzyFJLkozvGXsK2jbeQq0Xw8/m6bXy3N6RPraGTynp1SSeUkuBK4Hzqwqz+cRmKCdocPz2aR8NMb7JOS9J9fe86tqF2Ap8JYkLxxQdqI297V4eNalXW3ziX0aeDLwLGA18A/tetv5YUqyCfBvwFur6rZBRcdZZ1sPaZx29pyeYlV1X1U9C9iWpjf2GQOK287raIJ27vR8NikfjVXAdj3L2wLXdhTLrFVV17aP1wNfohkWdF37dRHt4/Vt8YnafFU7379eDzaV7frrOknmA49h+GEc67Wquq79R3A/cAzNOQ2288OSZAOaRPHEqvr3drXn9BQbr509p0enqm4BzgL2xPN5ZHrbuevz2aR8NL4HLE6yQ5INaQb4n9pxTLNKkkcnWTA2D7wUuJimHfdvi+0P/Ec7fyqwb3u18w7AYuC89mu+25Ps3o7lem1PHT1gKtu1d19/AHyzHWs35439U239Ps05DbbzOmvb5Z+BH1XVx3o2eU5PoYna2XN6aiVZmGSzdv6RwEuAH+P5PKUmaufOz+fJrgR1Wucre/eiuTr9cuDQruOZbRPNnWt+0E6XjLUhzXisbwCXtY+b99Q5tG3vS+m5wwqwpP3Duhz4FO0v2c7VCfg8zddy99B8kn/DVLYrsDHwRZoLYc4DntT1Mc+gdv4s8EPgovYNeyvb+WG38wtovhK+CLiwnfbynJ62dvacntp23hm4oG3Pi4H3t+s9n6ennTs9n8cqSpIkSeqIw1ckSZKkjpmUS5IkSR0zKZckSZI6ZlIuSZIkdcykXJIkSeqYSbkkrYeS3Jfkwp5p+RTue1GSiycvKUka1vyuA5AkjcQvqvkJaUnSLGBPuSTNIUmuTPKRJOe101Pa9U9M8o0kF7WP27frt0zypSQ/aKfntbual+SYJJck+Xr7q3gk+Ysk/93u56SODlOSZh2TcklaPz2yb/jKH/dsu62qdqX59blPtOs+BRxfVTsDJwJHtOuPAM6uqt8EdqH5hV1ofmb6yKp6OnAL8Kp2/XLg2e1+DhrNoUnS+sdf9JSk9VCSO6pqk3HWXwn8blVdkWQD4OdV9bgkN9D8pPQ97frVVbVFkjXAtlX1q559LALOrKrF7fJ7gA2q6kNJvgbcAZwCnFJVd4z4UCVpvWBPuSTNPTXB/ERlxvOrnvn7eOAapb2BI4HnAOcn8dolSRqCSbkkzT1/3PP43Xb+/wH7tvOvBr7dzn8DOBggybwkm0600ySPALarqv8C3g1sBjykt16S9FD2YEjS+umRSS7sWf5aVY3dFnGjJOfSdMzs1677C+DYJO8C1gAHtOv/Ejg6yRtoesQPBlZP8JzzgBOSPAYI8PGqumWKjkeS1muOKZekOaQdU76kqm7oOhZJ0gMcviJJkiR1zJ5ySZIkqWP2lEuSJEkdMymXJEmSOmZSLkmSJHXMpFySJEnqmEm5JEmS1LH/D2pQvQwtAqdCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## you can test what is happening to the learning_rate\n",
    "initial_learning_rate = 0.01\n",
    "decay_rate = 0.1\n",
    "## for 20 epoch => i have this number of steps if (batch_size=32)\n",
    "decay_steps = 20 * len(X_train) // 32\n",
    "\n",
    "## for this steps ==> 32, 64, ....\n",
    "steps = []\n",
    "batch_size_each = 32\n",
    "for step in range(decay_steps):\n",
    "    steps.append(batch_size_each)\n",
    "    batch_size_each += 32\n",
    "    \n",
    "steps = np.array(steps) \n",
    "lr_each_step = initial_learning_rate * decay_rate**(steps/decay_steps)\n",
    "\n",
    "### Visualization of this schedular\n",
    "plt.figure(figsize=(12, 4))\n",
    "plt.plot(range(1, len(lr_each_step)+1), lr_each_step, 'o--')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Learning-Rate')\n",
    "plt.title('Learning rate at diffrent Steps using Exponential Schedular')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d354ce83",
   "metadata": {},
   "source": [
    "#### `Piecewise Scheduling`\n",
    "\n",
    "for each number of epochs reduce learning_rate steps\n",
    "\n",
    "#### you can emplement you custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "728eb6fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "## le that initail LR = 0.01\n",
    "def piecewise_LR(epoch):\n",
    "    if epochch < 5:\n",
    "        return 0.01\n",
    "    elif epoch < 15:\n",
    "        return 0.005\n",
    "    else:\n",
    "        return 0.001\n",
    "    \n",
    "## pass this function to LearningRateScheduler\n",
    "lr_scheduler = tf.keras.callbacks.LearningRateScheduler(piecewise_LR)\n",
    "\n",
    "## pass above  'lr_scheduler' to callabaks in fit\n",
    "# history = model_fashion.fit(X_train, y_train, \n",
    "#                             validation_data=(X_val, y_val), epochs=30, \n",
    "#                             batch_size=32, callbacks=[early_stopp, lr_scheduler])    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a4f3328",
   "metadata": {},
   "source": [
    "#### `Performance Scheduling`\n",
    "\n",
    "`multiply the learning rate by a factor less than 1.0, if the monitor like 'val_loss' does not improve for number of epochs` \n",
    "#### you can emplement you custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "88f05652",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.5549 - accuracy: 0.8030 - val_loss: 0.4722 - val_accuracy: 0.8285\n",
      "Epoch 2/25\n",
      "1688/1688 [==============================] - 8s 4ms/step - loss: 0.4467 - accuracy: 0.8408 - val_loss: 0.4342 - val_accuracy: 0.8430\n",
      "Epoch 3/25\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.4192 - accuracy: 0.8496 - val_loss: 0.3668 - val_accuracy: 0.8663\n",
      "Epoch 4/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3973 - accuracy: 0.8579 - val_loss: 0.3629 - val_accuracy: 0.8655\n",
      "Epoch 5/25\n",
      "1688/1688 [==============================] - 8s 4ms/step - loss: 0.3849 - accuracy: 0.8618 - val_loss: 0.3704 - val_accuracy: 0.8588\n",
      "Epoch 6/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3753 - accuracy: 0.8639 - val_loss: 0.3749 - val_accuracy: 0.8632\n",
      "Epoch 7/25\n",
      "1688/1688 [==============================] - 8s 4ms/step - loss: 0.3610 - accuracy: 0.8703 - val_loss: 0.4501 - val_accuracy: 0.8397\n",
      "Epoch 8/25\n",
      "1688/1688 [==============================] - 7s 4ms/step - loss: 0.3503 - accuracy: 0.8734 - val_loss: 0.3601 - val_accuracy: 0.8685\n",
      "Epoch 9/25\n",
      "1688/1688 [==============================] - 8s 4ms/step - loss: 0.3511 - accuracy: 0.8754 - val_loss: 0.3739 - val_accuracy: 0.8705\n",
      "Epoch 10/25\n",
      "1688/1688 [==============================] - 8s 4ms/step - loss: 0.3371 - accuracy: 0.8775 - val_loss: 0.3601 - val_accuracy: 0.8730\n",
      "Epoch 11/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3390 - accuracy: 0.8787 - val_loss: 0.4173 - val_accuracy: 0.8708\n",
      "Epoch 12/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3248 - accuracy: 0.8838 - val_loss: 0.3892 - val_accuracy: 0.8773\n",
      "Epoch 13/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.3230 - accuracy: 0.8857 - val_loss: 0.3871 - val_accuracy: 0.8725\n",
      "Epoch 14/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2393 - accuracy: 0.9099 - val_loss: 0.2897 - val_accuracy: 0.8972\n",
      "Epoch 15/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2286 - accuracy: 0.9125 - val_loss: 0.3066 - val_accuracy: 0.8940\n",
      "Epoch 16/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2190 - accuracy: 0.9154 - val_loss: 0.3054 - val_accuracy: 0.8930\n",
      "Epoch 17/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2142 - accuracy: 0.9179 - val_loss: 0.3165 - val_accuracy: 0.8890\n",
      "Epoch 18/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2114 - accuracy: 0.9191 - val_loss: 0.3408 - val_accuracy: 0.8922\n",
      "Epoch 19/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.2081 - accuracy: 0.9207 - val_loss: 0.3249 - val_accuracy: 0.8950\n",
      "Epoch 20/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1713 - accuracy: 0.9349 - val_loss: 0.3008 - val_accuracy: 0.9022\n",
      "Epoch 21/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1659 - accuracy: 0.9362 - val_loss: 0.3020 - val_accuracy: 0.9020\n",
      "Epoch 22/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1626 - accuracy: 0.9374 - val_loss: 0.3021 - val_accuracy: 0.9015\n",
      "Epoch 23/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1577 - accuracy: 0.9396 - val_loss: 0.3013 - val_accuracy: 0.8998\n",
      "Epoch 24/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1559 - accuracy: 0.9408 - val_loss: 0.3071 - val_accuracy: 0.9047\n",
      "Epoch 25/25\n",
      "1688/1688 [==============================] - 8s 5ms/step - loss: 0.1379 - accuracy: 0.9474 - val_loss: 0.2994 - val_accuracy: 0.9053\n"
     ]
    }
   ],
   "source": [
    "## for performance scheduling\n",
    "lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=5)\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    tf.keras.layers.Dense(300, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    tf.keras.layers.Dense(100, activation=\"selu\", kernel_initializer=\"lecun_normal\"),\n",
    "    tf.keras.layers.Dense(10, activation=\"softmax\")\n",
    "])\n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=0.02, momentum=0.9)\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "n_epochs = 25\n",
    "history = model.fit(X_train, y_train, epochs=n_epochs,\n",
    "                    validation_data=(X_val, y_val),\n",
    "                    callbacks=[lr_scheduler])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "94622e7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAc8AAAEbCAYAAAC81w8SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABW5UlEQVR4nO2dd7hcVdX/P9/0Ir2GJNwkGGoCAUIJUkWQAK8BBaWoIL7EUCwoCogF4ccrRkHkBYKgCLwgiFJFEAEpIi0BQ2iJKVJCQkKHENLX7491hpxMppy5d+6Ue9fnec5z5pyz9z7rzNw739l7r72WzIwgCIIgCLLTpd4GBEEQBEGzEeIZBEEQBBUS4hkEQRAEFRLiGQRBEAQVEuIZBEEQBBUS4hkEQRAEFRLiGZRE0gJJx9bbjo6GpBclnVpvOxoRSV0k/VrSm5JM0t71tikI8gnx7ABIuir5kjFJyyS9LGmCpHXqbVs1kLR38mzrF7l+Vur5V0iaI+k6SQNrbWtiz7Epe0zSXEk3ShrcxjYXVNPOtpL3jO9LmiTps1Vo+kDgK8B/Af2AR6rQZhBUlRDPjsO9+BfNIOC/8S+eS+tpUI2Zhj//AOALwHDgxjraszCxZxPgKGAEcLukrnW0qT04Hn/OnYCngT9KGtWahiR1kyTg48BcM3vEzF4zsyWtbK97a+oFQRZCPDsOi5Mvmtlm9jfgD8D+6QKSviLpeUmLJP1b0imSuqSuf1zSA8n1aZIOzqs/KOlljMw7b5IOSx1vkvT83pS0UNJkSfukrv+XpCeT+/xH0rmSerTx+Zclzz/HzP4BXAHsKmnNUpUkfVbSM5IWS3pF0pnJF3ju+ouSfpAMI74nabak72awxxJ75prZ/cBPgGG4MBSy49uSpkj6QNKrkn4jae3k2t7A74C+qZ7eWcm1HpJ+ltj1gaSJkj6darerpN8m7/OHkqZL+l7e536VpDvy7DlL0rMZnvOd5DmnAuOARcBnkjb6S7pB0tvJ9hdJQ/PvkfSqZwKLgT8CvwQ2TZ7zxaRsT0kXSpqX/N08Jmn3VFu50YkDJT0haQnw6eTveYKk8yW9Jel1Sd9M2rtE0jvykZov5T3/ecn/wIfJ38B4Sb0K2H6EpJlJz/tW5Y2OSDom9fc1T9JVqWtrSbpc0vyk/oP5/1tB4xLi2QGRNAQ4AFiaOnc88D/Aj4CtgO8ApwEnJte7ALfgfxOjgOOAs4CeFd67L/Ag3gM+FO8Bnp26/mngOuBiYJvkPocltlUFSRsDnwWWJ1uxcjviX9Y3J3aeDpwBnJxX9BTgGWAH4GfAeFXeu/ow2RfrDa0AvoW/J0cBOwP/m1x7JLmW6832A36RXPsdsFdSZzhwNfBnSdsl17sArwKfxz/3M4Hv48OiVcXMlgLLgO6S+gD342K6F/43NRe4N7mWY3Bi++HAdsBX8b+X2azs0QKMx0cUjgO2xz+Pv0rql2fGz4AfAFsCjyfnjgbeB3YBzgMuBG4F/g2MxN+z30jaJNXOB8m9tsL/R47A37s0gxKbDsV/qG4PnJu7KOlrwK/xz2hbfDj6ueSagL8A/YGDk7oPAX8v8ExBI2JmsTX5BlyFf2ktwL+kLdlOSZV5GfhSXr1vAc8nr/fHhWbT1PXdk3aOTY4HJccj89ox4LDk9fH4F9X6RWx9CPhh3rlDEttVpM7eyT2KtXlWYvsCXGByz/+rMu/bdcDfC7Q1O3X8InB9XpnpwA9KtHsssCB1PAB4FHgF6JFq99QSbRyA98S6FGozObcZLrqb5p2/Fbi0RNvnAffm/f3cUeB9eLbM+5f+3HviomXAaFx4pqc/U6Ar8Cbw+dQ9lgIb5bV7KvBi6rgvsAT4cl5bM4H/l/c38rm8th4AHk0dC3gduD11rnvS/mElnnUcMCPv/VkErJU6d2ZemdnAeUXa+2Ty99o77/xk4Hul3vfYGmPrRtBReAgYC/TGBWwz4CIASRsAA4FfS5qQqtMN/zIB/4X9qpm9nLr+OP7lXAnbA1PM7I0i13cEdpZ0Wupcl8TujfHeSWuYif+y7wmMAT6H97BKsRX+6z/Nw8CPJa1pZu8l56bklZkDbFim7b5yBx8BfYCngM9akfk7SZ/Ee71bAWvh4tADf0/mFLnHDkn7z6dGmsHfg7+n2h6Hz4O34O9zd+ClMvZn5f+SocjewLv4D4K7JF2C9yrfz7OtD/63mWO2mc0rc4/NEpv/mTthZsslPQpsnVd2UoH6H31+ZmaS5uM919y5pZLeJvWZyqchvoUPs38M/zzy56tfMrN3U8cf/V1I2hDvVd5X5Jl2xN+L1/Pen16s+v4EDUqIZ8dhoZnNSF5/Q9L9wA/xX8i54flxFPdcVJHzaXJCmp4TzB+GLNdOF3z+748Frr2ewYZiLEk9/3PJ3NoleI+tGMJ7K4VIn19a4Fq5KY+FuJPQCmCemX1Q1AipBRfxK/Bh9TdxYbweF9BidEls2amAjR8mbX8BH6Y8Ff/s3wNOwocac6xg9c8tq7PNd4G/Au+Z2fw82ybjw535vJV6XfR9SZGzrdBnlX+uUHuFPr+in6mkXYEb8L/TU4B38HncX+TVKfV3keX/YB6wR4Fr7xU4FzQYIZ4dl58Ad0m63MzmSHoV2MzMrilS/nmgv6SBZvZKcm5nVhWJnLil52RG5LXzFPBFSesX6X0+BWyZErr24hxgmqT/NbMni5R5Hh+aTrM73ht6v433twqecSQukqeY2XIA5Tlr4cOK+T2ff+Ff0hubOyUVYnfgcTO7OHdCUn7P5nVW/xzzj4vxWpHnfAo4EnjDzN7J2FYxZuDPvzswC9wRCp9H/X0b2y7EJ/BRmHNyJ5IfOJkxs3nJ/9y+wD0FijwFbASsMLNZbTE2qA/hMNRBMbMHcOeEHySnzgK+J/ew3ULSMElflnRGcv1eYCpwjaQRiUPML/G51FybHwKPAadJ2kbSbqz+a/z3wHzgVkl7SBos6TNa6W17NnCUpLMTG7aUdJik8Rkea1hiW3or+DecfCHdjotoMc4H9ko8JzeXdDTuSJXFlmoyHf9f/Fbyfh2JDxmmeRHoJWk/SetL6mNm/8bnba9K3sMhkkZKOlUr11v+G9hB0mhJQyX9EHfgSfN3YHtJx8k9rr+HC0hbuA7vWd0maa/kufaUe70OLVc5TdJrnwCcJ/em3So53oj2WY71b/yH5NHJe3oC/kOgUs7FP9NTkr+vEZK+k1y7Fx+Gvi35bAZLGiXpJ5IK9UaDBiPEs2NzAfBVSS1m9hvcieNL+Hq8f+BzpP8BMLMV+FBeF3yu8xrg/+FOK2mOS/YTcU/CH6QvJl90e+Eenn/GBfwnJMNrZnY3cBCwD/BEsp2OOzSV4368t5Xe+pQofz4wOhH51TCzp3Avz88Bz+KONOfhnsA1w8ymAN8Evo33hv8bH2ZNl3kEuAwfyn0d+F5y6Su4N+d4/MfPHcCerJzT/DW+3vX3+Gc2CH9f0m3fjX9G5wJPJmXaJEpmtjCxYxY+RD8V92pdB3i7FU2ehj/H7/Dh4G2BA8ystXPkRTGzPwM/x4e7pwD74cPplbYzAR8iPx7/+/or7k2NmRk+R/93fLh+Gv58W1B8jjtoIOSfYRAEQRAEWYmeZxAEQRBUSIhnEARBEFRIiGcQBEEQVEiIZxAEQRBUSKzzTNGlSxfr3bt3vc0IgiBoKhYuXGhm1qk6YyGeKXr37s0HH2QJeBIEQRDkkPRh+VIdi071SyEIgiAIqkGIZxAEQRBUSIhnEARB0O5IOkCeYHyGpNNLlNtJ0vIksw1JONHJqe09Sd9Krp0lTx6fu3ZgjR4n5jyDIAiC9iUJ5H8JHupwNjBR0u1m9nyBcj8D7s6dM7NpJIkKkuuvArekqv3SzPJjbLc70fMMgiAI2pud8UThs5KctjfgeXfz+TpwE55cohD7AjPNrFr5aFtNTcWzXLddzkXJ9SmSdkjOD5R0v6QXJD0n6ZupOutKukfS9GS/TuraGUlb0yR9ur2e67rrYKcBc3lQe7HTwNe47rry5QcNgi5dfF+ufK3qhF0dw64gqAPdJE1KbWPzrvcHXkkdz07OfYSk/nhyistK3OcIPDlCmpMTvbgy/f3f7phZTTY8F+FMYAieu/BpYOu8MgcCd+E5CnfF8xCC54/cIXm9Bp4yaOvkeDxwevL6dOBnyeutk3v0xDPazwS6lrKxT58+VinXXmvWp4/ZJZxgy+hil3CC9enj50uVh5VbqfK1qhN2dQy7gqAeAB9Y6e//w4HfpI6/BPxvXpk/Arsmr68CDsu73gN4A9godW6jRFu64FmBrixlRzW3mmVVSfJDnmVmn06OzwAws5+myvwaeMDMrk+OpwF7W17aIUm3AReb2T3pMpL6JfW3yG9f0t3J/R8tZmPfvn2t0nWei7r0ppctWv08vfifH66+9Omii+Ddd1dvZ6214BvfKHyPWtQJu5rLrpYWePHFwnWCoNZIWmhmfUtcz/L9/x+84wSwPrAQGGtmtybXxwAnmdn+Re4xCLjDzIa1+YGyUCuVBg5j9V8eF+eVuQPYPXV8HzAyr8wgPPfjmsnxO3nX3072FwNfTJ3/LXm/ZJLzY4FJwKQePXoU/llVgn7MsT9z0EfdggX0sf/jaNuIuSbZalu6B5G/FSpfqzphV3PZJVX8pxoE7Qble57d8Nyug1k58rhNifJX5X9f4/OkX8k71y/1+hTghlJ2VHOr5ZynCpzL7/aWLCPpY/hk8rfM7L0q3A8zu9zMRprZyG7dKnc+7tHSj1cYgAHL6UIvFvEea9KrZWNWrGC1raWlcDstLauXrWWdsKu57Np008Lng6ARMbNlwMm4F+0LwI1m9pykcZLGlasvqQ/uqXtz3qXxkp6RNAXYBxfQ2lArlQZGAXenjs8Azsgr82vgyNTxNJJfFkB3/I3/dl6ddJl+wLRC7Sd1R5WysbVznrd2OdT+xXa2mG72a/7bbu1yaNPNlYVdHcOuIKgHlOl5dsStdjfK0G0HDmJVh6EnkvMCrgEuLNDuz1nVYWh88nobVnUYmkU7OAyZ+RfZ4Rs9aAZ2wvo3lv1iu/Zas5YWH3pracn2RViLOmFX49qVE9CsdYKglnRG8ayZwxBAEv3hQtw76kozOzfXZTezyyQJn6s8AJ8s/oqZTZK0O/AP4BlgRdLc983sTknrATcCm+JzoYeb2VvJ/c4EjgOW4UO9d5WyrzUOQx+xbBlsvDEccABce23r2giCInz72/DrX8OCBaBCExJBUEfKOQx1RGoqno1Om8QT4Nhj4bbbYP586N69anYFwS9/6QL61luwTu1WsgVBJjqjeEaEoWoyZgy88w784x/1tiToYAwY4PtXXildLgiC2hDiWU323x969YLbb6+3JUEHY+BA34d4BkFjEOJZTfr2hU99yoduYzg8qCIhnkHQWIR4VpsxYzz0yzPP1NuSoAOx8cbQtSvMnl1vS4IggBDP6nPwwe4Oedtt9bYk6EB07QqbbBI9zyBoFEI8q83GG8Muu4R4BlVn4MAQzyBoFEI824MxY+DJJ2OMLagqIZ5B0DiEeLYHY8b4PrxugyoycKD/HgtftCCoPyGe7cGWW8LQoTF0G1SVAQNg0SJ48816WxIEQYhneyB57/P++wsnYwyCVhDLVYKgcQjxbC/GjIGlS+Huu+ttSdBBCPEMgsYhxLO9GDUKNtgghm6DqpETz/BDC4L6E+LZXnTt6ms+77zTe6BB0EY23NDzDUTPMwjqT4hne5ILFP/QQ/W2JOgAdOkC/fuHeAZBIxDi2Z586lMeKD6GboMqEWs9g6AxCPFsT/r2hf32i0DxhZg7F/baC157rd6WNBUDBsScZ9CcSDpA0jRJMySdXqLcTpKWSzosde5FSc9ImixpUur8upLukTQ92dcs222IZ3szZgy8/DI8/XS9LWkszjkHHn4Yzj673pY0FblACStW1NuSIMiOpK7AJcBoYGvgSElbFyn3M6DQMoV9zGyEmY1MnTsduM/MhgL3Jcc1IcSzvYlA8avSu7e/HxMmuAJMmODHvXvX27KmYOBAWLIEXn+93pYEQUXsDMwws1lmtgS4ARhToNzXgZuA+RnbHQNcnby+GjikjXZmJsSzvdloI1+2EqH6nFmz4MADVx736QNHHw3/+U/9bGoiYq1n0KB0kzQptY3Nu94fSP/Vzk7OfYSk/sChwGUF2jfgb5KezGt7IzObC5DsN2zrg2SlpuJZbsxbzkXJ9SmSdkhdu1LSfEnP5tX5QzIOPjkZF5+cnB8k6cPUtUIfSG0YMwaeeiq+8QD69YNly/x1164eb27NNT0bTVCWAQN8H/OeQYOxzMxGprbL866rQJ18R5ALgdPMbHmBsp8wsx3wYd+TJO3ZdpPbRs3EM+OY92hgaLKNBSakrl0FHJDfrpl9IRkHH4F3929OXZ6Zu2Zm46r1LBUTgeJXJffNv8ceMG5cOA1VQPQ8gyZlNjAwdTwAmJNXZiRwg6QXgcOASyUdAmBmc5L9fOAWfBgYYJ6kfgDJPutwb5upZc8zy5j3GOAacx4D1s69MWb2EPBWscYlCfg8cH27WN8WttjCt5j3dHZIBhTeegsuuQRuvrl0+eAjNtgAevYM8QyajonAUEmDJfUAjgBW6U2Y2WAzG2Rmg4A/ASea2a2S+kpaA0BSX2B/IDcCeTtwTPL6GKBmX7K1FM+yY94ZyxRjD2CemU1PnRss6V+SHpS0R6FKksbmxumX5YYT24MxY+CBByJQPMDUqb6fNSuW8FSI5EO3IZ5BM2Fmy4CTcS/aF4Abzew5SeMklRsV3Ah4WNLTwBPAX8zsr8m184D9JE0H9kuOa0K3Wt2IbGPeWcoU40hW7XXOBTY1szcl7QjcKmkbM3tvlcZ9bP5ygL59+7bfN/lnPgPjx8Ndd8ERR7TbbRoeMxfPnj1hwQLPr7X++vW2qqmItZ5BM2JmdwJ35p0r6ItiZsemXs8CtitS7k1g3+pZmZ1a9jyzjHlnKbMakroBnwX+kDtnZouTNxYzexKYCWzeKsurwa67RqB4gDlzXDT33tuPw8u2YiLKUBDUn1qKZ9kx7+T4y4nX7a7Auzk35DJ8CphqZh/9Hpe0QeKkhKQhuBPSrGo8SKvo2hX+6788UPySJXUzo+7khmxHj/b9rPp9JM3KwIHw6quwvJBPYhAENaFm4plxzPtOXOBmAFcAJ+bqS7oeeBTYQtJsSV9NNX8EqzsK7QlMScbJ/wSMM7OiDkc1YcwYeO89ePDBuppRV6ZN8/0BieN09DwrZuBAX+0zb169LQmCzkst5zzLjnmbmQEnFal7ZIl2jy1w7iZ86Urj8KlPeSSd22/3mLedkalTYY01YPPNfa4zxLNi0ms9N9mkvrYEQWclIgzVkj59YP/9O3eg+KlTYcst3W108OAYtm0FsdYzCOpPiGetGTPGv/UmT663JfUhJ57g4hk9z4oJ8QyC+hPiWWsOPtizGndGr9sFC/wbPyeeQ4bASy+F50uFrLuuj/6HeAZB/QjxrDUbbAC77dY5xfPf//Z9uue5bFksWqyQXKCEeNuCoH6EeNaDMWN82HaXXTpXXNfcMpW0eEIM3baCWOsZBPUlxLMefOYzvp84sXMlg5461de7braZHw8Z4vsQz4oJ8QyC+hLiWWt69/Yg8eAet50pGfTUqd7b7NnTjzfd1Od/QzwrZuBAD9bUnuGYgyAoTohnrZk1C446CrolS2x79+48yaCnTVs5ZAvQvbtP3sVylYoZMABWrOhco/5B0EiEeNaafv08+XPOw7SzJINevtwdhtLiCT502xl+OFSZWK4SBPUlxLMezJsHn/+8v953387RfXj5Zf+hkC+esdazVYR4BkF9CfGsBzffDNdc40O3O+3UOZJB53va5hg8GObOhQ8/rL1NTUwuRF+IZxDUhxDPetGjhwvJM8/U25LaUEw8cx63L75YU3OanbXXhr59Y61nENSLEM96MmwYPPtsva2oDVOneiD49dZb9Xys9WwVUixXCYJ6EuJZT4YP9x7X++/X25L2Jx3TNk1OPMPjtmJCPIOgfoR41pNhw3z/3HP1taMWFBPPjTeGXr2i59kKBgwI8QyaB0kHSJomaYak00uU20nSckmHJccDJd0v6QVJz0n6ZqrsWZJelTQ52Q6sxbNAiGd9yYlnRx+6ffttmD9/ZXCINLnUZCGeFTNwoDtqL11ab0uCoDSSugKXAKOBrYEjJW1dpNzPgLtTp5cB3zGzrYBdgZPy6v7SzEYk2yr5otuTEM96MmiQe310dKehadN8X6jnCZHXs5UMHOhBqubMqbclQVCWnYEZZjbLzJYANwBjCpT7OnATMD93wszmmtlTyev3gReA/u1vcmlCPOtJly6wzTYdv+dZzNM2Ry5QQmdNEN5KYq1n0EB0kzQptY3Nu94fSP+lziZPACX1Bw4FLit2E0mDgO2Bx1OnT5Y0RdKVktZpy0NUQohnvRk+vOP3PKdO9aU5gwYVvj54MLz3ng/vBpnJrfWM5SpBA7DMzEamtsvzrqtAnfxfyxcCp5lZwQS/kj6G90q/ZWbvJacnAJsBI4C5wPmttL9iMounxGiJOySelxiYnPtviX3bz7xOwLBh8PrrPifYUZk6FYYOXRnPN5/wuG0V0fMMmojZwMDU8QAgf8JhJHCDpBeBw4BLJR0CIKk7LpzXmdlHUWXMbJ6ZLTezFcAV+PBwaaTRSHcgPY80MDn330gVaVkm8ZQ4GrgRmA4MBronl7oC38t6s3LeVnIuSq5PkbRD6tqVkuZLejavTlFvK0lnJG1Nk/TprHbWlOHDfd+Re5/FPG1zRGqyVrHmmr6FeAZNwERgqKTBknoARwC3pwuY2WAzG2Rmg4A/ASea2a2SBPwWeMHMLkjXkdQvdXgoUHoOTKqKlkH2nuf3gOPNOAX3fMrxGN5dLktGb6vRwNBkG4t3yXNcBRxQpPnVvK2Sto8AtknqXZrY0Fh0dI/bpUth5szS4hmBElpNrPUMmgEzWwacjHvRvgDcaGbPSRonaVyZ6p8AvgR8skAnabykZyRNAfYBTinT1veA4zFrtZblKDKOthpDgUcLnF8ArJmxjY+8rQAk5bytnk+VGQNcY2YGPCZpbUn9Em+rh5LJ4qyMAW4ws8XAfyTNSGwo9Bz1Y8MNPfJOR+15zpzpSSdLieeaa8K668awbSsYMCDmPIPmIOnY3Jl3rqBzkJkdm3r9MIXnTDGzL1VoRjW0DMje85wDbF7g/J7AzIxtlPW2ylimEIW8rTK1JWlszkNsWT0yC0s+dNtRe57lPG1zNGtqsrlzYa+96pYZJ3qeQVAR1dAyILt4Xg5cJPGJ5HigxDHAeFYdWi1FFm+rLGXyKeZtlaktM7s85yHWrZhDS3szbJhHGVqxoj73b09y4rl5ob/XFM0aKOGcc+Dhh+Hss+ty+4EDPcPd4sV1uX0QNBuXAxchfaRlSJVqGZBx2NaM8RJrAfcAvYD7gcXAL8y4JOO9snhbZSmTZ5vNy72WdAVwR2vbqhvDh8OCBfDSSyvn/zoK06bBJpv40GwpBg+GW2/1pNldG29qejV69/b8pDkmTPCtV6+aplfLedy++upKv6sgCIpgNh6poJZhllXLgAqWqphxJrA+Pm+4K7CBGT+s4F5lva2S4y8nXre7Au+a2dxSjZbwtrodOEJST0mD8bHuJyqwt3Z0ZKehcp62OYYMceeiZgmXM2sWHHXUyuU33bvD0UfXvPccaz2DoELMVtMyzCrRMiD7UpUrJdYwY6EZk8x4wowFEn0lrsxmbyZvqzuBWcAMfM3OiStt0PX4RO8WkmZL+mpyqaC3lZk9h7skPw/8FTip2OLburPNNr7vaE5DZtnFs9k8bvv18950bp586VL42Mc80H0NibWeQVAB0pVIa2C2ELNJmD2B2QKkvkiZtCxH1km+Y4DTgfzcWb2BLwPHZWmknLdV4mV7UpG6RxY5X9TbyszOBc7NYltdWXNNaGnpeD3P+fPhnXcqE89Zs2DPPdvVrKrx2ms+xNy/P7z8MkyeXHMTQjyDoCKqomVQpucpsa7EerjzzTrJcW7bADgYmFeqjSAjw4Z1vJ5nVk9b8B8PUvP0PMGdhZYvh7POchVbY42am9C3L6yzTohnEJREWhfpIy1LjnNbq7SsXM/zDdxD1Vh1PWYOA35cyQ2DIgwfDnffDUuWeBzYjkAl4tmjh0/gNZN4Tpzo+113hXHj4Mwz3UGqUOq1diTWegZBWaquZeXEcx9cqf8OfA54K3VtCfCSWYN6sDYbw4b5/Nn06SvnQJudqVO9a9Q/Y/agZktNNmmSz3NusQV89aveA730UvjVr2pqRqz1DIKylNUyzCrSspLiacaDABKDgVfM6IALERuEdIzbjiSem2/uqdeyMGQI/O1v7WtTNZk4EXbc0Z9vo43g8MPhqqvg3HNdVGvEwIHwRGP6kQdBY2D2IAC+8uIVPJB8m8j0rWbGS2askNhEYleJPdNbW40I8N5L164dy2koq6dtjsGDfalKev1ko7JkCTz9NOy008pzJ53kqdWuu66mpgwYAG+8UdPlpUHQnJi9hNkKpE2QdkXac5WtAjJ520psAvweD2FkePc3Ha2nCVa1Nzg9e7qAdhSnoQ8/9KAPX/lK9jo5j9uXXqr5vGHFPPusC+jIkSvPjRoFI0bAJZfA2LHuAFUD0oESPv7xmtwyCJoTqWpaljVIwoXAcjwbykJgD+BwfL1msUwnQaUMG9Zxep7Tp/s6z0p7ntAc8545Z6F0z1Py3uczz3jIvhoRy1WCIDMXUiUtyyqeewGnmTEVV+nXzbgZOA04p5IbBiUYNsyFY8GCelvSdirxtM3RTHk9J03yNSL54RSPOgrWWssdh2pEiGcQZGYv4DTMPtIyPLl2xVqWVTx7466+4F5KGyavnwe2reSGQQlyTkPPF/KkbjKmTvWe2NCh2etsvLEPXzeDeE6c6EO2+UOzffr4UPVNN9Us00rOmTnEMwjKUjUtyyqeU4FcF2IyME6iBY8G9GolNwxKkItx2xHmPadOhUGDPIB6Vrp08TqNPmz74Yc+vJ4esk1z4okeru+KK2piTp8+sN56sdYzCDKwmpYhtUrLsornr4Bc0M6zgf3xGLQnAt+v5IZBCYYMcbHpCPOelXra5miGvJ6TJ3tkobSzUJqhQ2H//eHXv14Z+7adibWeQZCJqmlZ1qUq15lxVfL6KWAQsBOwqRl/rOSGQQm6dPE1ns0unitWeKSd1ohnM+T1nDTJ98V6nuCOQ6++CrfdVhOTQjyDIANm12F2VfJ6FS3DrCIty5ySbNX7szAR0Q8kTm9NG0ERhg9v/mHb2bNh4cLWLTcZPNiDyb/9dtXNqhqTJnlQhFKRkw46yOP1XlJRisBWM2BAiGfQ2Eg6QNI0STMkFdUNSTtJWi7psHJ1Ja0r6R5J05P9OhUZ5dlVngI+oIRNhSgrnhLrSxwksb/ka2Akukt8C3gROLUiY4PSDBsG8+bB66/X25LW0xpP2xzN4HE7caL3Okut4+za1ePd3n9/TRzABg703xsffNDutwqCipHUFbgEGI0vEzlS0tZFyv0MT12Zpe7pwH1mNhS4LzkuZsT6SAch7Y+3CVJ3pG/RCi0rl1VlN2A68GfgLuCfElsCU/DcnOcAm1Zyw6AMOY/bZh66nTbN960dtoXGFc/33/cfB8XmO9N89ase8H7ChHY3K7dcJZyGggZlZ2CGmc0ysyXADcCYAuW+DtwEzM9YdwxwdfL6auCQgneXVtMypDZpWbme5zn4L4Bt8YnWnYE7gJ8CQ8242IyFldwwKEPO47aZxXPqVFh7bdhww7JFV6PRAyU89ZQHfyg135ljgw3g85+Hq6920W1HYq1nUGe6SZqU2sbmXe8PpP86ZyfnPkJSf+BQ4DJWpVTdjcxsLkCyL/alU1LLMLsYs4q0rJx4bgecY8azwA/wRaVnmHGN2SohjYJqsfHGsO66zT3vmfO0bU14urXX9uADjdrzzDkL7bhjtvInneTCee217WcTPucJIZ5B3VhmZiNT2+V51wt9GeRryIXAaWa2vBV1y7EdcA5mq2gZZtdg1iotKyee6wKvgzsJ4eGM/tWaGwUZkXzottl7nq0Zss3RyB63kyZ5N2+jjbKV32UX2GEHdxxq3f9oJnLiGcO2QYMyGxiYOh4Aq6WzHAncIOlF4DDgUkmHlKk7T1I/gGSfHu5N85GWJT3MNmtZFm/bdSTWlVgPV+s1k+OPtrYYEBQgF+O2Hb9s24333vPMKG0Vz0Ydts05C2UlF+/2uefgoYfazayePX2UPHqeQYMyERgqabCkHsARwO3pAmY22MwGmdkg4E/AiWZ2a5m6twPHJK+PAUqtDVsHaV2kj7QsOV65VUAW8XweV+z5wMeSB3k92d4gp+YZKOeqLOei5PoUSTukrl0pab6kZ/Pq/FzS1KT8LZLWTs4PkvShpMnJlj+O3rgMH+5DfS+/XG9LKqctzkI5hgyBF1/09aKNxNtvw8yZ2ZyF0hxxhA9Ft/OylVjrGTQqZrYMd8y5Gw/CfqOZPSdpnKRxrambXD4P2E/SdGC/5LgYVdMyKJ+SbJ9KGitFyt14P7wbPlHS7WaW9uMfDQxNtl2ACcke4CrgYuCavKbvAc4ws2WSfgacgQf5BZhpZiOq9Qw1I+001NJSX1sqpS3LVHIMHuzpvubOLb2WstZkCY5QiFy824su8l75JptU3zZ86HbGjHZpOgjajJndCdyZd65gp8bMji1XNzn/JrBvhttXTctylBRPMx6s4r0+cjcGkJRzN06L5xjgGvMJ3MckrS2pn5nNNbOHJA1a3Ub7W+rwMXysvLlJx7g96KD62lIpU6dCt24r12u2hrTHbSOKZ1ZnoTQnnAAXXODxbn/84+ralTBwIDzwQLs0HQTNjVk1tQxoZYShVlLWVTljmVIch6/hyTFY0r8kPShpj0IVJI3NuVcvq1Ec0rKstZZ/Ezaj09DUqbDZZtC9e+vbaNRACRMnerbpdSoLYgJ4vQMO8Hi3S5dW3zb8T+bdd9t9VUwQBNRWPLO4G7faJVnSmcAy4Lrk1FxgUzPbHvg28HtJa67WuNnlOffqbt3KjWLXkGHDmnO5Smtj2qZpaXFHm0YTz0mTKp/vTHPSST4UfeutVTMpTSxXCYLaUUvxzOKqnKXMakg6BjgYODoZ8sXMFifj4ZjZk8BMYPNWW19rhg/3Xlw79VLahWXLYPr0totnz54+L9hIHrfz5rkqtUU8R4/2lGsXXAB77VX1fJ8RKCEIakctxbOsq3Jy/OXE63ZX4N1c9IhiSDoAdxD6jKUiREjaIHFSQtIQ3Ampgb6NyzBsmDvNTJ9eb0uy8+KLbnNbxRMaLzVZa52F0nTt6nOfjz0G//gHnH12dWxLiBB9QVA7aiaeGV2V78QFbgZwBZ5jDQBJ1wOPAltImi3pq8mli4E1gHvylqTsCUyR9DS+Zmicmb3Vvk9ZRZoxxm01PG1zNFqghEmTfCh5++1b30bv3nBa4ghu5jFvpWwJw+fOLdtb3WQTby56nkHQ/mSa5JO4ssglAxbhYvcHs9JDrOVclZMh15OK1D2yyPmPFzl/Ex5guDnZckvvqTz7rMdHbQZy4tmaVGT5DB7s+TAXL/Zh3HozcSJstRWssUbr25g1C049FW68cdUk2StWeBSi7bbzbdttfb/eeivLnHMOPPyw91YvvbRg8z16eOCjEM8gKIKUScswKztdmNVDZgNgD2AFkOsKDcMdfJ4EPgucLbGHGZMzthmUolcvGDq0uZyGpk71b+/WeKPmM2SI985eegk2r/NUtZn3PD/96ba1068frLmmi2WvXv7DYJ99XDiffhruvBOuumpl+f79vceZDhYxYYJvvXrBhx+udosIlBAEJcmkZUh7YDa5VENZh23/iS8BGWDGnmbsiTvz3An8DWgB/gKcX9lzBCXJhelrFtoa0zZNI6Ummz3bHYbaMt+ZY948z/P52GM+/7nWWvDzn8Pf/ubXXnsN7r4bxo93Yd1881UD7PfpA0cfXfR9GTgw5jyDoAQfaRlme2LWai3L2vP8JvDJdPoxMxZKnAvcZ8Z4iZ8B91b2HEFJhg2Dm27yDMd9+9bbmvJMnQqf+1x12mqk1GQ5Z6G2eNrmuPnmla8LhevbaCPYf3/fcowd68EVunaFRYu897rxxgWbHzjQddisdUltgqCD803gk6ukHzNbiHQucB9m4/FIdWW1LGvP82NAvwLnN06uAbxHdjEOsjB8uH8LvvBCvS0pzxtvwJtvVq/nuckmPonXCD3PSZM8atJ229Xn/m+84blBd97Ze60lnIYGDIAFCzxYQhAEq1E1LcsqnrcAv5U4XGKQRIvE4cBvgdxP6Z2Bf2dsL8hCOkxfo1ONgPBpunTxNZGNIJ4TJ/pnkcUrtj24+Wb3tH3rLe+tpnuvecRazyAoyS3Ab5EORxqE1ILUKi3LKp7j8CUm1+LBBmYlr//KyuUkLwDHZ36EoDybbeaOIc0w71nNZSo5GmG5Ss5ZqBpDtm2hpcWz7JRJUxdrPYOgJFXTskzDrMlc5ziJ7wCb4Z5JM8z4IFVmcnb7g0x07Qpbb90cPc+pU13oN920em0OGQJPPFG99lrDrFmeiqwazkJtoaXFvWtff90TdxYhep5BUAKf6xyHtIqWYfZBqszkLE1VNEeZiOWUSuoEbWT4cPcAaXSmTnXP0K5dq9fm4MEuXO++616p9aCazkJtIfej5KWXSopnv34+4h3iGQQlcLFsk5ZlGraV6CVxmsTfJCZLTElvbTEgKMOwYb7W7803621Jaaq5TCVHIyxXmTjRgzTk5p/rRS6va5kE6d26uYCGeAZBAaReSKch/Q1pMtKUVbYKyNrzvBQ4FPgj8AgZM50EVSCdGHuvveprSzEWL/bhzaOOqm67udRks2bBiBHVbTsrkya5l22PHvW5f46ceL70UtmisdYzCIpSNS3LKp6HAIebxTrOmpOOcduo4jljhkfB6Wg9zxUr4Mkn4Zhj6nP/NGuv7aEBM4rn00+3v0lB0IQcAhyOWZu1LKu37UJWTVId1IpNNvEvzkZ2GqpmTNs066zjc531Es9p03zRZL3nO8EjHmy6aSbxHDDAh23LOOYGQU2RdICkaZJmSDq9wPUxkqYkCT4mSdo9Ob9Fci63vSfpW8m1syS9mrp2YBkzqqZlWcVzPPBtqaYpzALwL83hwxt7ucrEib5vD6eeIUPqF2WoUZyFcuSWq5Rh4EB3zH2reXIIBR2cJD3kJcBoYGvgSElb5xW7D9jOzEYAxwG/ATCzaWY2Ijm/Iy6At6Tq/TJ3PUk+UorxwLeR2qxlWRvYD/gC8B+JuyRuT29tNSIoQy7GbaN2JW5J/o7Pb4fQxm1Z65khjVdJJk70WLJbbdW6+tWmpSXzsC3EvGfQUOwMzDCzWWa2BLgBGJMuYGYLksxaAH0pPB+5LzDTzMr/IxTmIy1Dugvp9lW2Csgqnm/gSv934DXgzbwtaE+GD/flGo30bWjmXqgS/DsJxlFJfsqsDB7sSbbTmUWycvbZK9N4tYZJkzzjSTWX37SFlhbvTi5YULJYrPUMGpD+rDpcOjs5twqSDpU0FQ/OflyBdo4Ars87d3Iy3HulpHIpnaqmZVmDJHylkkaDKpP2uM19M9aLV1+Fq6+G3/0OlizxtRFmsHy599IOPRR+8Yvq3W/wYA+G/tprPv+bhd69vU6OMmm8CrJ0KfzrX575pFFIe9xus03RYgMG+D7EM6gh3SRNSh1fbmaXp44LpSlYrWdpZrcAt0jaEzgH+NRHDUg9gM8AZ6SqTEjKWbI/n8Kim7tB1bQs5jCbgVrGuC001Ll4MfzpT3Dgge60cuaZ/g19zTXuiWrmwlQm40eryC1XqWTodsaMlQoC3nM86qjK2nj+eX+eRpnvhFUDJZRg4439N02IZ1BDlpnZyNR2ed712UD6l/8AoGjCaTN7CNhM0vqp06OBp8xsXqrcPDNbbmYrgCvw4eGaULTnmQQ/2MuMtyWeocR6GDO2bQ/jgoR11vHEyLVwGjrnnJVDnSecAFdeCf/3fx6koX9/OOMM+MpXPO4u+HznuHGeNuvyy118q0l6uconPpGtzsUX+xC35MK5bJl7zlYi6jlnoXqH5UuTMVBC167eSW+kUf6g0zMRGCppMPAqPvy6ysJwSR/H5zNN0g5AD1YdSj2SvCFbSf3MLPelcygrE1ynC00B9sLsbaSSWoZZZi0rNWx7E7A4ef2nrA0G7cSwYe3b8yw21Alw+OFw3HGw336rz/+Vy0/ZVgYN8n1Wj9vf/Q7OO8/rjR7ton7kkb5e89pr4YtfzNbOxInuPZz7kdAI9OsH3btXtFwlCBoBM1sm6WQ8KHtX4Eoze07SuOT6ZcDngC9LWgp8CHwh50AkqQ/u7PO1vKbHSxqBC+KLBa5De2mZmdVsAw4ApgEzgNMLXBdwUXJ9CrBD6tqVwHzg2bw66wL3ANOT/Tqpa2ckbU0DPl3Ovj59+ljDcuqpZj17mi1d2j7tz5ljts8+ZpIZ+H7HHc2efbZ97lcJm2xiduyx5cs98IBZ9+5m++1ntmTJyvOLF/uz9ehh9vDD2e65445mn/xk6+xtT4YMMTvyyJJFrr3WrE8f/xhbWvy4HNde62Wl9qtTi3u0tk7QNoAPrIZa0ghbLYWzK54CZgjeHX8a2DqvzIHAXYmI7go8nrq2J7BDAfEcnxNi4HTgZ8nrrZN79AQGJ/fuWsrGhhbPq67yj2vkSLO5c6vb9htvmH3pS94+uMh06WJ2wgnVvU9r+cQnzPbaq3SZadPM1lnHbKutzN5+e/Xrb75pNnSo2frrm82cWbqtRYtchE87rbUWtx/77GO2225FL6eFM7f16VNaQGpRp1HtCqpDZxTPTN62EusC5+JrbDYkz9HIjDUzNPPROh9vU7l1Ps+nyowBrkk+jMckrZ0b0zazhyQNKtDuGGDv5PXVwAPAacn5G8xsMfAfSTMSGx7NYGvjkQvT9+STPh956aVtb9MMbrgBvvlNz16yxRbuLHTiie0zf9laBg+GBx8sfv2tt+Dgg31I+Y47PCJTPuuu69d23dXLPvpo8aAOU6a4t20jOQvl2HRTuLd4ZLEzz4SFC1c9t3ChT18/9VThOldc0f51anGPUnXOPBOOPrpwnaATIZXUMsyyaBmQPbbtb4HtgctxD6nWrNYvtM5nlwxl+gOlvsU3smTC2MzmSsrla+oPPFagrVWQNBYYC9Cj3sG/i5GejzRr3dKLfF5+2b+F7rwTdtnFv3VyAg3tM3/ZWoYMgeuu86Ux+Z/RkiXw2c/6PODf/77SO7cQm28ON90E++8Pn/88/OUv7paaTy5iUiM5C+VoaYE5c1zcu3df7XIxX6L33/ffQ4Uotmy0mnVqcY9SdTIEZgo6B9XQMiC7eO4L7GfG4629EdnW+WRaC1TF+2HuUn05QN++fRszhM+sWXDqqXD99S6euZB9F19ceVvLl3uv9YxkqdSFF8LJJzdOIIBCDB7sz/3yy/Dxj688b+aevg8+6M5AWbxx99kHLrsM/vu/vcd98cX+fqaZNAnWX7+6ib2rRUuLP/fs2Ss9kVMUC3/b0uKxJgoxaFD716nFPUrVacSPMqgL+wL7YdYWLQOyr/OcD5QOa1KeLOt8KloLlDBPUj9wt2Xc1ta21Zj06+frJ2FlUIIpU2DPPd0L96yzsoXve+452H13+MY3YI89vM43v9nYwgkrRSLf43b8ePeu/dGPKhuT++pX4bvf9R8RhX6ATJrkvc58UW0EyqQmO/dcj1WRpk8fP1+MWtRpVLuCTkc1tMzJMjEK9gWwm8A+1trJVbyXOwt33sk5DG2TV+YgVnUYeiLv+iBWdxj6Oas6DI1PXm/Dqg5Ds2hmh6FDDzU78USzyZN9f8ABZhddZLbnnis9ZLfYwuzMM83+9S+zFSu83pw5Zrvvbvbtb7sTzHrrufdE7noz8PLL/nyXXbby3E03+bkjjmjdsyxbZjZmjDtG/eUvK88vWODnfvjDNpvdLkyf7s991VVFizSqV2st7dp4Y3+b1l8/nIVqAc3iMARfMLjJoNValtuyiuczYO+DLQR7AWxKest8M/em/Tfu+Xpmcm4cMC55LTzy/kzgGWBkqu71+NznUrxX+dXk/Hp4NP7pyX7dVJ0zk7amAaPL2dfQ4lmKuXPNJkww23df/+IHs802c2/R/fe3j9wOv/hFs/nz621t5Sxbtqr368SJZr17m+26q9mHH7a+3QULzLbf3myNNcymTPFzDz/s79Xtt7fd7vbgww/dvrPPrrclDc3ixb6y6zvfqbclnYMmEs9nDN43WGjwgsGUVbYK2pI/d2kkfly698pPyjbSBPTt29c++OCDepvRNl5/HW67Db72tcLB1NviZFRPhg6F7bf3zC077+xB6R9/HDbaqG3tzp7t7fXo4e1dfz2ccorH8M0aS7fW9OsHBx0Ev/lNvS1paHbf3f8FHnmk3pZ0fCQtNLO+9bajLFJJLcMss5aVdRiS6I6nh7nEjNamgQlqxQYbuDPMQQfB17/uyzMWL26foO21pH9/94597jn44AO45562Cyd4KJ4//9nngA85xEMh9ugBXRo47HPG1GSdnVGj4KKL/M+/Z896WxPUHekjLaP1Kc0+ouw3hBlLgRMo7L0aNCr9+rmQLl3afkHba8nrr/uCveefhxtvXBksvxrsuKN76z72GNx1ly9/aW0as1oQ4pmJ3Xbzj7LYmtCgk2FWVS3L+vP6b8Anq3HDoIbMm+dLOR57zPetTQpdT3r3dq/X51OxNEaPrm7OUFjdW7c9cpNWi0039WU7rclx2okYNcr3jzZnWJSgfaialmVd53kf8D8S2wJPAqtMDJpxc8FaQX1p76DttSC3xvWWW3yutr2Gn2fNgu98x4MoLFnS2MPcLS0+Fvn669UZuu6gbLyxr3J65BH49rfrbU3QINwH/A9SQS3DLLOWZRXP3GK4bxS4Znjc2iCoPrk1rosXt+/wc79+Hq5v2bLGH+ZOr/UM8SzJqFHwwAMrY4sEnZ6qaVmmYVszupTYQjiD9qVWw8/NMsxdJlBCsJLddvNohhGeLwDArEuJrSIty9rzDIL6Uavh52YZ5g7xzEx63jP3tgVBNcgsnklmlQOATfEIQR9hRgO7JgZBB2OttXxIOcSzLNtu69PXjzwCRxxRb2uChsAzqxTUMswya1nWlGS7An/Bs3FvALwK9EuOX4QQzyCoKS0tMRaZgW7dPAZGeNwGAEhV07KsS1V+DlyHp/RahLv6bgpMAn6W9WZBEFSJWOuZmd12g8mTV8/zGXRKqqZlWcVzW+BiDw3IcqCnGfPwpNNnVXLDIAiqQIhnZkaNcifqSZPqbUnnRtIBkqZJmiHp9ALXx0iaImmypEmSdk9de1HSM7lrqfPrSrpH0vRkv04ZM7YFLk6Cny8HemLWKi3LKp5LUq/nAbmp9wVAgwYADYIOTEsLvPMOvPdevS1peHbd1fcR47Z+SOqKJ/0YDWwNHClp67xi9wHbmdkI4DggP3jzPmY2wsxGps6dDtxnZkOT+quJch5V07Ks4vkUsFPy+gHg/0kcA1wETKnkhkEQVIFcdueY9yzL+uvD5pvHvGed2RmYYWazzGwJcAMwJl3AzBbYykwlffF1l+UYA1ydvL4aOKRM+dW0DKlVWpZVPM9kZSLpHwCvA/8LrAOMreSGQRBUgViuUhG77eY9zwxJpIL2oT/wSup4dnJuFSQdKmkq7tRzXOqSAX+T9KSktOZsZGZzAZL9hmXsqJqWZfK2NWNS6vXreNc7CIJ6EeJZEaNGwVVXwcyZ8PGP19uaDkm39FwkcLmZXZ46LhTfabWfMmZ2C3CLpD2Bc4BPJZc+YWZzJG0I3CNpqpk9VLGVZpNSr9ukZRXlXZIYKfEFib7JcV8pAi0EQc3ZaCNPnRbimYnddvN9zHu2G8vMbGRquzzv+mxgYOp4ACt7gKuRCONmktZPjuck+/nALfgwMMA8Sf0Akv38TNZKI5G+gNQ3Oe6LVJGWZRJPiY0kHgeeAH4P5AJqXgCcX8kNgyCoAl26+LxniGcmtt7a40rEvGfdmAgMlTRYUg/gCOD2dAFJH5c8ArGkHfAABm9K6itpjeR8X2B/4Nmk2u3AMcnrY4DbSlohbYRUFS3LqrS/BF4D1gPSHgp/xMeLgyCoNbnUZEFZunRxr9voedYHM1sm6WTgbjz4+pVm9pykccn1y4DPAV+WtBT4EPiCmZmkjfChXHDN+r2Z/TVp+jzgRklfxbXp8DKmVE3LZBlm0CXmAfua8azE+8B2ZsySGAw8a+bDuM1O37597YMPPihfMAgageOOg7vvhldfrbclTcFZZ8E558Dbb3svNKgekhaaWePrgDQP2BezZ5HeB7bDbBbSYOBZKniGrHOevVl1fUyODfAoDZnIsEhWki5Krk9Juu4l60r6Q7JwdnKykHZycn6QpA9T1y7LamcQNAUtLTB3rucfDcqy226eP/yJJ+ptSVBHqqJlkF08HwKOTR2bRFc8KsN9WRrIuEh2NDA02cYCE8rVNbMvJAtnRwA3wSqJuWfmrpnZuIzPGgTNQUuLr7145ZXyZQN22cVzesa8Z6dmNS3D9SWzluXIKp7fA46XuAfoiU+sPg98AjgjYxtlF8kmx9eY8xiwduJBVbZuMtH8eeD6jPYEQXPTWQIlzJ0Le+3V5vyqa60F22wT856dnO8BxyO1RcuA7MmwnweGA48AfwN64ROs25sxM+O9siySLVYmS909gHlmNj11brCkf0l6UNIehYySNDaJozhp2bJlGR8lCBqAzrLW8yc/gYcfhrPbnrxp1CjPdb5iRRXsCpoPs6JahllWLQMqyOdpxmvAj9PnJFokbjTj8xmayLJItliZLHWPZNVe51xgUzN7U9KOwK2StjGzVYKBJuuRLgd3GCphfxA0FgMH+jhkRxXP3r1hUWoaasIE33r1gg8/bFWTu+0GV1wBU6f68pWgE2K2mpYhtSDdiFkWLQMqDJJQgLVx9+IsZFkkW6xMybryxa2fBf6QO2dmi83szeT1k8BMYPOMtgZB49OjB/Tr13HFc9YsOOiglcddusBRR8F//tPqJkeN8n3MewZ5rE12LQPaLp6VUHaRbHL85cTrdlfg3SReYbm6nwKmmtns3AlJGySORkgagjshzWqvhwuCutCRU5P16wf//re/7tbNx1pnzICNN251k5tvDuuuG/OeQdupmXia2TIgt0j2BeDG3CLZ3EJZ4E5c4GYAVwAnlqqbav4IVncU2hOYIulp4E/AODN7q10eLgjqRUcOlDBrFkyfDttuCxMnwtChvs7k4Ydb3aTkvc/oeQZtJVOQhKKVxXbAU2Z0rZ5J9SOCJARNx2mnwYUX+hxgl1oOJNWAr33No7n/5z+wySaeu3SHHXxd6+TJ3oVsBeeeCz/4Abz5ZqubCPJomiAJxZC2A57CLLOWlXQYklYbVs0n4nQEQT1paXExmTfPhzk7CrNnu3Aed5wLJ3hYoBtucK+fr34Vbr7Zu5IVkgsS//jjMDryQ3UOpKprWbmfqm+W2f4DXFPpTYMgqBIddbnKL34By5fD97636vmRI+G88+DWW+HSS1vV9E47eSc95j07FVXXsjYN23Y0Ytg2aDqefRaGD/ce2Re+UG9rqsP8+TBoEBx+OFx99erXV6yA//ovuO8+7z5ut13Ft9hhBx+yvffetpsbdIBh21bQwSZJgqCT0RGjDF14oa/vPKNIwJcuXXxId911/QdDK37wjhrlurt8eZssDToxIZ5B0MysuSasvXbHGbZ9+224+GI47DDYcsvi5TbYAK67zpeyfP3rFd9mt91gwQLvuAdBawjxDIJmpyOt9bz4Ynj/ffj+98uX3Wcfd5v93e9cSCsggiUEbSXEMwianY4ingsW+JDtwQfDiBHZ6vzoR7D77jBunAdQyMjgwbDRRuE0FLSeEM8gaHZaWjrGnOdll8Fbb8GZZ2av060b/P730L27z38uXpypWgRLCNpKiGcQNDubbgrvvutbs7JoEZx/Puy7L+y6a2V1Bw70odunniruZFSA3Xbzzur8+RXaGgSEeAZB89MR1npeeaXn66yk15lmzBh3HPrlL+GOOzJVyc17PvZY624ZdG5CPIOg2Wl28Vy6FH72M1ezvfdufTvjx/tc6bHHei+0TALtHXf00d6Y96wNkg6QNE3SDEmnF7g+RtIUSZOTHMu7J+cHSrpf0guSnpP0zVSdsyS9mtSZLOnAWj1PiGcQNDvNLp7XXutztj/4QavC7X1Er14eLGLRInc6KpNAu3dv2H77mPesBUmGq0uA0cDWwJGS8jOq3gdsZ2YjgOOA3yTnlwHfMbOtgF2Bk/Lq/tLMRiTbne35HGlCPIOg2dlgA+jZszmdhpYvh5/+1FWsGoFmR4zwoAlz53okogkTXJB79y5YfLfdPGHL0qVtv3VQkp2BGWY2y8yWADcAY9IFzGyBrQx51xew5PxcM3sqef0+nlmrf80sL0KIZxA0O126uNNQM/Y8//hHTzv2/e+3rdeZY9YsOPLIlRlmeveGo48umkB71ChPSPP0022/dSenWzLUmtvG5l3vD7ySOp5NAQGUdKikqcBf8N5n/vVBwPbA46nTJyfDvVdKWqetD5KVEM8g6Ag041rPFSvgf/4HttoKPvvZ6rTZrx+stRbkOjCLFnkUpiIJtHMZVmLes80sM7ORqe3yvOuFfhmtFljdzG4xsy2BQ4BzVmlA+hhwE/AtM3svOT0B2AwYAcwFzm/TU1RAiGcQdASaUTzvuAOeecaXl1QzF+m8eXDCCb7kpWdPePXVokUHDPAt5j3bndnAwNTxAGBOscJm9hCwmaT1ASR1x4XzOjO7OVVunpktN7MVwBX48HBNKJnPMwiCJqGlxT1LFy1yx5lGxwz+3//zUD9HHlndtm9Ovlv/8Q/Yc09fO1qC3XaLnmcNmAgMlTQYeBU4AjgqXUDSx4GZZmaSdgB6AG9KEvBb4AUzuyCvTj8zm5scHgrULFpx9DyDoCOQy64ye3Z97cjKvfe6p87pp3uUoPZgjz1cPMePLxl5aNQo97Uq0UEN2oiZLQNOBu7GHX5uNLPnJI2TNC4p9jngWUmTcc/cLyQORJ8AvgR8ssCSlPGSnpE0BdgHOKVWzxT5PFNEPs+gaXngAQ+Ufu+9ZXtaDcFee8HMmb717Nl+97n3XthvP/e6HTeuYJEnnoBddnHfpcMOaz9TOjKRzzMIguakWdZ6zp3ry0keegi++932FU5YGe7vvPOKrkcZMcJHumPeM6iEmopnhggTknRRcn1KMu5dsm6pCBOSzkjKT5P06fZ/wiCoEwMGuNNNo4vnOef4upBeveD449v/fhL88If+vlx7bcEif/yjT8FecAEMGpQtu9l113nZLl0aq05r7hG0EjOryQZ0BWYCQ/CJ4KeBrfPKHAjchbs17wo8Xq4ucBZwaoH7bZ2U6wkMTup3LWVjnz59LAialgEDzI49tt5WFKZXLzPXqFW3Xr3a/94rVpjtsIPZxz9utnTpKpeuvdasT59VTerd2+yyy8zeeKPwdtllXqbR6hQq36ePP2N7A3xgNdKSRtlqNucpaRRwlpl9Ojk+A8DMfpoq82vgATO7PjmeBuwNDCpWV9JZwAIz+0Xe/VZpX9LdSRtFB2dizjNoaj7xCR8G/fvf623J6syd6ynD/vEPP+7d29d2/uIXRddgVpVbbvH7XXutB01IGDSo8TvrbaWlBV58sX3v0RnnPGu5VKVQhIldMpTpn6HuyZK+DEzCYyC+ndR5LK9OoYgWY4GxAD169KjgcYKgwWhpgccfL1+uHvz97yuFs2dP934tEbyg6owZA8OGwbnnrhKBqFREw4suKnz+G99ozDrFyjdj1MZmoJbimSXCRLEypepOwCNRWLI/Hw/rlDWixeXA5eA9z0KGB0FT0NICf/qTR+6pZtCBtnL55e7puv76cMghcPLJfm7u3LJVq0aXLh54/ogjfB1o4lZbLKphS4tnOCvE+ec3Zp1i5XOrmILqUsv/sCwRJoqVKVrXikeYqCiiRRA0PS0t7lFaS1EqxwUXwNe+5kHfX34ZrrgCttsOLrlkZTCDWnHYYbDFFh6cIZmuOvdc6NNn1WJ9+vj5YjRqnULle/YsfY+gDdRqchXv5c7CnXdyTj/b5JU5iFUdhp4oVxfol6p/CnBD8nobVnUYmkU4DAUdmTvvdC+RRx6ptyXupPOTn7g9hx9utnhxvS1yrrnGbbrtto9OXXutWUuLmeT7LA42jVonXb5rV7OhQ8vfoxrQCR2Gansz96b9N+75emZybhwwLnktPLLETOAZYGSpusn5/0vKTgFuzxPTM5Py04DR5ewL8Qyammef9X/p66+vrx0rVpideqrbcswxq3m41pWlS82GDDEbOdLt7MBceKF/BI891v736oziGRGGUoS3bdDULFgAa6zhAQFOO60+NqxYASedBJdd5vuLLmqs+VeA3/zG15jedRcccEC9rWk33n8fBg6E/feHG29s33t1Rm/bBvurDoKg1XzsY7DuuvVbe7FsGRx7rAvnaafB//5v4wknwJe/7F4055yzMnVZR2LuXNhrL9b44DXGjoWbbiqazjRoAw34lx0EQatpaWn92oTkS5fXXqu87uLFvo7z//7PHXJ++tPqJLduD3r0cHF/5BGPCdzROOccePhhOPtsvvEN//3yq1/V26iORwzbpohh26DpOeQQD7b+zDOV1z3hBF9CMnasB1LPwty58PnPQ/fucP/9cOGF8M1vVn7vWrNoEQwZAltu2ZhBJVpD797+XHks6dKL9fp8yCuvwNprt8+tY9g2CILmJpcUu5Ifxb17ey/xsst8zvKyy/y4SxfvTX772x4J6Pe/957a9OmQ+5H5wx96L+f++30usRmEEzy27ne/63b/85/1tqbtLFzo6d3yc7l+4hNMu2sWCxb4KqGgekTPM0X0PIOm54IL4DvfgbfegnXWyVZn7lz4zGdg0iQ/7tbNA8336wdvvOGJLhcuzNZWr17w4Yets73WLFzo8fl22AH++td6W9M6PvjAf+z8/Ocwb55/Zq+95kPTuRymhxzC5+ZP4ImXN2bWLB8kqDbR8wyCoLlpTWqyd9+Ff/3LX/fq5b3P0aN9TvDf/3Yv3nffhRde8PyY11wD3/8+bL45dO3q9fr08ZixzeSZ0qeP/9C4+25PzN1MLFjggjl4MJx6qocefPBBT792wgkepnHcONhmG7jrLq6fsjV7zb6WG/8QnaWqUe+1Mo20xTrPoOmZONHygwCUZMkSX/PYvbuvyZw82ezEE80OPbR83XHjzLp08cwoXbqYnXBCm0yvC++9Z7buumaf+UzldefMMdtzT7O5c2tX5733zH76U7P11/fPeb/9zB5+uHT9F16wFaNGmYE9uObBtmL2q9nvnRE64TrPuhvQSFuIZ9D0zJvn/9YXXZSt/I9+5OX/9KfK73XooS60lQhuI3L22f4e7LBDdlFbvtzTv0lmRxxh9vzzHqRiyhR/P556yn/IPP642aOPmv3zn2b/+IfZmDFe56ijPI/YsmXl73XCCf7jZOedzdZbz2094IDKIkktW2aPHn6+LaSXLfnY2ma/+11Vg0R0RvGMOc8UMecZND1m0LcvnHiiO/mU4vHHPY3ZUUf5UGxn5Z13YMMNPS7w174GZ53l88Bz5vg+t+WOn3iiuvdfYw2fn15nHXeHzb2+5hpYvnz18un5zApYtAj22mQ6V3Ic27z9sA/N//rXHklh7lwPmv+HP7Qq002WOU9JBwC/wvMz/8bMzsu7PgZP7rECWAZ8y8weLlVX0rrAH/C0lS8CnzfPqtXuhHimCPEMOgRbbgnDh8Mf/1i8zAcfwPbb+5fwlCmw1lq1s6+RKLK8YzXWW8+dcfr1c4F77jn3Ol661MVsp53gmGM8SEWXLiu3rl19//bb7o38z3/6e96jh39Ge+/twSXeecfLvP32ytdvvrmq81XPnh7cvg05UH/yE/jJWSuYc8bFbPyrM9w57Pzz4cknfZnS174Gl15acbvlxFNSVzy86n540o6JwJFm9nyqzMfwHqxJ2ha40cy2LFVX0njgLTM7T9LpwDpmVpvwWvXu+jbSFsO2QYdg//3NdtqpdJkTTvDhw/vvr4lJDcucOT6E2ru3GZh16+bv3W9/68OtL75otmjR6vVaM9/bmjpjx1Z1Xnn+fG/q+OPNbMYMb9PHK1bdevWqqF3KDNsCo4C7U8dnAGeUKf9Cubp43PJ+yet+wLRSdlRzC2/bIOholIsydNddHgThlFO859OZ6dfPk3IvXrzS03jkSDjuOPdcbWnxHl8+8+a5N+tjj/k+S1Sm1tR5/fXK65Rggw08OuE118D8NTbzv5ORI1cWaL3XdDdJk1Lb2Lzr/YFXUsezk3OrIOlQSVOBv+B5mcvV3cjM5gIk+w0rNby11DIZdhAEtWDTTf2LetGi1RfNv/mmC8M220Sixxw5URs7NnuS7nQu0ksuyXafWtUpwymn+GNeeimcdVZ/F88nn/Sh5EWL/MdE5cPCy8xsZInrhWI1rjZnaGa3ALdI2hOf//xU1rq1JsQzCDoaubWeL7/sazFzmPmc1ptveu8zX1g7K+0gUI3MllvCwQf7o552GvSeN8/Xhlby46FyZgMDU8cDgDnFCpvZQ5I2k7R+mbrzJPUzs7mS+gHzq2x3UUI8g6CjkQ6UkBbPa6/1FBs//SmMGFEX04LG4DvfgX328Tj+Y2vz42EiMFTSYOBV4AjgqHQBSR8HZpqZSdoB6AG8CbxTou7twDHAecn+tvZ6gHxizjMIOhrpnmeOl1+Gk0/2pSnf/W597Aoahr328qiEF1zg07ztjZktA04G7gZewD1pn5M0TtK4pNjngGclTQYuAb6Q+CMVrJvUOQ/YT9J03Bt3leUv7UksVUkRS1WCDsGyZe7kcuaZcPbZ/u24774eu/bppz2bSNDp+f3v3Tfoz3/2Ydy2ELFtgyBofrp1g/79V8a3vfBCz4byq1+FcAYfcfjhHv///PPrbUlzEuIZBB2RXGqyZ5+FM87wrClf+Uq9rQoaiO7dPYPcAw/AU0/V25rmo6biKekASdMkzUiiQeRfl6SLkutTkknjknUl/VzS1KT8LZLWTs4PkvShpMnJdllNHjIIGoENN4RHH/V8nGut5ckcVcjjP+jMHH+8j/DvsYcHQho0CK67rt5WNQc1E88kxNIlwGhga+BISVvnFRsNDE22scCEDHXvAYaZ2bZ4CKczUu3NNLMRyTaOIOgsvPgiLFkCzz/vYeE2rNna8aCJuOMOD5+7cKGvZHrpJV+xEgJanlr2PHcGZpjZLDNbAtwAjMkrMwa4JvGwegxYO1m7U7Sumf0t8cYCeAxfAxQEnZPevb2HmR6HGzPGzwdBHmee6f5laRYu9PNBaWopnlnCMxUrkym0Ex7O6a7U8WBJ/5L0oKQ9ChklaWwupNSy/L+iIGg2Zs3yLCk5sezdu/mSVAc1o1gUx1LRHQOnluKZJcRSsTJl60o6E09jkxtwmAtsambbA98Gfi9pzdUaMbvczEaa2chu3SJmRNDk5MdqXby4teHWgk7ApptWdj5YSS3FM0t4pmJlStaVdAxwMHB0EuEfM1tsZm8mr58EZgKpcCtB0EFpTQDyoFNy7rkeCz5Nnz4R9jgLNQuSIKkb7tCzLx5iaSJwVCpSBJIOwiNJHAjsAlxkZjuXqpskSb0A2MvMXk+1tQGe5225pCHAP4DhZvZWMRsjSEIQBJ2N667zOc6XX/Ye57nn+kh/JXTGIAk1jTAk6UDgQjwb+JVmdm4uNJOZXSZJwMXAAcBC4CtmNqlY3eT8DKAnHgMR4DEzGyfpc8DZ+FDucuDHZvbnUvaFeAZBEFROiGcnJ8QzCIKgcjqjeEaEoSAIgiCokBDPIAiCIKiQEM8gCIIgqJAQzyAIgiCokHAYSiFpBfBhG5rohnv3dkbi2Tsvnfn5O/Ozw8rn721mnaozFuJZRSRNMrOR9bajHsSzd85nh879/J352aFzP3+n+qUQBEEQBNUgxDMIgiAIKiTEs7pcXm8D6kg8e+elMz9/Z3526MTPH3OeQRAEQVAh0fMMgiAIggoJ8QyCIAiCCgnxrAKSDpA0TdIMSafX255aI+lFSc9ImixpUr3taU8kXSlpvqRnU+fWlXSPpOnJfp162tieFHn+syS9mnz+k5MMSB0OSQMl3S/pBUnPSfpmcr7Df/4lnr1TfPaFiDnPNiKpK55rdD88afdE4Egze76uhtUQSS8CI83sjXrb0t5I2hNYAFxjZsOSc+Px3LHnJT+e1jGz0+ppZ3tR5PnPAhaY2S/qaVt7I6kf0M/MnpK0BvAkcAhwLB388y/x7J+nE3z2hYieZ9vZGZhhZrPMbAlwAzCmzjYF7YSZPQTkJ1QfA1ydvL4a/1LpkBR5/k6Bmc01s6eS1+8DLwD96QSff4ln77SEeLad/sArqePZdL4/KgP+JulJSWPrbUwd2MjM5oJ/yQAb1tmeenCypCnJsG6HG7bMR9IgYHvgcTrZ55/37NDJPvscIZ5tRwXOdbax8E+Y2Q7AaOCkZGgv6DxMADYDRgBzgfPrak07I+ljwE3At8zsvXrbU0sKPHun+uzThHi2ndnAwNTxAGBOnWypC2Y2J9nPB27Bh7I7E/OSOaHc3ND8OttTU8xsnpktN7MVwBV04M9fUndcPK4zs5uT053i8y/07J3ps88nxLPtTASGShosqQdwBHB7nW2qGZL6Jg4ESOoL7A88W7pWh+N24Jjk9THAbXW0pebkhCPhUDro5y9JwG+BF8zsgtSlDv/5F3v2zvLZFyK8batA4p59IdAVuNLMzq2vRbVD0hC8twmenuj3Hfn5JV0P7A2sD8wDfgzcCtwIbAq8DBxuZh3SqabI8++ND9sZ8CLwtdwcYEdC0u7AP4BngBXJ6e/jc38d+vMv8exH0gk++0KEeAZBEARBhcSwbRAEQRBUSIhnEARBEFRIiGcQBEEQVEiIZxAEQRBUSIhnEARBEFRIiGcQdGAkmaTD6m1HEHQ0QjyDoJ2QdFUiXvnbY/W2LQiCttGt3gYEQQfnXuBLeeeW1MOQIAiqR/Q8g6B9WWxmr+Vtb8FHQ6onS/qLpIWSXpL0xXRlScMl3SvpQ0lvJb3ZtfLKHJMkI18saZ6kq/JsWFfSHyV9IGlW/j2CIKicEM8gqC8/wWOjjgAuB66RNBJAUh/gr3jy6Z3x2KG7AVfmKkv6GvBr4HfAtsCBwHN59/gRHm91O+APwJWSWtrtiYKgExDh+YKgnUh6gF8EFuVdusTMTpNkwG/M7PhUnXuB18zsi5KOB34BDEgSECNpb+B+YKiZzZA0G7jWzE4vYoMB55nZGclxN+A9YKyZXVu9pw2CzkXMeQZB+/IQkJ8g/J3U60fzrj0KHJS83gqYkhPOhEfwwNxbS3oPT7x+XxkbpuRemNkySa/TwRM2B0F7E+IZBO3LQjOb0cq6onhidaNwIvZCLC1QN6ZsgqANxD9QENSXXQscv5C8fh7YLpcvNWE3/P/2BTObB7wK7NvuVgZBsArR8wyC9qWnpI3zzi03s9eT15+VNBF4ADgMF8JdkmvX4Q5F10j6EbAO7hx0c6o3ey7wS0nzgL8AfYB9zez89nqgIAhCPIOgvfkUkJ8c+FVgQPL6LOBzwEXA68BXzGwigJktlPRpPNH6E7jj0W3AN3MNmdkESUuA7wA/A94C7mynZwmCICG8bYOgTiSesIeb2Z/qbUsQBJURc55BEARBUCEhnkEQBEFQITFsGwRBEAQVEj3PIAiCIKiQEM8gCIIgqJAQzyAIgiCokBDPIAiCIKiQEM8gCIIgqJD/D1i+LMYclcevAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### Visualization of performance scheduling\n",
    "\n",
    "lr_history = history.history['lr']\n",
    "val_loss_hist = history.history['val_loss']\n",
    "n_epochs = history.epoch\n",
    "plt.xlabel(\"Epoch\", fontsize=14)\n",
    "plt.ylabel(\"Learning Rate\", color='b', fontsize=14)\n",
    "plt.plot(n_epochs, lr_history, 'bo-')\n",
    "\n",
    "ax2 = plt.gca().twinx()\n",
    "ax2.plot(n_epochs, val_loss_hist, 'r*-')\n",
    "ax2.set_ylabel(\"Learning Rate\", color='r', fontsize=14)\n",
    "plt.title(\"Reduce LR on Plateau Performance\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2c1b2f8",
   "metadata": {},
   "source": [
    "#### `1Cycle Scheduling`\n",
    "\n",
    "\n",
    "#### you can emplement you custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c9a26ae3",
   "metadata": {},
   "outputs": [],
   "source": [
    "## i will implement my own function later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dc1a7d7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "154b291c",
   "metadata": {},
   "source": [
    "### Regularization techniques\n",
    "1. L1 or L2 or L1_L2 norm\n",
    "2. max norm\n",
    "3. Dropout ==> Note if you use selu activation function with lecun init. , you should use Alpha Dropout\n",
    "4. Monto Carlo (MC) Dropout\n",
    "5. Batch Normalization ==> works as a regularizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b27bc025",
   "metadata": {},
   "source": [
    "### $\\ell_1$ and $\\ell_2$ Reguarizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a6caaaff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x1b91c7b60d0>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## set kernel_regularizer for (l1 or l2 or l1_l2)\n",
    "tf.keras.layers.Dense(100, activation=\"elu\",\n",
    "                           kernel_initializer=\"he_normal\",\n",
    "                           kernel_regularizer=tf.keras.regularizers.l2(0.01))\n",
    "# or l1(0.1) for ℓ1 regularization with a factor or 0.1\n",
    "# or l1_l2(0.1, 0.01) for both ℓ1 and ℓ2 regularization, with factors 0.1 and 0.01 respectively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "70b53df7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_11\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_11 (Flatten)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_34 (Dense)             (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_36 (Dense)             (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "## you can use this method for making yout code more neat and beatiful\n",
    "## using partial from functools , you can get a class contains all what you need\n",
    "from functools import partial\n",
    "\n",
    "## this will return a class contains these data\n",
    "func_ready = partial(tf.keras.layers.Dense, \n",
    "                        activation='elu', \n",
    "                        kernel_initializer='he_normal', \n",
    "                        kernel_regularizer=tf.keras.regularizers.l2(0.01))\n",
    "\n",
    "model_new = tf.keras.models.Sequential()\n",
    "model_new.add(tf.keras.layers.Flatten(input_shape=[28,28]))\n",
    "model_new.add(func_ready(300))\n",
    "model_new.add(func_ready(100))\n",
    "model_new.add(func_ready(10, activation='softmax'))\n",
    "\n",
    "## check summary\n",
    "model_new.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06531757",
   "metadata": {},
   "source": [
    "### Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e0379d89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_12\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_12 (Flatten)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_37 (Dense)             (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "## using dropout \n",
    "model_drop = tf.keras.models.Sequential()\n",
    "model_drop.add(tf.keras.layers.Flatten(input_shape=[28,28]))\n",
    "model_drop.add(tf.keras.layers.Dense(300, activation='elu', kernel_initializer='he_normal'))\n",
    "model_drop.add(tf.keras.layers.Dropout(0.5))    ## here i add layer dropout with factor=0.5\n",
    "model_drop.add(tf.keras.layers.Dense(100, activation='elu', kernel_initializer='he_normal'))\n",
    "model_drop.add(tf.keras.layers.Dropout(0.5))\n",
    "model_drop.add(tf.keras.layers.Dense(10, activation='softmax'))\n",
    "\n",
    "model_drop.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "53713af6",
   "metadata": {},
   "outputs": [],
   "source": [
    "## using Alpha dropout for selu with lecun\n",
    "model_drop = tf.keras.models.Sequential()\n",
    "model_drop.add(tf.keras.layers.Flatten(input_shape=[28,28]))\n",
    "model_drop.add(tf.keras.layers.Dense(300, activation='selu', kernel_initializer='lecun_normal'))\n",
    "## version of dropout to self normalize with selu and lecun init.\n",
    "model_drop.add(tf.keras.layers.AlphaDropout(rate=0.5, seed=123))    ### you can use seed\n",
    "model_drop.add(tf.keras.layers.Dense(100, activation='selu', kernel_initializer='lecun_normal'))\n",
    "## version of dropout to self normalize with selu and lecun init.\n",
    "model_drop.add(tf.keras.layers.AlphaDropout(rate=0.5, seed=123)) \n",
    "model_drop.add(tf.keras.layers.Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3ce110a",
   "metadata": {},
   "source": [
    "### Monto_Carlo (MC) Dropout\n",
    "`it provides much better measure of the model's uncertainity , gives you much insights for the output` <br />\n",
    "`MC dropout ==> get the probability by Activating the dropout during testing`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "76b964ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.5800 - accuracy: 0.7914 - val_loss: 0.4152 - val_accuracy: 0.8513\n",
      "Epoch 2/5\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.4533 - accuracy: 0.8371 - val_loss: 0.4001 - val_accuracy: 0.8488\n",
      "Epoch 3/5\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.4244 - accuracy: 0.8456 - val_loss: 0.3672 - val_accuracy: 0.8623\n",
      "Epoch 4/5\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.4051 - accuracy: 0.8534 - val_loss: 0.3597 - val_accuracy: 0.8657\n",
      "Epoch 5/5\n",
      "1688/1688 [==============================] - 9s 5ms/step - loss: 0.3923 - accuracy: 0.8590 - val_loss: 0.3552 - val_accuracy: 0.8662\n"
     ]
    }
   ],
   "source": [
    "### train and compile a model \n",
    "model = tf.keras.models.Sequential([\n",
    "                tf.keras.layers.Flatten(input_shape=[28, 28]),\n",
    "                tf.keras.layers.Dense(300, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "                tf.keras.layers.Dropout(rate=0.3),\n",
    "                tf.keras.layers.Dense(100, activation=\"elu\", kernel_initializer=\"he_normal\"),\n",
    "                tf.keras.layers.Dropout(rate=0.3),\n",
    "                tf.keras.layers.Dense(10, activation=\"softmax\")\n",
    "                            ])\n",
    "\n",
    "## you can try these criteria\n",
    "## => batch size=32 ==> (len(X_train) // 32)=no of batches in 1 epoch\n",
    "## for 20 epoch ===> we get the number of batches in 20 epoch\n",
    "\n",
    "n_epochs = 5\n",
    "s = n_epochs * len(X_train) // 32  \n",
    "lr_schedular = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01, \n",
    "                                                              decay_steps=s, decay_rate=0.1)\n",
    "\n",
    "optimizer_with_lr = tf.keras.optimizers.SGD(learning_rate=lr_schedular, momentum=0.9, nesterov=True)\n",
    "model.compile(loss='sparse_categorical_crossentropy', \n",
    "                      metrics=['accuracy'], \n",
    "                      optimizer=optimizer_with_lr)\n",
    "\n",
    "early_stopp = tf.keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True)\n",
    "history = model.fit(X_train, y_train, \n",
    "                            epochs=n_epochs,\n",
    "                            validation_data=(X_val, y_val), \n",
    "                            batch_size=32, callbacks=[early_stopp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "23f698f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_probability_MC.shape => (100, 10000, 10)\n",
      "\n",
      "y_mean_each_MC.shape => (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "## you predict X_test ('another method without calling predict')\n",
    "## I set trainig=True => to allow dropout during testing, that is not allowed in the ordinary cases\n",
    "\n",
    "## here i choose to make 100 prediction(randomly because dropout is active) for each instance\n",
    "## for each instance you have 100 diffrent proabilities, you can try other number\n",
    "y_probability_MC = np.stack([model(X_test, training=True) for sample in range(100)])\n",
    "y_mean_each_MC = y_probability_MC.mean(axis=0)\n",
    "# shape of the 100 prediction for each output , and 10 output classes (probabilty using softmax)\n",
    "print('y_probability_MC.shape =>', y_probability_MC.shape)\n",
    "print()\n",
    "## after thaking the mean ==> shape\n",
    "print('y_mean_each_MC.shape =>', y_mean_each_MC.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1dbb3008",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.  , 0.  , 0.  , 0.  , 0.  , 0.14, 0.  , 0.11, 0.  , 0.74],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## for example for each instance >>> that is the mean probability of the prediction\n",
    "## this mean is over 100 randomly activating dropout during testing\n",
    "## that what is called MC Drop out\n",
    "## it gives you much uncertainity of your output\n",
    "## if you predict without MC, you will get a hard prediction cancelling the dropout at testing\n",
    "np.round(y_mean_each_MC[0], 2)  ## seems Good!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c1ea287",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "cb12fc3a",
   "metadata": {},
   "source": [
    "### Max_Norm Regularizer\n",
    "\n",
    "`you can specify that your waeights is constraints by not exceed specified threshold` <br />\n",
    "\n",
    "`Max_Norm Regularization loss does not loss term to cost function, after each training step, compute the norm2 of weights and if they need scaling to not exceed a specified threshold`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a6546b7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "1688/1688 [==============================] - 11s 7ms/step - loss: 1.2056 - accuracy: 0.7354 - val_loss: 1.0087 - val_accuracy: 0.7973\n",
      "Epoch 2/3\n",
      "1688/1688 [==============================] - 11s 6ms/step - loss: 0.9835 - accuracy: 0.8041 - val_loss: 0.9421 - val_accuracy: 0.8248\n",
      "Epoch 3/3\n",
      "1688/1688 [==============================] - 11s 6ms/step - loss: 0.9491 - accuracy: 0.8150 - val_loss: 0.9235 - val_accuracy: 0.8253\n"
     ]
    }
   ],
   "source": [
    "### you can add kernel_constraint to the layers\n",
    "\n",
    "model_maxnorm = tf.keras.models.Sequential()\n",
    "\n",
    "from functools import partial\n",
    "## make ready layers to be added\n",
    "ready_add_layers = partial(tf.keras.layers.Dense, \n",
    "                            activation='elu', \n",
    "                            kernel_initializer='he_normal', \n",
    "                            kernel_regularizer=tf.keras.regularizers.l2(0.001),\n",
    "                            kernel_constraint=tf.keras.constraints.max_norm(1.0))  ## not exceed=1.0\n",
    "\n",
    "## add layers\n",
    "model_maxnorm.add(tf.keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model_maxnorm.add(ready_add_layers(300))\n",
    "model_maxnorm.add(ready_add_layers(100))\n",
    "model_maxnorm.add(ready_add_layers(10, activation='softmax'))\n",
    "\n",
    "## making exponential schedular for learning rate, compile and fit the model\n",
    "## let's use only n_spochs = 3\n",
    "n_epochs = 3\n",
    "s = n_epochs * len(X_train) // 32\n",
    "lr_expon = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.001, \n",
    "                                                          decay_steps=s, decay_rate=0.1)\n",
    "optim_sgd = tf.keras.optimizers.SGD(learning_rate=lr_expon, momentum=0.9, nesterov=True)\n",
    "## compile\n",
    "model_maxnorm.compile(loss='sparse_categorical_crossentropy', \n",
    "                      metrics=['accuracy'],\n",
    "                      optimizer=optim_sgd)\n",
    "## fit\n",
    "histroy = model_maxnorm.fit(X_train, y_train, validation_data=(X_val, y_val), \n",
    "                            epochs=n_epochs, batch_size=32, callbacks=[early_stopp])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c370195",
   "metadata": {},
   "source": [
    "-------\n",
    "-----\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcca8c04",
   "metadata": {},
   "source": [
    "## Done!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f60ef97d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}