11_Custom_Models&Training_TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7d0db13d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Most Important\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "## less Important\n",
    "from functools import partial\n",
    "import os\n",
    "from scipy import stats\n",
    "import missingno as msno\n",
    "import joblib\n",
    "import tarfile\n",
    "import shutil\n",
    "import urllib\n",
    "\n",
    "## Sklearn\n",
    "from sklearn import datasets\n",
    "## Preprocessing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "## Metrics\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "## tensorflow & Keras\n",
    "import tensorflow as tf    ## i will use tf for every thing and for keras using tf.keras\n",
    "import tensorflow_datasets as tfds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f67b92bc",
   "metadata": {},
   "source": [
    "## Custom Loss Functions\n",
    "`I will build a custom huber function with diffrent thresholds`  <br />\n",
    "`Passing this class or function to the loss hyperparameter when compiling the Model` <br />\n",
    "`Saving the model, and when you loading it again, every thing is saved, this class or function is saved as json file in hdf5 model` <br />\n",
    "`When loading the Model you should pass the dictionary of custom objects (as you will see)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "da641cd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "## simple function ==> what i want to do\n",
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    like_MSE = tf.square(error) / 2\n",
    "    like_MAE = tf.abs(error) - 0.5\n",
    "    \n",
    "    ## when erroe is less than trheshold => then like_MSE, else like_MAE\n",
    "    return tf.where(is_small_error, like_MSE, like_MAE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3fc6594f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAloAAAEaCAYAAADJ6/kbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABDoElEQVR4nO3deZyVc//H8denhWnFnVREGzctEtIibUSpUMmdEsq+3tbITrLehNBtVzdRJGWXpWnRprIWv6RFWSuiaa/5/P74njJGy5mac65zzryfj8c8mrNe73OuZuZzvtf3+nzN3RERERGRwlcs6gAiIiIimUqFloiIiEiCqNASERERSRAVWiIiIiIJokJLREREJEFUaImIiIgkiAotEdkuM2tlZm5me0adRf5kZgvM7Oqoc4jI1qnQEslQZjbYzN7YwvUNY0VT9QhixcXMeplZTtQ5AMyseuz9yv81KokZbjWzL7dw0xHAoGTlEJGCKxF1ABEpusxsF3dfF3WOOLUDPstzeU1UQTZx9yVRZxCRbdOIlkgRt6XDgnlGcRrmu3sTM/vUzNaY2QwzOzzfcx1pZuPMbJWZfW9m/zWz8nluz45dd5+ZLQE+2sHM+5nZq2a2IvY10syq5rl9XzMbbWa/xrJ8bWan5rn9ZjNbaGZrzewnM/tfHJtd5u4/5flaHnsuN7Ou+fL95ZBe7D7nmdnLZrbSzOaZWc98j9nbzIaa2bJY5k/NrLWZ9QJuAermGU3rtZXtbO99udXMvjSzU83s29h9RumQsEjiqNASkYK4D7gWaAjMA940s9IAZnYwMAZ4DTgE6AI0AJ7J9xw9AQOaA2cUNICZGTAKqAQcDbQG9gZGxW6DcDitdOy2usDlwPLY408GrgYuAg4AOgLTCppjB9wMjCa8N8OBZ8ysWixTGWAcUB3oDBwM9Is9bjhwP/B/QJXY1/D8Tx7n+0JsG91i2zkOOBS4o7BepIj8lQ4dimS2dluY67QzH7Bud/d3AcysN7AY6AE8BfQBhrv7/ZvubGYXAp+Y2V7u/kvs6vnuftVOZGhDKFZqufuC2HZ6AHOBY4D3gWrAK+6+6VDf/DyPrwb8CIxx9/XAd8D0OLY73sxy81w+3t0nFCD3c+7+fCzvTcBlhGJzIeE9rAw0dfelsft/u+mBsX24wd1/2sbzx/O+QPi938vdf4/d5wmgdwFeh4gUgEa0RDLbeMKoUt6vHjvxfJM3fePuOcAXQJ3YVYcDPc0sZ9MXfx4arJXnOWbsxPYBagM/bComYlnmAT/kyfIQcKOZTTaz/vkOcb4MZAHzzexpMzvFzHaNY7s9+Ov7GE9xltfnefJuAJYAe8WuOhT4PE+RtSPieV8AFm4qsmJ+yJNDRAqZRrREMtsqd5+b9woz2z3ffTaN0uQ9vFRyB7ZVjDCy9cAWbvs+z/crd+C58zLAt3KbA7j702b2LtCeMNIzyczucvdb3X2RmR1IGOVpQzgsd4uZNXb3bWVbnP+9zLNNy3fdlt6/9Vt43KYPu/kfvyO2+77EkUNECpl+uERk05lrVfJc12Ar922y6ZvYvKJ6wFexq2YCdd197ha+Vhdi3tnAPnnbU5hZTcJ8pNmbrnP3xe7+hLv/izA/6rw8t61x9zfd/QpCi4S6QLMdzLOEPO+dmVXir+9lPGYC9bcxKX0dUHw7zxHX+yIiyaURLRGZCywCbjWzvoTJ0jdu5b43xs4W/IFQvKwDXojddg8wxcweAx4HVgAHASe4+/k7kKuYmTXId90Gwlyjz4ChZvZvwkjOw4Ri5UMAM3sIeBuYA5QntGaYHbutF+F331QghzAxfD3wzQ5kJLbNi81sErARuJOCt354AehLmLh+HWHu28HACncfCywAqpnZYYQ5ZSvcfW2+59ju+yIiyacRLZEiLjYh/FSgJuEP9W3A9Vu5e1/CobaZxM7Y23S4zd0/B1oQCrVxsee6C/h5B6OVAj7J95Xt7g50IowkZQNjgZ+ATrHbIPxue5hQXL0Xy3Bm7LblwNnABOBL4GSgi7vnnTBfEFcRzsDMBkYQDp/+sq0H5Bd7D1sSDrG+Dswi7IdNr+cV4C3gA8Lr7r6F54jnfRGRJDP9/ImIiIgkhka0RERERBJEhZaIiIhIgqjQEhEREUkQFVoiIiIiCZKy7R323HNPr169etQx0sbKlSspU6ZM1DEkj4zaJzNmwOGHb/9+KW7GjNCU/vAMeC2Zsk8gw35WMoT2ScHNmDFjqbtXzH99yp512LBhQ58+vaArXBRd2dnZtGrVKuoYkkdG7RMzSNHfFQWxaW3lVP29VyAZsk8gw35WMoT2ScGZ2Qx3b5j/eh06FBEREUkQFVoiIiIiCaJCS0RERCRBVGiJiIiIJIgKLREREZEEUaElIiIikiAqtEREREQSpMgWWk89Be+8E3UKERERSZRPP4Xbb482Q5EstDZsgMceg44d4Yknok4jIiIihe3tt6F58/B3ftmy6HIUyUKrRAkYOxaOOw7OPx/69oXc3KhTiYiISGF4/HE44QQ44ACYOhUqVIguS5EstADKlYPXXoMLLoB77oHu3WHNmqhTiYiIyI7KzYVrrw1/29u1g/HjYe+9o82UsotKJ0OJEjBoENSsCddcA4sXw+jRsOeeUScTERGRgli9Gs48E15+GS68EAYODH/no1ZkR7Q2MYM+fcKOmTkTmjaFb76JOpWIiIjEa8kSOOYYGDEC7rsPHn00NYosUKG1Wdeu8OGHsHw5NGkCEydGnUhERES2Z86cMEjyySdh0OSqq8IgSqpQoZVH06YwZUo4dHjMMTBsWNSJREREZGsmTgx/u//4I5zkdvLJUSf6OxVa+dSqBZMmQePGYYL8XXeBe9SpREREJK8XXwyDIhUrhkGSJk2iTrRlKrS2oEIFeO896NEDrr8ezj0X1q+POpWIiIi4h0GQHj1CcTVpUjipLVWlyFSx1LPrrvD882Hn9e8P330XJtmVLx91MhERkaJp/fpwRuHTT4dC65lnwt/rVKYRrW0wC637n346HPs96ihYtCjqVCIiIkXP779Dhw7hb/JNN4XBkFQvskCFVlzOOiu08l+4MMzdmjkz6kQiIiJFx6JFYTmdsWPDKFa/fql1ZuG2qNCKU5s28NFHoS9Hixbw5ptRJxIREcl8M2eGQY6FC8OgR+/eUScqGBVaBVCvXlgz6cAD4cQTQ1d5ERERSYw33wyDGyVLhsGONm2iTlRwKrQKqEoVGDcO2reHiy+Gq6/WgtQiIiKFbdCgMKhx4IGhfUO9elEn2jEqtHZA2bIwahRccgncfz+ccgqsWhV1KhERkfSXmxsGMS6+OEx+HzcuDHKkKxVaO6h48bBg5QMPwKuvwtFHwy+/RJ1KREQkfa1eDf/6VxjEuPTS8Pe1bNmoU+0cFVo7wQwuvxxeeQU+/zw0Tvv666hTiYiIpJ9ffgmDFiNHhkGMgQPDoEa6U6FVCDp3huxsWLkSjjwyDHOKiIhIfL7+OgxWfPZZGLy4/PKoExWepBZaZlbczD4xszeSud1kaNQoTNarVAmOPTY0UhMREZFtGz8+DFKsXBkGLTp3jjpR4Ur2iNZlwFdJ3mbS1KgR1lxq1gxOPz10ldeC1CIiIls2dGgYnKhUKQxWNGoUdaLCl7RCy8yqAh2Ap5K1zSjssQe8+24otG6+OXSVX7cu6lQiIiKpwz2sI9yzZxjNmjQpDFZkomQuKv0gcA1Qbmt3MLPzgPMAKlWqRHZ2dlKCJULv3lCsWHUGD67OZ5/9Rr9+syhbdkPCtpeTk5PW71cmyqR90goy5rVAZryWVmTG64DM+lnJFIncJxs2GPff/0/eeacKxx77E336/B+ffZa5h3/Mk3Bsy8w6Au3d/SIzawVc7e4dt/WYhg0b+vTp0xOeLdGGDIFzz4UDDoC33oJq1RKznezsbFq1apWYJ5cdklH7xCwjjoNbbHG0ZPzeS7gM2SeQYT8rGSJR+2T5cujaFT74AG65JXyly5qF22NmM9y9Yf7rk3XosBlwopktAIYBR5tZkZgufuaZ4VDi99+HtZoyoHYUEREpsIUL4aijwuT3wYPh1lszp8jalqQUWu5+nbtXdffqwKnAh+7eMxnbTgWtW4fjz1lZ0LIljB4ddSIREZHkmTEjtG9YvBjeeScMQhQV6qOVJHXqhDMq6tYNp64OHBh1IhERkcR7/fWwMPSuu4ZBh6OPjjpRciW90HL37O3Nz8pUlSuHHiEnnQSXXRYasm3cGHUqERGRxHjkEejU6c/Bhjp1ok6UfBrRSrLSpWHEiFBkPfQQnHxyaNImIiKSKTZuhCuuCOsVnnBCGGSoXDnqVNFQoRWB4sXDOk4PPxyGVFu1gp9+ijqViIjIzlu1KpxZ+OCDf64HXKZM1Kmio0IrQpdcAqNGwezZYZLgrFlRJxIREdlxP/8cBg9Gjw5HbR54IDMWht4ZKrQidsIJ4VTXtWvD0j0ffhh1IhERkYL76qs/Bw1GjYJ//zvqRKlBhVYKOPzwMEmwalVo2zY0ORUREUkXY8eGpXRWr4Zx4+DEE6NOlDpUaKWIatXgo4/CkGuvXmGdxAxp+iwiIhnsuefCIMHee8PUqdDwb73RizYVWilkt93CMj29e8Ptt8MZZ4RDiiIiIqnGHW67Lfytat48DBYkapm5dJbMRaUlDiVLwtNPQ61acOONsGgRvPoq7LFH1MlERESCdevCOr7/+184CvP447DLLlGnSk0a0UpBZnDDDTB0KEyeDE2bwrx5UacSEREJC0O3axeKrH794JlnVGRtiwqtFNajB7z3HvzySziTY+rUqBOJiEhRtmBBmPQ+cWKYm3XTTUVjYeidoUIrxbVoEUa1ypULE+VHjow6kYiIFEUffwyNG8OPP4ZBgJ49o06UHlRopYEDDwztHxo0CN12BwzQGYkiIpI8o0dDy5ahw/ukSeF7iY8KrTRRsWJoZtqlC1x1Vegqv2FD1KlERCTTPfQQdO4M9euHD/21a0edKL2o0EojpUrBSy9Bnz4waFBYET0nJ+pUIiKSiTZuhMsuC+sVduoUPuzvtVfUqdKPCq00U6wY3HtvKLTefjvM4frhh6hTiYhIJlm9uhhdusDAgXDllfDyy1C6dNSp0pMKrTR14YXw+uswZ044I3HevCK8NLqIiBSan36Cyy8/lDfegEcegfvv18LQO0OFVhpr3x4mTAjDu5deeihjxkSdSERE0tmsWeHD+3fflWb0aLj44qgTpT8VWmnu0ENDf63KldfQvj089VTUiUREJB198AE0axa6vj/00Cd07Bh1oswQV6FlZq3NrEbs+ypmNsTMnjGzyomNJ/GoWhUGDvyENm3Ckgg33AC5uVGnEhGRdDF4cOj2vu++4czCf/5TZ1oVlnhHtAYBG2Pf3w+UBBx4IhGhpODKlNnI66+HQuvOO+G002DNmqhTiYhIKnOHm2+G3r1DU+yJE2G//aJOlVniXVR6H3f/zsxKAG2BasA6QOe7pZCSJcPCnrVqQd++sHgxjBoFFSpEnUxERFLN2rVwzjnw/PNw9tnw3/+GvyNSuOId0frDzCoBLYHZ7r5pTFG7JMWYwbXXwrBhYbmEpk1h7tyoU4mISCr57Tdo2zYUWXfcAU8+qSIrUeIttB4GPgaGAo/GrmsGfJ2IULLzunULExt//TWcQTJpUtSJREQkFcybFz6ET54ML7wA11+vhaETKa5Cy93vAdoAzdx9WOzq74FzEhVMdl6zZuEHaY894OijQ1d5EREpuqZODR++lyyB99+H7t2jTpT5CtLeYT6wt5l1i13+HphX+JGkMB1wQCi2GjYMo1z33qsFqUVEiqKRI8OE93LlwlGO5s2jTlQ0xNve4WBgDvAk8HTs6pbAMwnKJYVozz3DJ5dTTw3zty64QAtSi4gUFe4wYAB07QoNGoT2DQceGHWqoiPeEa3/Aje7+0HA+th144CjEpJKCl1WFgwdCtddB088AR07wh9/RJ1KREQSacMGuPRSuOoqOPnksDB0xYpRpypa4i206gLPx753AHdfCZRKRChJjGLFQo+tJ58MI1zNm4cWECIiknlycqBzZ3j0UejTB4YPh1L6q5108RZaC4DD815hZo0ANQ5IQ+ecA2++CfPnQ+PG8OmnUScSEZHC9MMP0KIFvPVW6I91773hw7YkX7xv+03Am2Z2G7CLmV0HvAzcmLBkklBt24YOwMWKhZGtt9+OOpGIiBSGL74IZxbOmQOvvx7m5Up04m3v8AZwPFCRMDerGtDF3cckMJskWP364VTfAw6AE06Axx6LOpGIiOyM996Do46CjRthwgRo3z7qRBLvEjy4+0zgogRmkQjsvTeMHx9aP1x4YWhkd/fdGmIWEUk3Tz8dRq/q1AnTQ6pWjTqRQJyFlpn129pt7n5z4cWRKJQtC6NHw7//Df/5DyxYAEOGaNKkiEg6cIebbgpL6bRtG5pTly8fdSrZJN4RrX3zXa5M6KP1auHGkaiUKBHOTKlVK5ydsnhxKL50GrCISOpauxZ694YXX4Rzzw2/x7VmYWqJq9By9975rzOzdoCa92cQs9BrpXp16NkzTKZ86y01thMRSUXLloX2DRMmhCkf11yjNQtT0c7MxBkDdIrnjmaWZWbTzOwzM5sVO3tRUtTJJ8PYsbBiRVh4dMKEqBOJiEhe334LRx4J06bBsGFh1Q8VWakp3iV4aub7qgf0BxbFuZ21wNHufgjQAGhnZk12KLEkRZMmYZmGvfaCNm3CCu8iIhK9yZPD7+hly+CDD8LJTJK64h3Rmgt8E/t3LjAFaA6cGc+DPciJXSwZ+9LSximuZs2w8GiTJnDaaWGipRakFhGJzogR0Lo17L57KLiaNYs6kWxPvHO0dvpkfzMrDswA9gcedfepW7jPecB5AJUqVSI7O3tnN1tk5OTkJOz9uuEGY5ddDuLGGyvx0Uc/cuWVcyhRQhXX9iRynyRbK8iY1wKZ8VpakRmvAzLrZyVR3GH48H15/PFa1K37O3fc8SXff7+e779PzPa0TwqPeZKHKMxsd8LZipe6+5dbu1/Dhg19+vTpScuV7rKzs2nVqlXCnt8dbrkFbr89HEocMQJ22y1hm8sIid4nSWWWEcOZFpvEkuzfewmRIfsEMuxnJQE2LQz92GPhMOHgwZCVldhtap8UnJnNcPeG+a/f6oiWmS0ijsN77r5fQYK4+3IzywbaAVsttCS1mEG/flCjBpx3Xug8/OabsF+B9r6IiBTEihWhuHr7bejbN0zhUEPp9LKtQ4c9C2sjZlYRWB8rskoBbYB7Cuv5JXl69w7FVZcuYUHqN96Aww/f/uNERKRgvv8eOnYMaxc+8UTokyXpZ6uFlruPK8TtVAGGxOZpFQNeiq2fKGnomGPCJPn27cPq8MOGhbUSRUSkcHz+efgd+/vv4ehB27ZRJ5IdFfdah2bWgHCm4Z7A5m4d8SzB4+6fA4fuQD5JUXXrhgWpO3aETp1g4EC4+OKoU4mIpL9334VTTgnL6EycCIccEnUi2Rnx9tE6D/gIOBq4FjgYuIpwBqEUUZUrw7hx0KEDXHIJXHllWDFeRER2zBNPhN+ptWqFD7MqstJfvFPqrgHauXtnYHXs367A+oQlk7RQpgy8+mpYkPqBB8KnsFWrok4lIpJecnPhuuvg/PPhuONg/HjYZ5+oU0lhiLfQ2svdNy3Ekmtmxdz9bUAzc4TixeGhh+DBB2HUqNBM7+efo04lIpIe1qyBHj3CeoUXXACvvQblykWdSgpLvIXWYjOrHvt+DnCSmTUH1iUklaSlyy4Lo1tffBG6yX/1VdSJRERS29KloTfh8OFw770waBCUiHv2tKSDeAute4Hase/7Ac8DHwJaHFr+4qSTwrytVavCgqdjx0adSEQkNX3zDTRtCtOnw0svQZ8+Whg6E22z0DKzl8ysPfC/2KFCYv/uAezh7v9NQkZJM0ccESZxVqkSTkl+7rmoE4mIpJaPPgpF1vLl8OGHYX6rZKbtjWh9DzwN/GBmA8ysPoC7r8uzSLTI31SvHnptNW8OZ5wBt92WMauFiIjslOHDQz/CChVgypQw+i+Za5uFlrtfAewDnAVUBiab2admdqWZVUpGQElfu+8elo0480y49Vbo1QvWaVafiBRR7nDPPXDqqWHkf9Kk0MZBMtt252i5e667v+XuPQgd3gcCHYGFZqbu7rJNu+wCzz4bRrT+9z9o1w5++y3qVCIiybV+fWjd0LcvdO8O770XRrQk8xVoaUp3/wN4O/b1M6FTvMg2mcHNN4dCa+JEaNYMFiyIOpWISHL88UdYpuzJJ+GGG+D55yErK+pUkizxdobPMrMeZvYusABoC9xEGOESicvpp8OYMfDjj2FB6mnTok4kIpJYixeHuarvvw9PPQX9+0OxAg1xSLrb3lmHrczsGcLo1c3AeGB/d2/j7v9zd/UAlwJp1SrMSyhTJnw/alTEgUREEuTTT8OHygULwnzVs8+OOpFEYXt19avAWsLyOwe5+x3uvigJuSSD1a4dzrQ5+GDo0iV0lBcRySRvvx1GsooXD1Mmjj026kQSle0VWpXd/UJ3n5yUNFJk7LVXaGbaqRNccUVYK1ELUotIJnjssTAn64AD/vxQKUXX9to7rE1WECl6SpeGl1+GK6+Ehx+Gzp1h5cqoU4mI7JjcXLjmGrjwwnCG9fjxsPfeUaeSqGlKnkSqeHG4/3545BF4801o2TJMlhcRSSerV0O3bvCf/8BFF4X5p2XLRp1KUoEKLUkJF18Mo0eHhaibNIFZs6JOJCISnyVLQqf3V17584OjFoaWTXao0DKzmmZWrbDDSNHWsSNMmBAa+x15ZDgdWkQklc2ZEz4cfvIJjBgRpkJoYWjJK94+Wi+a2ZGx73sDs4DZZqaTVaVQHXZYmDy6335w/PGhq7yISCqaMCEsDL1iRTi5p0uXqBNJKop3ROsYYHrs+yuBNkAjoG8iQknRtt9+4XTo1q3hrLPgppu0ILWIpJYXXoA2baBixfDhsEmTqBNJqoq30NrF3deZ2T7AP9z9I3efBWhhaUmI3XYLk+PPPjt0Uu7ZE9bqHFgRiZg73HknnHZaGM2aNAlq1ow6laSyeKfrfWpm1wHVgDcBYkXXH4kKJlKyZFgbrGbNsD7Y4sXw6qvwj39EnUxEiqL160PrhqefDoXW00/DrrtGnUpSXbwjWmcDBwOlgBtj1zUFhiYilMgmZnD99WGYfsqUMEn+22+jTiUiRc3vv0OHDqG4uukmeO45FVkSn7hGtNz9W6BHvutGACMSEUokv+7doWrV0Em+SRN47bUwbC8ikmjffReKrK+/Difo9OoVdSJJJ/GeddjdzGrHvj/QzMab2YdmdlBi44n8qXlzmDw5zN86+ujQs0ZEJJFmzgwf7r77Dt55R0WWFFy8hw77A7/Gvr8PmAaMBwYlIpTI1vzzn6HYOvRQOOUUuO8+nZEoIonxxhvQokWYLzppUmhKKlJQ8U6Gr+juP5tZFnAU0BVYDyxNWDKRrahYET74AM48E/r0gXnzYOBAdWIWkcIzaBBcemn4UPf661ClStSJJF3F+6dpiZntT5gQ/7G7rzWz0oD630okSpWCYcPCGYn33AMLF4bL5cpFnUxE0llubvgAN2AAnHACvPgilCkTdSpJZ/EWWrcDM4CNQLfYdccAnyUilEg8ihWDu++GGjXCWoktWoSh/n32iTqZiKSjVavg9NNh5MgwmvXAA2Hhe5GdEdccLXcfDFQBqrr7e7GrpwKnJiiXSNzOPz8M7c+dC40bw+efR51IRNLNL7+Ek2xefRUefDBMR1CRJYWhIItK7wp0NbPrzOwMYIO7/5SgXCIFcvzxYdkegKOOgnffjTaPiKSPr78OZxZ+/nkYzbrssqgTSSaJt71DU+Bb4AKgPnA+MDd2vUhKOOSQ0NS0Zs3Q8+aJJ6JOJCKpbty40JNv5crwfadOUSeSTBPviNaDwEXufqS7d3f3ZsCFwMCEJRPZAVWrwoQJcOyx4ZBi375hcquISH7PPx9+V1SpEj6kHXFE1IkkE8VbaP0TeCnfdSOA/Qs3jsjOK1cuzNk6//xwRmL37rBmTdSpRCRVuMPtt4eJ70cdBR99FE6qEUmEeAutb/j7xPdTCIcTRVJOiRLw3//CvffCSy9BmzawVF3fRIq8devgrLPg5pvhjDNCt/c99og6lWSyeAuty4FHzGyKmQ03s6mErvD/jufBZravmY01s6/MbJaZaaqhJJxZ6Ifz0kswfXqYh/HNN1GnEpGoLF8eTpwZPBhuvTX8u8su0WaSzBdve4dJQC3gEUI/rYcJhw2nxLmdDcBV7l4baAJcbGZ1Ch5XpOBOOQU+/DD8km3a9M+zE0Wk6Fi4EJo1C3M4hwyBW24JH8ZEEi3u9g7u/pu7P+/u97r788BKwjI88Tz2R3efGft+BfAVoLaSkjRHHhnWSPzHP8J6ZcOHR51IRJJl+vTQvuH770PrlzPOiDqRFCU7uzpcgT8PmFl14FBCw9P8t50HnAdQqVIlsrOzdzJe0ZGTk6P3Kw7/+U8JbrqpHqeeujsffDCP7t2/S9in2kzaJ60gY14LZMZraUVmvA5I7M/KRx9VoH//Ouy++zoeeugLzFaRIW9bQmXS76+ombvv2APNdgVWuXvcvXPNrCwwDrjD3Udu674NGzb06dOn71C2oig7O5tWrVpFHSMtrFkTJsO++CKcc05YPLZkycLfTkbtE7Nwqlaas1hVvaO/91JKhuwTSNzPysMPh+ajDRuGM5ErVSr0TWSsjPr9lSRmNsPdG+a/fmdHtAoSoCTwCjB0e0WWSCJlZYX+OTVrwh13wHffwcsvQ/nyUScTkcKwcSNcfXVYSqdTJxg6FEqXjjqVFFXbLLTMbAKwtY9Mcc/vsvAx8mngK3cfEH88kcQoVgz69w+9c84/P/TSefNN2HffqJOJyM5YuRJOOw1Gj4bLL4f77tOahRKt7Y1oPbWd25+MczvNgNOBL8zs09h117v7W3E+XiQhzj4b9tsPunYNk2XfeAMOPTTqVCKyI37+GU44AWbMCItCX3pp1IlEtlNoufuQwtiIu09kBybOiyTDsceGlg8dOkDz5qHvVvv2UacSkYKYPTv8DP/yC7z6Kpx4YtSJRIK4D/+JZLKDDw5rnR14YPhEPGhQ1IlEJF5jx4YWLmvWwPjxKrIktajQEonZe28YNy50jr744tBVXgtSi6S2//0P2rYNC8pPmQKHHx51IpG/UqElkkfZsjBqVCi07rsP/vUvWL066lQikp97WEbnzDOhRYtw+L9atahTifzddgstMytuZkNifbNEMl6JEqH/zoABMHIkHH10mPchIqlh3bpQYN12G/TqBW+9BbvvHnUqkS3bbqHl7huB4wAdRJEiwwyuuAJeeQU++yyckfj111GnEpHffoN27eC55+D22+GZZ7QwtKS2eA8dPgDcFms6KlJkdO4cJtrm5ITJtuPHR51IpOiaPz/8HH70UWg6fOONWhhaUl+8hdalQB9ghZktMrPvNn0lMJtISmjcOEyyrVQptIIYOjTqRCJFz7RpYWT5559hzJjQlFQkHcS7BE/PhKYQSXE1a8KkSdClC/TsGT5Z33CDPk2LJMOoUdCjB1SuHOZjHXRQ1IlE4hdXoeXu4xIdRCTV7bEHvPtuWIj6ppvg22/h8cc1P0QkkR58EK68Eho1gtdeg732ijqRSMHEdejQzHY1szvMbJ6Z/R677jgzuySx8URSyy67wJAhcMstMHhw6CC/fHnUqUQyz8aN8O9/h5NSOneGDz9UkSXpqSCT4esBp/HnItOzgAsTEUoklZmF/j2DB4cGp0cdBQsXRp1KJHOsXBmKq4cfhquugpdfhtKlo04lsmPiLbQ6Az3cfTKxNg/u/j2wT6KCiaS6M88MhxIXLw6TdKdPjzqRSPr78Udo2RLefBMefTQ0Di6m1tqSxuL977uOfPO5zKwisKzQE4mkkaOPDpPkd901/HF47bWoE4mkr1mz/uxZN3o0XHRR1IlEdl68hdbLwBAzqwFgZlWAR4BhiQomki7q1AntH+rUgU6dYODAqBOJpJ/33w89stavD/3qOnaMOpFI4Yi30LoeWAB8AewOfAP8ANyWkFQiaaZyZcjOhhNPhMsug8svD5N5RWT73n67MscfD/vtB1OnwmGHRZ1IpPDE295hHXA5cHnskOFSd/dtP0qkaClTJizZc/XV4ZT06dPr0aRJuF5E/s4dbr4Z7r33II49Nkx63223qFOJFK54G5ZiZgcA/wL2Bn4ws5fc/ZuEJRNJQ8WLwwMPhAanl19egVat4PXXw4iXiPxp7Vo4++yw0kKHDj/w6qt7U1KLvEkGirePVg/gE6A+sBI4GJgZu15E8rn0UujX70tmzw6Te2fPjjqRSOr49Vc47rhQZN15J1x11RwVWZKx4p2j1R9o7+7d3P0adz8VaA/cmbhoIumtWbNljBsHa9aESb4ffhh1IpHozZsXfh6mTIEXXoDrrtNSVpLZ4i20ygGT8103BdDsE5FtaNgwTO6tWhXatg1d5UWKqilTwgjvkiXhLMPu3aNOJJJ48RZaA4A7zSwLwMxKAXfErheRbahWDSZODH22evUKy/foVBIpal55BVq3hvLlYfJkaN486kQiybHVyfBmtog/l9sxoDJwmZn9BuwRu+5H4K5EhxRJd7vvDm+9BRdcAP36wfz58OSTodGpSCZzhwEDoE+fMJo1ejRUrBh1KpHk2dZZhz2TlkKkCNhlF3j66XBG4k03waJFMHIk7LFH1MlEEmPDhtBXbtAgOOWUcOi8VKmoU4kk11YLLXcfl8wgIkWBGdx4I9SoAWedFSYFv/VWuCySSXJy4NRTw5qF11wDd92lNQulaIqrj5aZ9dvabe5+c+HFESkaTjstTJDv3DkcTnntNWjcOOpUIoXjhx/CEjqffw6PPQbnnx91IpHoxPv5Yt98X0cAVwO1EpRLJOO1bBkmBZctC61ahcOIIunuiy/Ch4ZvvgnNelVkSVEXV6Hl7r3zfR0PdAE2JDaeSGY78MBQbB1yCHTtGrrK64xESVdjxkCzZpCbCxMmwPHHR51IJHo7c8R8DNCpkHKIFFl77QVjx0KXLnDllaGr/AZ9hJE08/TT0L59mG84dSo0aBB1IpHUEO8crZr5rioN9AAWFXoikSKoVCl46SW49lq47z5YuBBefDEcVhRJZbm54SzaO+8MTXlfein0yhKRIN5FpecSemptWihhFWHtwzMTEUqkKCpWDP7zn9D+4ZJLoEULeOMN2HvvqJOJbNmaNdC7NwwbBuedB488gtYsFMknrkLL3XVSrkiSXHhh6Cb/r3+FMxLffBMOPjjqVCJ/tWwZdOoUVj24557QkFRrFor8nQookRTUvn2YTLxhAxx1FLz3XtSJRP40dy40bQoffxxGs665RkWWyNZsc0TLzCbw5zI8W+TuLQo1kYgAcOihYVJxhw6h8HrsMTj77KhTSVE3aRKcdFI4O/aDD8JZhiKydds7dPhUnu8NeBS4KHFxRCSvffcNh2b+9S845xyYNw9uv10dtiUaL78Mp58e/l++/Tbsv3/UiURS3zYLLXcfkveymQ3If52IJFb58qHx40UXhTO75s+HZ5/VgtSSPO7hRI1rrw0jWKNGwZ57Rp1KJD0k5XOxmT1jZr+Y2ZfJ2J5IpilZEp54IqwX9+KLcOyxYTKySKJt2BCK/GuvhW7d4P33VWSJFESyDkAMBtolaVsiGckM+vYNk4+nTQuTkefOjTqVZLIVK+DEE8P8wL594YUXICsr6lQi6WV7k+GPzn9/M2vNn/20cPcPt7cRdx9vZtV3KKGI/EW3bmFB6pNOCsXW6NFw5JFRp5JM8/334USML78Mo6nnnht1IpH0ZL6NhdXMbP52Hu/unr9r/NaeqzrwhrvX28Z9zgPOA6hUqdLhw4YNi+epBcjJyaGs2oinlETvk8WLS9G378H88ksW11//Fa1aLUnYtlq1bk322LEJe/5kad26NQBjM+C1JHKfzJ1bhuuuq8+qVcW55ZZZNGr0W0K2s4l+f6Ue7ZOCa9269Qx3b5j/+m0WWoUpnkIrr4YNG/r06dMTGyqDZGdn06pVq6hjSB7J2CdLl4aRrUmTEtw00iwjVru22JuTrN97CZWgffLOO3DKKbD77qFZbv36hb6Jv9Hvr9SjfVJwZrbFQksniYuksT33DL2MunULk5UvvFALUsuOe+IJ6NgxtG2YMiU5RZZIplOhJZLmsrLCJOW+feHxx+GEE8IkZpF45eaG/z/nnx8Whh4/HvbZJ+pUIpkhWe0dXgQmAwea2WIzU39rkUJUrFho/fDEE2G5nubNYfHiqFNJOlizBrp3D4eeL7ggnFxRrlzUqUQyR1yLSu8sd++ejO2IFHXnngv77Rfm2DRuHObYNGgQdSpJVXnn+P3nP3DVVVqzUKSw6dChSIZp2zYs21OsWBjZevvtqBNJKvrmm9AeZObMsLTO1VeryBJJBBVaIhmofv0wmXn//cOcrccfjzqRpJKJE0ORtXw5fPghdO0adSKRzKVCSyRD7bNPmNTctm2Ye3PttWHSsxRtw4fDMcdAhQqhGG/aNOpEIplNhZZIBitXLkxuvuACuPdeOPVUWL066lQSBXe4++7wf6Bx4zAvq1atqFOJZL6kTIYXkeiUKAGDBoU/qn36hLMRR4+GihWjTibJsn59WBj6qaegRw945hnYddeoU4kUDRrREikCzMJk55dfhk8+CYeL5syJOpUkwx9/hCakTz0FN94Izz+vIkskmVRoiRQhXbvC2LHhj2/TpjBhQtSJJJEWLYKjjgoT3p9+Gm6/XWcWiiSbCi2RIqZJE5g8OSzf06YNvPhi1IkkET75JOzrhQtDi4+zzoo6kUjRpEJLpAiqVSsUW40bhzk7d96ZEWtGS8xbb4UeasWLw0cfhYJaRKKhQkukiPrHP8JyPT16wA03hK7y69dHnUp21n//G3qnHXhgaN9Qr17UiUSKtrQ663D9+vUsXryYNWvWRB0l5ey222589dVXUcdIG1lZWVStWpWSJUtGHSVSu+4aJkfXqhXm73z3XZgwv9tuUSeTgsrNDb3S7rsPOnSAYcOgbNmoU4lIWhVaixcvply5clSvXh3TjM6/WLFiBeW0Emxc3J1ly5axePFiatSoEXWcyJlBv35Qowacd16YPP3mm2HNREkPq1fDGWfAiBFw8cXw4IOhrYeIRC+tDh2uWbOGChUqqMiSnWJmVKhQQSOj+fTuHSZNf/ddmEQ9c2bUiSQeS5bA0UfDK6/AgAHw8MMqskRSSVoVWoCKLCkU+n+0ZW3ahMnTJUtCixbwxhtRJ5Jt+b//C0Xxp5+G0awrrlD7BpFUk3aFlogkVr16YRL1QQfBSSfBo49GnUi2pkkTWLECsrOhS5eo04jIlqjQKqCy+WaXDh48mEsuuWSbj8nOzqZjx46JjCVSqKpUgXHjwqTqTf+9tSB16njmmfDv3nvD1KmhTYeIpCYVWmlg48aNUUeQIqhMGXj1Vbj00nD5lFNg1apoMxV1ubnQty+cfXa4PGlSOIlBRFJX2k6ZvPzyMC+hMDVoEM7W2VG9evWiY8eOdO3aFQijXzk5OQD88ccfdO7cmf/7v/+jRYsWDBo0iGLFijFmzBhuueUW1q5dS61atXj22WcpW7Ys1atX56yzzmLMmDFccsklnHrqqTv/AkUKqHhxGDgQeDgUXa1bw2uvQaVKUScrelatgtNPh5Ej4YILgMfUhkMkHWhEq4BWr15NgwYNNn/dfPPNcT1u2rRp3H///XzxxRd8++23jBw5kqVLl9K/f3/ef/99Zs6cScOGDRkwYMDmx2RlZTFx4kQVWZISRo6EL76Ahg1h2rSo0xQt334b1qZ89dVwZuGgQVEnEpF4pe2I1s6MPO2MUqVK8WmeobTBgwczffr07T6uUaNG1KxZE4Du3bszceJEsrKymD17Ns2aNQNg3bp1NG3adPNjunXrVrjhRXZCp04wcSKcfHJY3mXgwNB3S2e5JdYbb0DPnlCsWPi+ffuoE4lIQaRtoZWKSpQoQW5sxrC7s27dus235W8nYGa4O8ceeywvbmVV3zJlyiQurMgOOOwwmD49/OG/4IJwduKgQVCqVNTJMs/GjaGRbL9+cOihoU+W5mOJpB8dOixE1atXZ8aMGQCMHj2a9XkWjps2bRrz588nNzeX4cOHc9RRR9GkSRM++ugj5s6dC8CqVauYM2dOJNlF4lWhQhhZueUWGDwYjjgCPv886lSZZdEiOOaYUGT16hV6m6nIEklPKrQK0bnnnsu4ceNo1KgRU6dO/cuIVNOmTenbty/16tWjRo0adO7cmYoVKzJ48GC6d+9O/fr1adKkCV9//XWEr0AkPsWLw623wrvvwtKl0KhROJToHnWy9DdiBBxySBg5fPbZ0MpBI4Yi6cs8RX8zNmzY0PPPffrqq6+oXbt2RIlSm9Y6LLhE/3/Kzs6mVatWCXv+pDLbahX1yy9w1llhfcTjj4enngr9nVLRpkP4qfh7748/4Mor4emnwyjhCy/A/vtv4wHb2CfpJqN+VjKE9knBmdkMd2+Y/3qNaInITtlrL3j9dXjkERg7FurUCaMwGVIDJMVbb0HduuF969s3HCrcZpElImlDhZaI7DQzuPhi+OwzqF8/NNRs2xYWLIg6WWpbtiz0xurQAcqXDw1I77orrDUpIplBhZaIFJp//jOsuzdoEEyeHEZp7rgD1qyJOllq2bgxHGI96CAYNgxuuglmzgxrF4pIZlGhJSKFqlgxuPBCmDUL2rWDG28MBddrr+lwIoRRq8aN4dxzQ6E1Y0Y4u3DXXaNOJiKJoEJLRBJiv/1C76f33oOsLDjppHA4MY7+vhlp7lw47TRo1gx++ilMdh8/PhxqFZHMpUJLRBKqTZuwLumDD4bDY0ccEbrLz5oVdbLkWLQodNA/6KCwhM7118PXX0P37uqqL1IUqNAqgOXLlzMozyJj2dnZdOzYsdC306tXL0aMGBH3/RcsWEDjxo23eFurVq22ukRQ165dmTdv3g5ljEf16tVZunQpEBbYjkKbNm347bffItm2/KlkSbjsMpg3L/Tfeu89OPjgUGx88knU6RLj22/DCQIHHABDhsBFF4XXf8cdENGPg4hEQIVWAeQvtOK1cePGBKTZObNmzWLjxo2b119MZfnfv3jeT3cnNzeX008/fYf2mSRG+fKho/z8+XDNNaH31mGHhVGvd97JjDlcU6dC166hwHrqqXBW4Zw5oaFr5cpRpxORZFOhVQB9+/bl22+/pUGDBvTp0weAnJwcunbtykEHHcRpp522uRFi9erV6devH0cddRQvv/wyY8aMoWnTphx22GGccsop5OTkbH7OOnXqUL9+fa6++urN2xo/fjxHHnkkNWvW3Dy65e706dOHevXqcfDBBzN8+PC/ZVy9ejWnnnoq9evXp1u3bqxevXqLr2Xo0KGcdNJJQChcevXqtfl5H3jgASCMhl1xxRW0aNGC2rVr8/HHH9OlSxcOOOAAbrzxxs3P1alTJw4//HDq1q3LE088UaD39Pnnn6dRo0Y0aNCA888/f3MRVbZsWW6++WYaN27M5MmT/3Z5wIAB1KtXj3r16vFgbIXxBQsWULt2bS666CIOO+wwFi1axIknnrjVtSQlOhUqwN13h8Nq994bDqUdf3w4vHbvvfDzz1EnLJjff4fHHguHRZs0gQ8+CP2wFiyAJ5+EatWiTigikXH3lPw6/PDDPb/Zs2f/9YrwAbhwv7Zh/vz5Xrdu3c2Xx44d6+XLl/dFixb5xo0bvUmTJj5hwgR3d69WrZrfc8897u6+ZMkSb968uefk5Li7+9133+233XabL1u2zP/5z396bm6uu7v/9ttv7u5+5plneteuXX3jxo0+a9Ysr1Wrlru7jxgxwtu0aeMbNmzwn376yffdd1//4YcffP78+V67dm13d7///vu9d+/e7u7+2WefefHixf3jjz/+22tp0aKFf/755+7uPn36dG/Tps3m2zblaNmypV9zzTXu7v7ggw96lSpV/IcffvA1a9b4Pvvs40uXLnV392XLlrm7+6pVq7xu3bqbr69WrZovWbLE3d3LlCnztwyzZ8/2jh07+rp169zd/cILL/QhQ4bEdi0+fPjwzffNe3n69Oler149z8nJ8RUrVnidOnV85syZPn/+fDcznzx58l+2s//++2/OlH/7iTR27NiEPn9SbednY2etXev+3HPuzZuHTZUo4d6pk/vw4e4rVhTedgCnkF7L2rXub73lfsYZ7qVKhdwHH+z+8MPuf/xRKJvYtgTvk2TKqJ+VDKF9UnDAdN9CPZPeI1qJKLUKqFGjRlStWpVixYrRoEEDFuTp0NitWzcApkyZwuzZs2nWrBkNGjRgyJAhLFy4kPLly5OVlcU555zDyJEjKV269ObHdurUiWLFilGnTh1+jn28nzhxIt27d6d48eJUqlSJli1b8vHHH/8lz/jx4+nZsycA9evXp/5WTmn68ccfqVixIgA1a9Zk3rx5XHrppbzzzjuUL19+8/1OPPFEAA4++GDq1q1LlSpV2HXXXalZsyaLFi0CYODAgRxyyCE0adKERYsW8c0338T13n3wwQfMmDGDI444ggYNGvDBBx9snjNWvHhxTj755M33zXt54sSJdO7cmTJlylC2bFm6dOnChAkTAKhWrRpN8jUj2muvvfjhhx/iyiTR2GUX6NkznIX39ddwxRWhDUK3blCxInTuDM89F87Wi9Ly5TByZDgcuNde0L59mOB++ukwbVpo2HrJJaDVsERkkxLJ2pCZtQMeAooDT7n73cnadiLtmqf5TfHixdmwYcPmy5sWlXZ3jj322C0ewpo2bRoffPABw4YN45FHHuHDDz/82/N6rAD0OAtBi+NUplKlSrEm1kVyjz324LPPPuPdd9/l0Ucf5aWXXuKZZ575S45ixYr9JVOxYsXYsGED2dnZvP/++0yePJnSpUvTqlWrzc+7Pe7OmWeeyV133fW327KysihevPgWL2/rfci7kPcma9asoZRW5U0bBx4YDh/edRdMnBgWWR45EkaNCrfXrQvHHAMtW0LDhrDvvok7e+/nn0Ofq/Hjw+HAmTMhNxf22CMUf127hvll6oElIluTlBEtMysOPAocD9QBuptZnWRsuzCVK1eOFStWFPhxTZo04aOPPmLu3LkArFq1ijlz5pCTk8Pvv/9O+/btefDBB/n000+3+TwtWrRg+PDhbNy4kSVLljB+/HgaNWr0t/sMHToUgC+//JLPP/98i89Vu3btzXmWLl1Kbm4uJ598MrfffjszZ86M+7X9/vvv7LHHHpQuXZqvv/6aKVOmxP3YY445hhEjRvDLL78A8Ouvv7Jw4cLtPq5FixaMGjWKVatWsXLlSl599VWaN2++xfu6Oz/99BPVq1ePO5ekhuLFQzH18MNhLtfHH8M998A++4R5TyefHOY+VawIxx0XFmR++OEwwX72bPj119CBfVtyc8Mo1Zw58O67YZ7VtdfCCSeE7VSuHJbHuf/+0Avspptg3LhQgD37bLhNRZaIbEuyRrQaAXPdfR6AmQ0DTgJmJ2n7haJChQo0a9aMevXqcfzxx9OhQ4e4HlexYkUGDx5M9+7dWbt2LQD9+/enXLlynHTSSaxZswZ33zwJfWs6d+7M5MmTOeSQQzAz7r33XipXrvyXw5UXXnghvXv3pn79+jRo0OBvhdgmHTp0IDs7mzZt2vD999/Tu3dvcnNzAbY4wrQ17dq147HHHqN+/foceOCBfztsty116tShf//+HHfcceTm5lKyZEkeffRRqm1n5vBhhx1Gr169Nr+2c845h0MPPfQv78MmM2bMoEmTJpQokbTBW0mAYsXC6FXDhuFsxbVrQ1uITz4Jo0wzZ8Ljj8OqVX9/bPnysPvuf10/sEaNMIH9999DsZXXLruEMwaPOSacEbnpSy0ZRGRHWLyHo3ZqI2ZdgXbufk7s8ulAY3e/JN/9zgPOA6hUqdLhw4YN+8vz7LbbbuyvJe23aOPGjX851LY9q1evpkOHDrz33nsFely6ueaaa2jfvj2tWrX6221z587l999/T9i2c3JyIusfVthatW5N9tixUcfYJnf47beS/PRTKX78MYvffitJTk5JVqwoQU5OCXJzjfffD/0Vjj32R8qW3bD5a7fd1lOlyhqqVFlDhQprKZYGs1fTYZ/EK5N+VjKF9knBtW7deoa7N8x/fbI+5m9pBsXfKjx3fwJ4AqBhw4ae/4/jV199RTnNMt2iFStWFOi9KVeuHP379+ePP/5gv/32S2CyaB122GGccMIJW7wtKyuLQw89NGHbzs7O3mKBl64y4bVsmss1ZkxmNLTKhH0Cmfezkgm0TwpPsgqtxcC+eS5XBXQaWMTatm0bdYSEO/fcc6OOICIiRViyBsg/Bg4wsxpmtgtwKvDajjxRMg51SubT/yMREUmGpBRa7r4BuAR4F/gKeMndC7ykbFZWFsuWLdMfSdkp7s6yZcvIysqKOoqIiGS4pJ2K5e5vAW/tzHNUrVqVxYsXs2TJkkJKlTnWrFmjwqEAsrKyqFq1atQxREQkw6XVOe8lS5akRo0aUcdISdnZ2Qmd2C0iIiIFlwYnMYuIiIikJxVaIiIiIgmiQktEREQkQZLSGX5HmNkSYPsL38kmewJLow4hf6F9kpq0X1KP9knq0T4puGruXjH/lSlbaEnBmNn0LbX+l+hon6Qm7ZfUo32SerRPCo8OHYqIiIgkiAotERERkQRRoZU5nog6gPyN9klq0n5JPdonqUf7pJBojpaIiIhIgmhES0RERCRBVGiJiIiIJIgKrQxjZlebmZvZnlFnETCz/5jZ12b2uZm9ama7R52pqDKzdmb2f2Y218z6Rp1HwMz2NbOxZvaVmc0ys8uiziSBmRU3s0/M7I2os6Q7FVoZxMz2BY4Fvos6i2z2HlDP3esDc4DrIs5TJJlZceBR4HigDtDdzOpEm0qADcBV7l4baAJcrP2SMi4Dvoo6RCZQoZVZHgCuAXSGQ4pw9zHuviF2cQpQNco8RVgjYK67z3P3dcAw4KSIMxV57v6ju8+Mfb+C8Id9n2hTiZlVBToAT0WdJROo0MoQZnYi8L27fxZ1Ftmqs4C3ow5RRO0DLMpzeTH6g55SzKw6cCgwNeIoAg8SPrTnRpwjI5SIOoDEz8zeBypv4aYbgOuB45KbSGDb+8XdR8fucwPhMMnQZGaTzWwL12nkN0WYWVngFeByd/8j6jxFmZl1BH5x9xlm1iriOBlBhVYacfc2W7rezA4GagCfmRmEw1MzzayRu/+UxIhF0tb2yyZmdibQETjG1bguKouBffNcrgr8EFEWycPMShKKrKHuPjLqPEIz4EQzaw9kAeXN7Hl37xlxrrSlhqUZyMwWAA3dXSuvR8zM2gEDgJbuviTqPEWVmZUgnIxwDPA98DHQw91nRRqsiLPwyXAI8Ku7Xx5xHMknNqJ1tbt3jDhKWtMcLZHEegQoB7xnZp+a2WNRByqKYickXAK8S5hw/ZKKrJTQDDgdODr28/FpbCRFJGNoREtEREQkQTSiJSIiIpIgKrREREREEkSFloiIiEiCqNASERERSRAVWiIiIiIJokJLRFKCmQ02s/6x75ub2f8labtuZvsn4HkXmNk2m9mKSOZToSUicYsVD6vNLMfMfjazZ2PLpxQqd5/g7gfGkaeXmU0s7O3HnvtxM/vfFq6vb2ZrzewfidiuiGQWFVoiUlAnuHtZ4DDgCODG/HeIdWJPd4OBLmZWJt/1ZwBvuPuvyY8kIulGhZaI7BB3/x54G6gHmw/BXWxm3wDfxK7rGOv2vdzMJplZ/U2PN7NDzWymma0ws+GEddU23dbKzBbnubyvmY00syVmtszMHjGz2sBjQNPYCNvy2H13NbP7zOy72KjbY2ZWKs9z9TGzH83sBzM7axuvbzJhuZ6T8zy2ONADGGJmtczsw1iepWY21Mx239Jz5T0supXXt7eZvRJ7ffPN7N95bmtkZtPN7I/Y6xmwtcwiknpUaInIDjGzfYH2wCd5ru4ENAbqmNlhwDPA+UAF4HHgtVghtAswCngO+AfwMnkKmnzbKQ68ASwEqgP7AMPc/SvgAmCyu5d1991jD7kH+CfQANg/dv+bY8/VDrgaOBY4ANjeHKr/EUawNmkDlCQUmAbcBewN1CYsWn3rdp5vS6+vGPA68Fks6zHA5WbWNnaXh4CH3L08UAt4qaDbEJHoqNASkYIaFRs9mgiMA+7Mc9td7v6ru68GzgUed/ep7r7R3YcAa4Emsa+SwIPuvt7dRxAWet6SRoRipo+7r3T3Ne6+xXlZsUWKzwWuiOVYEct3auwu/wKedfcv3X0l2y+MngNamlnV2OUzgBdimee6+3vuvja2YPgAoOV2nm9LjgAquns/d1/n7vOAJ/NkXg/sb2Z7unuOu0/ZgW2ISEQyYR6FiCRXJ3d/fyu3LcrzfTXgTDO7NM91uxCKJge+978utrpwK8+5L7AwtjD09lQESgMzQs0FhJGn4rHv9wZmxLFNANz9OzMbD/Q0s0cII3bNAcxsL2Bg7HI5wgfX3+LImF81YO9Nhz5jigMTYt+fDfQDvjaz+cBt7v7GDmxHRCKgQktEClPewmkRcIe735H/TmbWEtjHzCxPsbUf8O0WnnMRsJ+ZldhCseX5Li8FVgN1Y3PI8vuRULhtst/WX8pmQ4C+scfOd/eZsevvim2/vrsvM7NOwCNbeY6VhAJwk8p5vl8Ue94DtvRAd/8G6B47xNgFGGFmFWIjciKS4nToUEQS5UngAjNrbEEZM+tgZuWAycAG4N9mVsLMuhAOEW7JNEKRc3fsObLMrFnstp+BqrE5X7h7bmy7D8RGnDCzffLMd3oJ6GVmdcysNHBLHK/jFUJxdhuh6NqkHJADLDezfYA+23iOT4H2ZvYPM6sMXJ7v9f1hZteaWSkzK25m9czsiFj+nmZWMfbalsceszGO3CKSAlRoiUhCuPt0wnypRwiH1OYCvWK3rSOMzvSK3dYNGLmV59kInECY2P4dsDh2f4APgVnAT2a2NHbdtbFtTTGzP4D3gQNjz/U28GDscXNj/27vdazkz2JraJ6bbiO0uPgdeHNr+WOeI0x2XwCMAYZv4fU1AOYTRuWeAnaL3aUdMMvMcggT40919zXbyy0iqcH+OkVCRERERAqLRrREREREEkSFloiIiEiCqNASERERSRAVWiIiIiIJokJLREREJEFUaImIiIgkiAotERERkQRRoSUiIiKSIP8PLOAGliYImM4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## plotting the huber_fn function\n",
    "plt.figure(figsize=(10, 4))\n",
    "pred_vals = np.linspace(-5, 5, 400)\n",
    "losses = huber_fn(y_true=0, y_pred=pred_vals)\n",
    "plt.plot(pred_vals, losses, 'b', label='Huber')\n",
    "plt.axvline(-1, c='r', linewidth=1, label='threshold (small error)')\n",
    "plt.axvline(1, c='r', linewidth=1)\n",
    "plt.axvline(0, c='k', linewidth=2)\n",
    "plt.title('Huber Loss Function', fontsize=14)\n",
    "plt.xlabel('Predicted Values', fontsize=12)\n",
    "plt.ylabel('Huber Loss Values', fontsize=12)\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.show()   ## thresholds between -1, 1 ==> small error ==> squared MSE, else ==> linear MAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "86f7600f",
   "metadata": {},
   "outputs": [],
   "source": [
    "## create a General Class, more flexiblity\n",
    "class HuberLoss(tf.keras.losses.Loss):\n",
    "    def __init__(self, threshold, **kwargs):   ## **kwargs => to handle the main Hyperparameters\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)             ## inherit from the main class\n",
    "    \n",
    "    def call(self, y_true, y_pred):\n",
    "        ''' this function is used instead of using MSE or MAE, it combine both of them\n",
    "            when the loss is less than a specified threshold, it will be as MSE\n",
    "            else when the loss is high than specified threshold, it will be as MAE\n",
    "        '''\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        like_MSE = tf.square(error) / 2\n",
    "        like_MAE = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        \n",
    "        ## when erroe is less than trheshold => then like_MSE, else like_MAE\n",
    "        return tf.where(is_small_error, like_MSE, like_MAE)\n",
    "\n",
    "    def get_config(self):\n",
    "        ''' this function is to save what model threshold is specified\n",
    "        '''\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, 'threshold': self.threshold}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d5faa6e",
   "metadata": {},
   "source": [
    "`Now you can build your Model, compile it and pass the (HuberLoss) class to loss hyperparameter` <br />\n",
    "`fit the Model and save it` <br />\n",
    "`When you loading the Model again, you must specify a dictionary of custom_objects`\n",
    "\n",
    "``` python\n",
    "        ## when compiling the model, then fit it, then save it\n",
    "        model.compile(loss=HuberLoss, optimizer='adam', metrics=['mae'])\n",
    "        model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))\n",
    "        model.save('my_model_with_HuberLoss.h5')\n",
    "        \n",
    "        ## when loading the model again, you must specifiy as dictionary of custom objects\n",
    "        my_model = tf.keras.models.load_model('my_model_with_HuberLoss.h5',\n",
    "                                              custom_objects={'HuberLoss': HuberLoss})\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1de2b80",
   "metadata": {},
   "source": [
    "`Now you can build your Model, compile it and pass the (HuberLoss) class to loss hyperparameter` <br />\n",
    "`fit the Model and save it` <br />\n",
    "`When you loading the Model again, you must specify a dictionary of custom_objects`\n",
    "\n",
    " python\n",
    "        ## when compiling the model, then fit it, then save it\n",
    "        model.compile(loss=HuberLoss, optimizer='adam', metrics=['mae'])\n",
    "        model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))\n",
    "        model.save('my_model_with_HuberLoss.h5')\n",
    "        \n",
    "        ## when loading the model again, you must specifiy as dictionary of custom objects\n",
    "        my_model = tf.keras.models.load_model('my_model_with_HuberLoss.h5',\n",
    "                                              custom_objects={'HuberLoss': HuberLoss})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddb6a390",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58fd7c97",
   "metadata": {},
   "source": [
    "## Custom Activation Function, Initializers, Regularizers, and Constraints"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5562850a",
   "metadata": {},
   "source": [
    "#### `Custom soft_plus Activation Function`\n",
    "\n",
    "$$f(x) = In(e^{x} + 1)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5359abdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "## for custom Activation function\n",
    "## let's implement soft_plus Activation function\n",
    "def my_softplus(x):\n",
    "    ''' this Function is to create a custom Activation Function (soft_plus)\n",
    "    '''\n",
    "    soft_plus = tf.math.log(tf.math.exp(x) + 1.0)\n",
    "    return soft_plus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e0cad1d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAEaCAYAAABZ+IDUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1AklEQVR4nO3deXxU1fnH8c/DJiq4onFBCy64QFuEIFVcErFWxda6IhVFXHCrCxWtVuvWUv1Va9FiqRuiiI0LVBQtqJiAoKJhUVEUUVFAEMWKgLKEPL8/zqBDSMiQ3Jmbmfm+X6/7mrnL3PPM4TI8nHPuuebuiIiIiEhmNYo7ABEREZF8pCRMREREJAZKwkRERERioCRMREREJAZKwkRERERioCRMREREJAZKwkSynJkdb2YfmFmFmQ1LUxltzMzNrDAd5083Myszs8EZKOcsM1ue7nIaKjMrSlwnreKORSQbKAkTiYmZ7WBm/zSzuWa2ysw+N7PxZvbzTTzV/cBI4EfAZWY2zMzGRB9x5pjZW4mkst0mfq6mJOhE4Jpoovu+LDezk6tsfgzYI8pyaij7xkT5VZdfp7vspBjmmtmAKptfAXYGlmQqDpFs1iTuAETy2EhgC+AcYA6wI3A4sH2qJzCzbYBWwDh3X5DYFnmgmWRmBwI7AA8T6ub39T2nu39V33OkWM53wHeZKAt4Hyiqsu1/GSq7Wu6+GlgUZwwi2UQtYSIxSCRPhwJXu/t4d//E3d9w99vdvSTpuG3N7CEz+5+ZfWdmL5pZ+8S+In74R/elREtIGdAH6JHUOlJUSyzruhp/Y2aTzGylmb1nZkdt5DMbdDtV7bI0s6ZmdpeZfZZo6ZtnZremUD3nAI8CDwJ9zGy9/yya2VZmNsTMFiZinWVmPRPf80Fgy6TvfmPiM993R5rZLWY2tZrv9IqZ3Zl438XMnjezL83sm0S9HJR07NzE2ycS5cxNbN+gJc7MzjezOWa2OvF6XpX9bmb9zOwJM1thZh+ZWe8U6qnC3RdVWVbVEMN6f17rjjGz7mY2M1FuqZm1rfK5HmY2JXHtLTGzZ8yseeI6+xFw27q6rq6cxLYTzeztpGvgWkv6n0KiRe06M7snUdfzzezKFL6/SNZTEiYSj+WJ5Vdm1nwjxw0DugLHAwcC3wJjzWxzQtdP+8RxJxG6gX4FPA68mFjfOXFcKv4K3AV0BF4ARpvZril/ow1dCpwAnAbsDfQktN7UyMy2SBz/CDCJ8H2PS9pvwH8JLYZ9gf2B3wGrCd/z8sRn1n3326spZjjQycz2TTpvW+CgRLkALRPHHUqo9xnAc0nJRZfE63mJctatV/0+JwCDgUFAB+BO4J9m9ssqh14PjAZ+SujSHGpmP6runBHajNBFezbhu28D/Csp9qMTMb0AdAaKgQmEfzdOBOYDN/NDXW/AzDoDTwCjgB8DVyfK/G2VQ/sDbwOdgP8D/pqc9IrkLHfXokVLDAshcfoKWAm8SkgYuibt3xtw4LCkbVsDS4FzE+utEscUJR0zDBizCXG0SZzj2qRtjYDZwJ+rHFOYWC9KrLeq5jzrjrkLGA/YJsTSF5iZtH4z8EzS+s+BSmC/Gj5/FrC8mu1lwOCk9enAn5LWrwPe30hcBiwEeidtc+DkjZUPTAaGVjlmGDCpynluSVpvQkgke28knhuBtfyQzC8H3qmpDqr+eSWOcWCfpGNOJySzjZJiL9lIDHOBAbWUMwJ4qZrY51c5z7+rHPMBcF0m/h5q0RLnopYwkZi4+0hgF+CXhNadg4HXzOwPiUP2IyQcryZ9ZimhxWD/NISUXE4lMKWe5QwjtKrNNrO7E11btf3mnENogVpnOHC0me2SWD8AWOjus+oRF4QWr98krZ/OD61gmNmOie6x2Wa2FFhGGLO3+yaWsx8hmUk2iQ3r9a11b9y9AvgiUd7GfEio33XLsZsY2yp3T26Z/AxoSmgRg1DX4zfxnFXV9P13NbOtkra9VeWYz6j9+4tkPSVhIjFy95Xu/oK73+zuBwMPADeaWTNC60uNH81MhDWqTLwmx9g0+QB3n0ZoHfsD4bfmIeCFmhKxRPdgN+AvFu6MrABmEVqG+lZTXn08CrQxs4PMrBOwL6HVZp2HCF2M/QnJcUdC91uzOpRV3Z9V1W1rqtlf2+/zanefk7R8ktheyYb11JQNVdQQU5T/Lhg1X6vJ2+vy/UWyni5ykYblXULS0TzxvhFhvA4QBqUTxta8u5FzrAYa16HsnyWVY4SxUDW1OH2ReE0eC9Sx6kHuvszdn3D3C4EewBHAXjWc8xxC69tPWb+F50bg7ERM04CdzWy/Gs6R0nd394XAS4QWsNOBV9z9o6RDDgH+4e7Puvs7hJawquOe1qRQ1qzEuZIdwsb//OrrC2CLKi1NHetwnulA943sT6Wu36X67z/f3ZfVISaRnKIpKkRiYGbbEwYsDyV0xSwDCoGrgPHu/g3wjZmNBu4xs37A18BA4BtCS05N5gLHmNk+hPmalrp71ZaG6lxoZrMJ3Z0XEe5+G1LDsXOAeYRWu6sJLV7XVfmOvyOMo5pBSFh+k4h9ftWTmVlT4ExgoLvPrLJvCWHgejGhe2wKMNLM+hPGre0FbOnuTyW+e3MLc61NB751929r+A6PEMbhrQb+XGXfbKC3mU0BtiTctLC6yjFzge5mNoHQtVfd9BC3Ee6gnAo8DxxNSPpOrCGmKEwBVgC3mNnfCUntRXU4z0DgGTObQ7jeDDgKuCdRp3OBQ83sEcL3/7Kac/wNeCNxl+qjhNbFKwitoyJ5Ty1hIvFYDrwGXEa44+wd4C+Ef6h6Jh3XF3gdeDrxugVwtIf5qGpyH6EFppzQKtItxZiuJtxp+CYhWTjB3TdImAASSd1phIlJ3wRuYsN/WJcBVybinkZojTmmhqTol4S5wUZWU9ZCwriicxNj1Y5JrD+S+J53kugmdPdXCHf4/Zvw3a/ayPddN0/bDoQ7SpOdDbQApgIlhGR5bpVjriAkhvMICd8GEonhJYRuzXcJf94XufszG4mrXjzMiXY64SaGt4F+wB/rcJ7nCHe3HkP4fhMI33ddV/T1wG6EsWlf1HCOacAphJtQZgK3Jpa0P71AJBuYe9xDS0QkTmbWBvgY6OLu5TGHIyKSN9QSJiIiIhIDJWEiOc7M/pCYHb265b9xxycikq/UHSmS48xsO2C7GnZ/54lnToqISGYpCRMRERGJQdZNUdGqVStv06ZNWstYsWIFW265ZVrLyDeq02ipPqM1dWp4nnfnzp1jjiR36BqNnuo0epmo06lTp37p7jtUty/rWsIKCwu9vDy9N3CVlZVRVFSU1jLyjeo0WqrPaIV5YMOzdCUaukajpzqNXibq1Mymunthdfs0MF9EREQkBkrCRERERGKgJExEREQkBlk3ML86a9asYf78+axcuTKS82299dbMmlXTc4tzT/PmzWndujVNmzaNOxQREZG8kRNJ2Pz582nZsiVt2rT5foBtfSxbtoyWLVtGEFnD5+4sWbKE+fPn07Zt27jDERERyRs50R25cuVKtt9++0gSsHxjZmy//faRtSKKiIhIanIiCQOUgNWD6k5ERCTzciYJExEREUnFihXQvz8sXRrvWOicGBMmIiIikooPPoATT4R33oEWLbbl+OPji0UtYREaOHAg7du35yc/+QkdO3ZkypQpNR778ssv0759ezp27Mirr77Kc889V6cy586dS4cOHeoasoiISN4YPRoKC+Gzz2DsWOjefXGs8SgJi8irr77KmDFjmDZtGm+99RYvvvgiu+22W43HjxgxggEDBjBjxgzef//9OidhIiIisnEVFXDNNfDrX0O7djBtGhx1VNxR5WB35OWXw4wZ9TvH2rWb07jxD+sdO8KgQRv/zMKFC2nVqhWbbbYZAK1atQJg/PjxDBgwgIqKCrp06cKQIUMYPnw4jz/+OOPGjeP5559n8uTJfPfdd0yaNIlrrrmGnj17bnD+G2+8kQ8//JAFCxYwb948rrrqKs4777z1jhk2bBjl5eUMHjwYgOOOO44BAwZw6KGHcs4551BeXo6ZcfbZZ9O/f/8614+IiEi2+OIL6NULxo+Hfv3gzjuhefO4owpyLgmLy1FHHcXNN99Mu3btOPLII+nZsyddu3blrLPOYvz48bRr144zzzyTIUOGcPnllzNp0iSOO+44Tj755A2Sp5q89dZbvPbaa6xYsYIDDjiAHj16pBTbjBkzWLBgATNnzgTg66+/ru/XFRERafCmTIGTTw6J2NCh0Ldv3BGtL+eSsNparFKxbNl3mzxZa4sWLZg6dSovv/wypaWl9OzZk2uuuYa2bdvSrl07APr06cPdd9/N5ZdfXqe4jj/+eDbffHM233xziouLef311+nYsWOtn9tjjz346KOPuOSSS+jRowdHNYQ2WBERkTRxh3vugUsvhV13hVdegU6d4o5qQxoTFqHGjRtTVFTETTfdxODBg3nxxRcjPX/V+byqrjdp0oTKysrv19dNwLrtttvy5ptvUlRUxN133825554baVwiIiINxbffwllnwYUXwpFHwtSpDTMBAyVhkXn//ff54IMPvl+fMWMGBQUFzJ07lzlz5gAwfPhwDj/88A0+27JlS5YtW1ZrGaNHj2blypUsWbKEsrIyunTpst7+Nm3aMGPGDCorK5k3bx6vv/46AF9++SWVlZWcdNJJ/OlPf2LatGn1+aoiIiIN0ocfwkEHwfDhcOONMGYMbLdd3FHVLOe6I+OyfPlyLrnkEr7++muaNGnCXnvtxb333kuvXr045ZRTvh+Yf8EFF2zw2eLiYm699VY6duxY48B8gAMPPJAePXrw6aef8sc//pFddtmFuXPnfr+/W7dutG3blh//+Md06NCBTonUf8GCBfTt2/f7VrJbbrkl+goQERGJ0Zgx0Ls3NGoEzz4LxxwTd0S1UxIWkc6dO/PKK69ssL179+5Mnz59g+3Dhg37/v12223HG2+8UWsZ7dq14957711vW5s2bb4fcG9mjBgxotrPqvVLRERy0dq1odXrz3+GAw6AkSOhbdu4o0pNRrojzWyomS02s5nV7BtgZm5mrTIRi4iIiOSGJUvg2GNDAnb22TB5cvYkYJC5lrBhwGDg4eSNZrYb8HPg0wzF0eA9+OCD3Hnnnett69atG3fffXdMEYmIiDQ85eVw0kmwaBHcdx9k4z1nGUnC3H2imbWpZtffgauA0RGUscHdgtmob9++9M3wRCbuntHyRERE6uP+++Hii2GnnULrV2Fh3BHVjWXqH+BEEjbG3Tsk1n8FdHf3y8xsLlDo7l/W8Nl+QD+AgoKCziUlJevtb9GiBQUFBWy99daRJGJr166lcfKU+TnM3Vm6dCmff/45y5cvT1s5y5cvp0WLFmk7f75RfUaruLgYgNLS0pgjyR26RqOnOoVVqxpx551789//7kxh4Vdcd90stt56TZ3Pl4k6LS4unuru1aaJsSRhZrYFUAoc5e5La0vCkhUWFnp5efl629asWcP8+fO/nxervlauXEnzhvJMgwxo3rw5rVu3pmnTpmkro6ysjKKiorSdP9+oPqO17j9vahWOjq7R6OV7nX78ceh+nD4drrsuDMavb3tJJurUzGpMwuK6O3JPoC3wZuLHrzUwzcwOdPdFm3qypk2b0jbCkXhlZWUccMABkZ1PRERE6u6//4XTT4fKSnjmGTjuuLgjikYsk7W6+9vuvqO7t3H3NsB8oFNdEjARERHJTZWVcNNN0KMH7L57mP0+VxIwyNwUFf8GXgX2MbP5ZnZOJsoVERGR7PTVVyHhuvFGOOOM8PzHPfeMO6poZeruyF617G+TiThERESk4Zs+HU48ERYsgCFD4PzzIQcmQNiAnh0pIiIiDcaDD4bnP1ZUwMsvwwUX5GYCBkrCREREpAFYuRL69Qsz3x9yCEybBl27xh1VeikJExERkVh98gkcemiY+f6aa2DcONhhh7ijSj89wFtERERi8/zz0KtX6H586ik4/vi4I8octYSJiIhIxlVWhgdvH3007LJLeBZkPiVgoJYwERERybCvvw7TTowZEyZhvece2HLLuKPKPCVhIiIikjFvvhmmn/j0Uxg8GC66KHfvfqyNuiNFREQkIx5+GH72s3An5MSJcPHF+ZuAgZIwERERSbNVq0KLV58+IQmbNi3MBZbvlISJiIhI2sybB4cdFma+v/JKeOEFKCiIO6qGQWPCREREJC3Gj4fTTgstYU8+CSedFHdEDYtawkRERCRS7nDrrXDUUbDjjvDGG0rAqqOWMBEREYnM0qVh7Nfo0dCzJ9x/P7RoEXdUDZOSMBEREYnE22+H6SfmzoVBg+DSS/P77sfaKAkTERGRehsxIjyAe6utoLQ0PIRbNk5jwkRERKTOVq+GSy6B3r2hc+cw/YQSsNQoCRMREZE6WbAAiorCzPf9+4e7IXfeOe6osoe6I0VERGSTlZWFgfcrVsBjj8Gpp8YdUfbJSEuYmQ01s8VmNjNp221m9p6ZvWVm/zGzbTIRi4iIiNSdO9x2Gxx5JGy3Hbz+uhKwuspUd+Qw4Ogq214AOrj7T4DZwDUZikVERETq4Jtv4OST4aqr4IQTQgK2//5xR5W9MpKEuftE4Ksq255394rE6mtA60zEIiIiIpvunXfgwAPD/F+33w6PPw4tW8YdVXYzd89MQWZtgDHu3qGafc8Aj7n7IzV8th/QD6CgoKBzSUlJOkNl+fLltNDMcpFSnUZL9Rmt4uJiAEpLS2OOJHfoGo1enHX60ks7cNtt+7L55mu5/vp36NhxaSxxRC0TdVpcXDzV3Qur2xd7EmZm1wKFwImeQjCFhYVeXl6eniATysrKKCoqSmsZ+UZ1Gi3VZ7QsMZtkpn4P84Gu0ejFUadr1oSux0GD4OCD4YknYJddMhpCWmWiTs2sxiQs1rsjzawPcBzQPZUETERERDJj4cIw4H7SpDDz/W23QbNmcUeVW2JLwszsaOD3wOHu/m1ccYiIiMj6Xn45JGDffAOPPgq9esUdUW7K1BQV/wZeBfYxs/lmdg4wGGgJvGBmM8zsX5mIRURERKrnDn//OxQXh0H3U6YoAUunjLSEuXt1f4QPZKJsERERqd2yZXDuueGuxxNOgAcfhK23jjuq3KbHFomIiOS5996Drl3hySfh//4PRo5UApYJemyRiIhIHnvySejbFzbfHF54AY44Iu6I8odawkRERPJQRQUMGACnnALt28O0aUrAMk0tYSIiInlm0SI47TSYMAEuugjuuAM22yzuqPKPkjAREZE8MnlyaP36+mt4+GE444y4I8pf6o4UERHJA+5w111QVARbbAGvvaYELG4pJWFm1rWG7QdGG46IiIhEbcUKOP10uOwyOOYYKC+Hn/wk7qgk1ZawF2rYPjaqQERERCR6s2eH6SceewwGDoSnnoJttok7KoFaxoSZWSPAwluzxPt19gQq0hibiIiI1MN//gN9+oRnPo4dCz//edwRSbLaWsIqgNXAFon3a5KWd4F/pjU6ERER2WQVFXD11XDiibDvvmH6CSVgDU9td0e2JbR+TQAOS9ruwBfu/l26AhMREZFNt3hxeN7jSy/B+efDnXdq+omGaqNJmLt/knj7owzEIiIiIvXw2mtw8smwZEl49uNZZ8UdkWxMSvOEmdnDNe1z9zOjC0dEREQ2lTsMGQKXXw6tW8Orr0LHjnFHJbVJ9e7ID6ssK4BjgK/SFJeIiIik4Ntvw+D7iy8O476mTlUCli1Saglz95uqbjOzB4AbIo9IREREUjJnDpx0Erz9Ntx8M1x7LTTSNOxZoz6PLZoBHB5RHCIiIrIJnnkmzHjfuDE89xwcfXTcEcmmSnVMWNXnqm8BnEaYpkJEREQyZO1auOGGMPFqp04wciS0aRN3VFIXqbaEPVBlfQWhJaxXKh82s6HAccBid++Q2LYd8BjQBpgLnOru/0sxHhERkbzz5Zfwm9/ACy/AOefA4MHQvHncUUldpdRz7O5tqywd3L23u3+cYjnDgKoNpVcD4919b2B8Yl1ERESq8d57LencGSZOhPvug/vvVwKW7VIeE2Zm2wA9gF2Az4DnUm25cveJZtamyubjgaLE+4eAMuD3qcYjIiKSD9xD0nXppQewyy4weTJ07hx3VBIFc/faDwpjwkYB7wOfALsD+wInufv4lAoKSdiYpO7Ir919m6T9/3P3bWv4bD+gH0BBQUHnkpKSVIqss+XLl9OiRYu0lpFvVKfRUn1Gq7i4GIDS0tKYI8kdukajsWpVIwYN2puxY3emU6fFXH/9bLbeWo9tjkomrtPi4uKp7l5Y3b5Uk7B3gRvd/fGkbacAf3L3fVMJoj5JWLLCwkIvLy9Ppcg6Kysro6ioKK1l5BvVabRUn9EyMwBS+T2U1Ogarb+PPgrTT8yYAddfD4cdVkb37kVxh5VTMnGdmlmNSViqs4nsAoyssu0/wE71iOtzM9sZIPG6uB7nEhERyRnPPRe6HOfOhTFj4KabwlQUkltSTcIeBi6usu3CxPa6ehrok3jfBxhdj3OJiIhkvXXTT/ToEaadmDo1vJfclOrA/E7AhWZ2FbAA2BXYEZhiZhPXHeTuh1X3YTP7N2EQfiszm0+Yaf9W4HEzOwf4FDilrl9CREQk2y1ZAr17w9ix4TFEQ4bA5pvHHZWkU6pJ2H2JpU7cvab5xLrX9ZwiIiK5YurUMP5r4UL417+gXz9IDFWUHJZqEvaeu0+putHMDnT31yOOSUREJG888EB4+PaOO8KkSdClS9wRSaakOibshRq2j40qEBERkXyyciWcdx6cey4ceihMm6YELN9sNAkzs0Zm1ji8NUusr1v2BjRZiYiIyCaaOxcOOSTMev+HP4RxYK1axR2VZFpt3ZEVgCe9T1YJDIw8IhERkRw2diycfnq4E3L0aPjVr+KOSOJSWxLWFjBgApB856MDX7j7d+kKTEREJJdUVsKf/ww33gg//jGMHAl77RV3VBKnjSZh7v5J4u2PMhCLiIhITvrqKzjjjDAJ6xlnhDsgt9gi7qgkbindHWlmNU7K6u5nRheOiIhIbpk+HU48ERYsgH/+Ey64QNNPSJDq3ZEfVllWAMcAX6UpLhERkaz34INw8MFQUQETJ8KFFyoBkx+k1BLm7jdV3WZmDxBmvhcREZEkK1fCZZfBvffCEUdASQnssEPcUUlDk2pLWHVmAIdHFIeIiEhO+OSTMO/XvffC1VfDuHFKwKR6qY4JO6LKpi2A04B3I49IREQkSz3/PPTqFbofn3oKjj8+7oikIUv1sUUPVFlfQWgJq+mZkCIiInmjshIGDoQbboD27WHUKNh777ijkoYu1TFhbdMdiIiISDb63//CtBPPPhsmYb3nHthyy7ijkmyQanfkHkB3oBXwBfCSu3+UzsBEREQauhkz4KSTYN48GDwYLrpIdz9K6modmG9mdwKzgWuBXwHXAbPN7K40xyYiItJgPfQQHHQQrFoFEybAxRcrAZNNU9sDvAcQ5gM72N3buPtB7t4GOBg42syuzECMIiIiDcaqVWHC1bPOCknYtGnhVWRT1dYSdh7Q291fT96YWD8zsV9ERCQvfPppmH7innvgqqvC3ZA77hh3VJKtahsT1hoor2Hf68Cu9Q3AzPoD5xIeCv420NfdV9b3vCIiIlF68UU47TRYvTrc/XjCCXFHJNmutpawL4F9ati3L7CkPoWb2a7ApUChu3cAGhPmHxMREWkQKivhL3+BX/wCdtoJysuVgEk0akvCHgUeNLNdkjcmkqehwIgIYmgCbG5mTQiTwH4WwTlFRETq7euv4de/hmuvhZ49YcoUaNcu7qgkV5i717zTrBnwJHAkMAVYCOwMdAXGAye5++p6BWB2GTAQ+A543t1Pr+aYfkA/gIKCgs4lJSX1KbJWy5cvp0WLFmktI9+oTqOl+oxWcXExAKWlpTFHkjty4RqdM2dLbrihA59/vhkXXfQhJ5ywINa7H3OhThuaTNRpcXHxVHcvrG7fRpOw7w8y605IxFoRuihfdPfx9Q3MzLYFRgI9ga+BJ4An3f2Rmj5TWFjo5eU1DVOLRllZGUVFRWktI9+oTqOl+oyWJf5lTeX3UFKT7dfo8OFw/vmw7bbw+OPQrVvcEWV/nTZEmahTM6sxCUt1xvzxhJavqB0JfOzuXwCY2SjC9Bc1JmEiIiLpsmoV9O8PQ4bA4YfDY49BQUHcUUmuqnWy1jT7FPiZmW1h4b+i3YFZMcckIiJ5aN68kHgNGQJXXhnuhlQCJumU6gO808Ldp5jZk8A0oAKYDtwbZ0wiIpJ/xo8P00+sWgVPPhkeRSSSbnG3hOHuN7j7vu7ewd3PcPdVccckIiL5obISbrkFjjoqTLr6xhtKwCRzYm0JExERicvSpdCnD4weHaafuP9+0M2HkkkpJWFm1pYwjURHYL1L1N13jz4sERGR9Hn7bTjxRJg7FwYNgksv1cO3JfNSbQl7FPgQuAL4Nn3hiIiIpNeIEXDeebDNNlBaCoccEndEkq9STcLaA93cvTKdwYiIiKTL6tXwu9/B3XfDYYeF6Sd22inuqCSfpTowfyJwQDoDERERSZf588P0E3ffDVdcEaafUAImcUu1JWwuMC4xmeqi5B3ufn3UQYmIiESltDQMvP/uuzD7/SmnxB2RSJBqS9iWwDNAU2C3pKV1muISERGpF3f461/hyCOhVSt4/XUlYNKwpPrYor7pDkRERCQqS5dC377wn/+ExOuBB6Bly7ijEllfyvOEmdneQC9gV2AB8G93/yBdgYmIiNTFzJlh+omPPoI77oDLL9f0E9IwpdQdaWa/BKYC+wJfAfsA5Wb2qzTGJiIiskkefRS6doVly8JYsP79lYBJw5VqS9hfgOPdvXTdBjMrAgYDT0cfloiISOpWr4YBA+Af/wjzfj3+OOy8c9xRiWxcqgPzWwMvV9k2CQ3MFxGRmC1YAMXFIQHr3x9eekkJmGSHVJOwGYTZ8pP9LrFdREQkFmVl0KkTvPlmmHz1jjugadO4oxJJTardkRcCz5jZZcA8wvQUKwCNCRMRkYxzh9tvh2uugb32CuO/9t8/7qhENk2qU1S8Z2b7AQcBOwOfAVPcfU06gxMREanqm2/C9BOjRsFJJ8HQobDVVnFHJbLpUp6iwt0r2HBcmIiISMa8806YfuLDD0NL2O9+p7sfJXvVmISZ2Sx33y/xfh7g1R3n7runKTYREZHvPfYYnHMOtGgB48eHZ0GKZLONtYSdl/S+d7oDERERqc6aNXDllXDnndCtW5h+Ypdd4o5KpP5qTMLcfVLS6o7u/kTVY8zs5PoGYGbbAPcDHQitbWe7+6v1Pa+IiGS/zz6DU0+FyZPhssvgttt096PkjlSnqHighu33RhDDncBYd98X+CkwK4JziohIlpswIUw/MX16mAl/0CAlYJJbNjow38z2SLxtZGZtgeThj3sAK+tTuJltBRwGnAXg7quB1fU5p4iIZDf3MN/X738Pe+4Zxn+1bx93VCLRM/dqx9uHnWaVhC7C6u49WQTc5O731Llws46E1rR3Ca1gU4HL3H1FleP6Af0ACgoKOpeUlNS1yJQsX76cFi1apLWMfKM6jZbqM1rFxcUAlJaW1nKkpKqu1+i33zbmr3/dhwkTduTQQ7/g979/jy23XJuGCLOP/t5HLxN1WlxcPNXdC6vbt9Ek7PuDzCa4e+T3oZhZIfAa0M3dp5jZncA37v7Hmj5TWFjo5eXlUYeynrKyMoqKitJaRr5RnUZL9RktS8xxkMrvoaSmLtforFlh+onZs+HWW8OzIDX9xA/09z56mahTM6sxCUt1TNjpZrZtlZNua2b1vT9lPjDf3ack1p8EOtXznCIikmUefxy6dIGvvoIXXwx3QyoBk1yXahL2HzZ8WHfrxPY6c/dFwDwz2yexqTuha1JERPLAmjXhods9e8JPfgLTpoWHcYvkg1RnzN/H3d9O3uDub5vZvhHEcAkwwsyaAR8BfSM4p4iINHALF4bpJyZNgksuCTPgN2sWd1QimZNqErbYzPZy9znrNpjZXsCS+gbg7jOAavtKRUQkN02cGBKwZctgxAj4zW/ijkgk81LtjhwKjDSz48xsfzP7JWH81v3pC01ERHKNe2jxOuII2HprmDJFCZjkr1Rbwm4F1gC3A7sB8wgJ2B1piktERHLM0qVw9tkwahScdBIMHQpbbRV3VCLxSSkJc/dK4LbEIiIiskneeiskXh9/DH/7WxiMr7sfJd+l2hJG4g7GnwLrzWrm7kOjDkpERHLH8OFw/vmwzTZQWgqHHhp3RCINQ0pJmJn9AbgeeBP4NmmXE8aLiYiIrGfVKrj8cvjXv+Dww6GkBHbaKe6oRBqOVFvCLgcOdPe30hiLiIjkiEWLNuOQQ6C8HK66CgYOhCYp972I5IdU/0p8B7yXzkBERCQ3jB0L558fZh4aNQpOOCHmgEQaqFSnqPgj8A8z29nMGiUv6QxORESyR2Ul3HQTHHsstGq1ivJyJWAiG5NqS9iwxOu5SduMMCascZQBiYhI9vnyS+jdG8aNgzPPhF69prH33ofFHZZIg5ZqEtY2rVGIiEjWeuMNOPlkWLQI7rkHzjsPJkyojDsskQYv1XnCPkl3ICIikl3cQ9J12WWw884weTIU6iF0IilLdYqK4YSuxw24+5mRRiQiIg3eihVwwQXwyCNw9NHhdfvt445KJLukOrB+DvBh0rICOAb4Kk1xiYhIAzV7NvzsZ+HB2zfdBM8+qwRMpC5S7Y68qeo2M3sAuCHyiEREpMEaNQrOOguaNYP//hd+8Yu4IxLJXvWZYmIGcHhEcYiISAO2Zg0MGBCe/7jffjBtmhIwkfpKdUzYEVU2bQGcBrwbeUQiItKgLFwIPXvCyy/DRRfBHXfAZpvFHZVI9kt1iooHqqyvILSE9Yo0GhERaVAmToRTT4VvvgkP4u7dO+6IRHLHRpMwM9vJ3Re5e1rnCTOzxkA5sMDdj0tnWSIiUjt3+Nvf4OqrYc894cUXoUOHuKMSyS21jQmbnbxiZqPSFMdlwKw0nVtERDbB//4XHjd05ZXw61+HyViVgIlEr7YkzKqsF0UdgJm1BnoA90d9bhER2TRTp0LnzmHaiUGD4IknYKut4o5KJDeZe7VzsIadZt+4+1ZJ61+5+3aRBmD2JHAL0BIYUF13pJn1A/oBFBQUdC4pKYkyhA0sX76cFi1apLWMfKM6jZbqM1rFxcUAlJaWxhxJfNxh9Ohd+Oc/92LbbVdzww3vsv/+39T5fLpGo6c6jV4m6rS4uHiqu1f7LInaBuY3MbNifmgRq7qOu79U18DM7DhgsbtPNbOimo5z93uBewEKCwu9qKjGQyNRVlZGusvIN6rTaKk+0yNf63TZMujXD0pK4JhjYPjw5my/fad6nVPXaPRUp9GLu05rS8IWA0OT1pdUWXdgj3qU3w34lZkdCzQHtjKzR9xd99+IiGTAzJnh4dsffAADB4aB+I3qM4OkiKRso0mYu7dJZ+Hufg1wDUCiJWyAEjARkcx46CG48MIw5uvFFyHRKysiGaL/74iI5JnvvoNzzw2PH+raFWbMUAImEocGk4S5e5nmCBMRSa91D99+4AG49lp44QXYaae4oxLJT6nOmC8iIlnuiSfgnHPCw7efey4MwheR+DSYljAREUmP1avh0kvD44fat4fp05WAiTQESsJERHLYJ5/AoYfCP/4B/fvDhAmw225xRyUioO5IEZGc9dRT0LcvVFbCqFHhUUQi0nCoJUxEJMesXAmXXBKSrr32gmnTlICJNERKwkREcsgHH8DBB8PgwaH7cfJk2HPPuKMSkeqoO1JEJEeMGAEXXBDufnz6afjlL+OOSEQ2Ri1hIiJZbsUKOPts6N0bOnYMk68qARNp+JSEiYhksZkzoUsXGDYMrrsOSkt196NItlB3pIhIFnKH++8P839tvTU8/zwceWTcUYnIplBLmIhIllm6FHr1gn79whxgb76pBEwkGykJExHJIpMnh3FfTz4Jt9wCY8dCQUHcUYlIXSgJExHJAhUVcMMNcNhhYAYvvwxXXw2N9CsukrU0JkxEpIH76CM4/XR47TXo0wfuugu22iruqESkvpSEiYg0UO4wfDj89rehxaukBHr2jDsqEYmKGrJFRBqgr78Og+/79IEDDoC33lICJpJrlISJiDQwEyfCT38KI0fCwIHw0kuw++5xRyUiUYs1CTOz3cys1Mxmmdk7ZnZZnPGIiMRp5UoYMACKisKjhyZPhj/8ARo3jjsyEUmHuMeEVQBXuPs0M2sJTDWzF9z93ZjjEhHJqDfeCF2Ps2aF5z/edhu0aBF3VCKSTrG2hLn7Qneflni/DJgF7BpnTCIimbR6Nfzxj3DQQbBsGYwbB0OGKAETyQdxt4R9z8zaAAcAU2IORUQkI956C848M8x436cPDBoE22wTd1Qikinm7nHHgJm1ACYAA919VDX7+wH9AAoKCjqXlJSkNZ7ly5fTQv8NjZTqNFqqz2gVFxcDUFpampHyKiqMkpLdeOihNrRsWcEVV7xPt25LMlJ2pugajZ7qNHqZqNPi4uKp7l5Y3b7YkzAzawqMAca5+x21HV9YWOjl5eVpjamsrIyioqK0lpFvVKfRUn1Gy8wAyMTvYXk5nHtuaP069VS4+25o1SrtxWacrtHoqU6jl4k6NbMak7C474404AFgVioJmIhItlqxAq64Arp2hcWLYdQoeOyx3EzARCQ1cc8T1g04AzjCzGYklmNjjklEJFLPPw8dOsAdd8B558G778IJJ8QdlYjELdaB+e4+CbA4YxARSZfFi8O8X8OHwz77wIQJ4QHcIiIQf0uYiEjOqaiAf/wD2rWDf/8brrsOZsxQAiYi62swU1SIiOSCiRPDA7fffht+/nO46y7Yd9+4oxKRhkgtYSIiEfjsM+jdGw4/HJYuDQPvx41TAiYiNVNLmIhIPSxbBrffHpa1a8Ps91dfDVtsEXdkItLQKQkTEamDNWvgvvvgppvCAPyePeEvf4E99og7MhHJFkrCREQ2QWVl6Gq89lqYPTt0P44ZA126xB2ZiGQbjQkTEUlBZSWMHAkdO8Ipp0CTJvDMM1BaqgRMROpGSZiIyEZUVsKTT8JPfwonnwyrV8OIEeHh28cdB6aZDkWkjpSEiYhU49tvYciQcHfjKaeEub8efRTeeQd+8xto3DjuCEUk22lMmIhIkkWLYPDgkIB99VXoaiwpCa1gSrxEJEpKwkREEk45BZ56Kkw1cfzx4YHb3bqpy1FE0kNJmIjkrXnz4OGHf1gvLYVLL4ULLoC9944vLhHJD0rCRCSvLFoUBtqXlMDkyevvW7AANtssnrhEJP9oYL6I5LzZs2HQIOjeHXbdFS65BL75Bv78Z5gz54fjlICJSCapJUxEcs4334RWrnHj4Nlnf0i09tsvTLLasye0bx9vjCIiSsJEJOstXgyvvAITJ4Zl+vQwv9dmm8ERR8Dll8Oxx0LbtnFHKiLyAyVhIpI1Kith/nx4802YNg2mTg2vCxaE/c2bw89+BtddB4cdBgcdpAdpi0jDpSRMRBqUigpYuDDcufjxx/D++z8ss2fDd9+F4xo1ChOpFhdDp05hPq8uXTSuS0SyR+xJmJkdDdwJNAbud/dbYw5JRCJWWQlLl8IXX1S/LFoUkq5PP4XPPgvzdK3TqFHoRtxnn9C1uM8+0KFDeIbjllvG9pVEROot1iTMzBoDdwM/B+YDb5jZ0+7+bpxxiWQ795D4rF0bWpbWrq1+qWnf6tWwciWsWpXa64oVsGxZGBC/7jX5/fLlIabqtGgBBQWw226hVWu33WD33cPrj34Ee+6p1i0RyU1xt4QdCMxx948AzKwEOB6ILQl7+GG4/vouG4wjqekfkJq21+UzUW1viGWvWvUzmjWLp+w4t6erjLVrD/n+ETrJCde6pbKy5nii1qxZaJHaaito2TK8brttSKCSt22zDeyww/pLq1aw+eaZi1VEpCGJOwnbFZiXtD4f6Fr1IDPrB/QDKCgooKysLG0BLViwHa1bt6JJkxUb7Kvp0SUbe6SJ2Ub+dd6Ec9W8fdPOH1cZFRVraNKkaWRlbLzO01tGQ/jzXr16Dc2aNV3v+EaNnMaNnUaNwvt161W3he3Jx6+/rVEjp2nTSpo1c5o1q0y8/+F1/W1ep0f6rGsp+/DDTf9sOqXztyXfLF++XPUZMdVp9OKu07iTsOp+vjf4V8zd7wXuBSgsLPSioqK0BVRUBAcdVEY6y8hHZWWq0yipPtNDdRodXaPRU51GL+46jXvG/PnAbknrrYHPYopFREREJGPiTsLeAPY2s7Zm1gw4DXg65phERERE0i7W7kh3rzCz3wLjCFNUDHX3d+KMSURERCQT4h4Thrs/BzwXdxwiIiIimRR3d6SIiIhIXlISJiIiIhIDJWEiIiIiMVASJiIiIhID8409b6UBMrMvgE/SXEwr4Ms0l5FvVKfRUn1GT3UaLdVn9FSn0ctEnf7I3XeobkfWJWGZYGbl7l4Ydxy5RHUaLdVn9FSn0VJ9Rk91Gr2461TdkSIiIiIxUBImIiIiEgMlYdW7N+4AcpDqNFqqz+ipTqOl+oye6jR6sdapxoSJiIiIxEAtYSIiIiIxUBImIiIiEgMlYUnM7BQze8fMKs2sMGl7GzP7zsxmJJZ/xRlntqipPhP7rjGzOWb2vpn9Iq4Ys5mZ3WhmC5Kuy2PjjikbmdnRietwjpldHXc8ucDM5prZ24nrsjzueLKRmQ01s8VmNjNp23Zm9oKZfZB43TbOGLNJDfUZ+2+okrD1zQROBCZWs+9Dd++YWC7IcFzZqtr6NLP9gdOA9sDRwD/NrHHmw8sJf0+6Lp+LO5hsk7ju7gaOAfYHeiWuT6m/4sR1qXmt6mYY4fcx2dXAeHffGxifWJfUDGPD+oSYf0OVhCVx91nu/n7cceSKjdTn8UCJu69y94+BOcCBmY1OBAjX3Rx3/8jdVwMlhOtTJFbuPhH4qsrm44GHEu8fAn6dyZiyWQ31GTslYalra2bTzWyCmR0adzBZbldgXtL6/MQ22XS/NbO3Ek3t6prYdLoW08OB581sqpn1izuYHFLg7gsBEq87xhxPLoj1NzTvkjAze9HMZlazbOx/vwuB3d39AOB3wKNmtlVmIm7Y6lifVs02zZVSjVrqdwiwJ9CRcI3+Lc5Ys5SuxfTo5u6dCN28F5vZYXEHJFKN2H9Dm2S6wLi5+5F1+MwqYFXi/VQz+xBoB+T9gNO61CehtWG3pPXWwGfRRJRbUq1fM7sPGJPmcHKRrsU0cPfPEq+Lzew/hG7f6sbayqb53Mx2dveFZrYzsDjugLKZu3++7n1cv6F51xJWF2a2w7qB42a2B7A38FG8UWW1p4HTzGwzM2tLqM/XY44p6yR+hNc5gXAjhGyaN4C9zaytmTUj3DDydMwxZTUz29LMWq57DxyFrs2oPA30SbzvA4yOMZas1xB+Q/OuJWxjzOwE4B/ADsCzZjbD3X8BHAbcbGYVwFrgAndvcAP8Gpqa6tPd3zGzx4F3gQrgYndfG2esWeqvZtaR0H02Fzg/1miykLtXmNlvgXFAY2Cou78Tc1jZrgD4j5lB+DfmUXcfG29I2cfM/g0UAa3MbD5wA3Ar8LiZnQN8CpwSX4TZpYb6LIr7N1SPLRIRERGJgbojRURERGKgJExEREQkBkrCRERERGKgJExEREQkBkrCRERERGKgJExEREQkBkrCRCSnmNlcM6vLkxw2pYwbzeyRdJYhIrlPSZiIiIhIDJSEiUhOMrOzzGySmd1uZv8zs4/N7Jik/WVmdouZvW5mS81stJltl9hXlJhVO/l8c83sSDM7GvgD0NPMlpvZm5n9ZiKSK5SEiUgu6wq8D7QC/go8YInn6SScCZwN7EJ4hNZdtZ0w8QievwCPuXsLd/9p5FGLSF5QEiYiuewTd78v8WzSh4CdCc82XGe4u8909xXAH4FTzaxxHIGKSP5REiYiuWzRujfu/m3ibYuk/fOS3n8CNCW0momIpJ2SMBHJZ7slvd8dWAN8CawAtli3I9E6tkPSsZ6R6EQkpykJE5F81tvM9jezLYCbgScTXZezgeZm1sPMmgLXAZslfe5zoI2Z6TdUROpMPyAiks+GA8MI3ZbNgUsB3H0pcBFwP7CA0DKWfLfkE4nXJWY2LVPBikhuMXe1qotI/jGzMuARd78/7lhEJD+pJUxEREQkBkrCRERERGKg7kgRERGRGKglTERERCQGSsJEREREYqAkTERERCQGSsJEREREYqAkTERERCQG/w8S+NMD5QQo9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Visulaizing soft_plus\n",
    "plt.figure(figsize=(10, 4))\n",
    "range_soft = np.linspace(-15, 15, 1000)\n",
    "vals_soft = my_softplus(range_soft)\n",
    "plt.plot(range_soft, vals_soft, 'b', label='Soft_plus')\n",
    "plt.axvline(0, c='k', linewidth=2)\n",
    "plt.title('Soft_plus Activation Function', fontsize=14)\n",
    "plt.xlabel('Input', fontsize=12)\n",
    "plt.ylabel('Function Output', fontsize=12)\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.show()   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb236f4a",
   "metadata": {},
   "source": [
    "#### `Custom Intializers `\n",
    "`glorot_normal ==> variance = (1 / fan_avg), mean = 0`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "99756f10",
   "metadata": {},
   "outputs": [],
   "source": [
    "## make a custom glorot_normal intializer\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    ''' this Function tries to make weights intialization with glorot_normal Method\n",
    "        take the shape (fan_in, fan_out)\n",
    "    '''\n",
    "    std_dev = tf.sqrt(2.0 / (shape[0] + shape[1]))\n",
    "    weights_glorot = tf.random.normal(shape, mean=0, stddev=std_dev)\n",
    "    return weights_glorot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b411fa4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAATs0lEQVR4nO3df4xlZ13H8ffXlsZtp922lg5127hgoAa6gN2LVmpwhl+pLaFoiFILtlIyAQNWs42WNAYSQlzRYjAazSqbViCdaCnSFFAqdmxMSmW3tkzLQgu4wi51V1LYMrVSVr/+MbdhvHt/nHPvuT8e9v1KJjv33Oec89lnZj575uw950ZmIkkqzw9NO4AkaTgWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBa7jWkScGREfi4gnIuLfI+JXpp1JqurEaQeQpuxPgaeAeeDFwCci4oHMfGiqqaQKwisxdbyKiFOAbwEXZObD7WUfAg5m5vVTDSdV4CkUHc+eB/zP0+Xd9gDwginlkWqxwHU8mwOOdCw7Apw6hSxSbRa4jmdrwGkdy04DvjOFLFJtFriOZw8DJ0bEczcsexHgf2CqCP4npo5rEbEMJPAW1l+F8kngpb4KRSXwCFzHu18HNgGHgVuAt1neKoVH4JJUKI/AJalQFrgkFcoCl6RCWeCSVKiJ3szqrLPOyq1bt1Ye/8QTT3DKKaeML1ADSsgIZeQ0YzNKyAhl5JyVjHv37v1mZj7zmCcyc2If27dvzzruuuuuWuOnoYSMmWXkNGMzSsiYWUbOWckI7MkuneopFEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKtREL6WXjmdbr/9E1+X7d1424ST6QeERuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQAws8InZHxOGIeLBj+Tsi4ksR8VBEvG98ESVJ3VQ5Ar8JuGTjgohYBC4HXpiZLwD+sPlokqR+BhZ4Zt4NPNax+G3Azsz8bnvM4TFkkyT1EetveDxgUMRW4I7MvKD9+H7g46wfmf83cF1mfq7HukvAEsD8/Pz25eXlyuHW1taYm5urPH4aSsgIZeT8Qc+4evBI1+XbtmweJdIxSphHKCPnrGRcXFzcm5mtzuXD3szqROAM4CLgJcBfR8Rzssu/Bpm5C9gF0Gq1cmFhofJOVlZWqDN+GkrICGXk/EHPeHWvm1ldOdz2eilhHqGMnLOecdhXoRwAbst1/wL8L3BWc7EkSYMMW+B/C7wcICKeB5wEfLOhTJKkCgaeQomIW4AF4KyIOAC8C9gN7G6/tPAp4Kpup08kSeMzsMAz84oeT72x4SySpBq8ElOSCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVathL6aWZsLXX5ek7L5twEmnyPAKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFWpggUfE7og43H7zhs7nrouIjAjfTk2SJqzKEfhNrL/7/P8TEecBrwK+1nAmSVIFAws8M+8GHuvy1B8Bvw34VmqSNAVR5a0sI2IrcEdmXtB+/FrgFZl5bUTsB1qZ2fVNjSNiCVgCmJ+f3768vFw53NraGnNzc5XHT0MJGWF6OVcPHum6fNuWzccsGyZjne03YWPGXvuuq+msfk82Z1YyLi4u7s3MVufy2jezioiTgRuAV1cZn5m7gF0ArVYrFxYWKu9rZWWFOuOnoYSMML2cV/e62dSVC8csGyZjne03YWPGXvuuq+msfk82Z9YzDvMqlB8Hng080D76Phe4LyKe1WQwSVJ/tY/AM3MVOPvpx4NOoUiSxqPKywhvAe4Bzo+IAxFxzfhjSZIGGXgEnplXDHh+a2NpJEmVeSWmJBXKApekQlngklQoC1ySCmWBS1KhLHBJKlTtC3mkkm2tefn7/p2XjSnJYL2yTjOTZotH4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCVXlDh90RcTgiHtyw7A8i4osR8fmI+FhEnD7WlJKkY1Q5Ar8JuKRj2Z3ABZn5QuBh4J0N55IkDTCwwDPzbuCxjmWfzsyj7YefZf2NjSVJE9TEOfA3A59qYDuSpBoiMwcPitgK3JGZF3QsvwFoAb+YPTYUEUvAEsD8/Pz25eXlyuHW1taYm5urPH4aSsgI08u5evBI1+Xbtmw+ZtkwGetsv9/4Xjq3szFj3W1V3cfT6v7dnub3ZHNmJePi4uLezGx1Lh+6wCPiKuCtwCsy87+qhGi1Wrlnz57KoVdWVlhYWKg8fhpKyAjTy1nnjnrDZKx7x75R70a4MWPdbVXdx9OGvRuh35PNmZWMEdG1wIe6nWxEXAL8DvBzVctbktSsKi8jvAW4Bzg/Ig5ExDXAnwCnAndGxP0R8edjzilJ6jDwCDwzr+iy+INjyCJJqsErMSWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKtRQV2JKak5Tl+Tr+OMRuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQVd6RZ3dEHI6IBzcsOzMi7oyIR9p/njHemJKkTlWOwG8CLulYdj3wmcx8LvCZ9mNJ0gQNLPDMvBt4rGPx5cDN7c9vBl7XbCxJ0iCRmYMHRWwF7sjMC9qPv52Zp294/luZ2fU0SkQsAUsA8/Pz25eXlyuHW1tbY25urvL4aSghIzSXc/Xgka7Lt23ZXGt8N/Ob4Owzu2+nbp6mdP69Ns7juPc9rPlNcOjJ7z+u+7XpNb5pJfzszErGxcXFvZnZ6lw+9ptZZeYuYBdAq9XKhYWFyuuurKxQZ/w0lJARmst5dY8bL+2/svu2e43vZse2o/xSzYx1tj+Mzr/Xxnkc976HtWPbUW5c/f6Pdt2vTa/xTSvhZ2fWMw77KpRDEXEOQPvPw81FkiRVMWyB3w5c1f78KuDjzcSRJFVV5WWEtwD3AOdHxIGIuAbYCbwqIh4BXtV+LEmaoIHnwDPzih5PvaLhLJKkGrwSU5IKZYFLUqEscEkqlAUuSYWywCWpUBa4JBVq7JfSS3Vs7XV5987LJpxkXWeeHduOzuwl9OM2a18beQQuScWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKtRIBR4RvxURD0XEgxFxS0T8cFPBJEn9DV3gEbEF+A2g1X63+hOANzQVTJLU36inUE4ENkXEicDJwDdGjyRJqiIyc/iVI64F3gs8CXw6M6/sMmYJWAKYn5/fvry8XHn7a2trzM3NDZ1vEkrICM3lXD14pOvybVs21xrfzfwmOPRk9+ea2H4T+mWcFaNmrDvXvcYPUsLPzqxkXFxc3JuZrc7lQxd4RJwBfBT4ZeDbwN8At2bmh3ut02q1cs+ePZX3sbKywsLCwlD5JqWEjNBczro3NOo1vpsd245y42r3+6s1sf0m9Ms4K0bNWHeuh72ZVQk/O7OSMSK6Fvgop1BeCfxbZv5nZn4PuA146QjbkyTVMEqBfw24KCJOjohg/V3q9zUTS5I0yNAFnpn3ArcC9wGr7W3taiiXJGmAkU7mZea7gHc1lEWSVINXYkpSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVKjZvqmDijHpe5JI8ghckoplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVKiRCjwiTo+IWyPiixGxLyJ+pqlgkqT+Rr0S8wPA32Xm6yPiJODkBjJJkioYusAj4jTgZcDVAJn5FPBUM7EkSYNEZg63YsSLWX8T4y8ALwL2Atdm5hMd45aAJYD5+fnty8vLlfextrbG3NzcUPkmpU7G1YNHui7ftmVzk5G6amoue/0dmjC/CQ492f25XnM0zjzd9Ms4K0bN2ORc99tWt5yT+FmoY1Y6aHFxcW9mtjqXj1LgLeCzwMWZeW9EfAB4PDN/t9c6rVYr9+zZU3kfKysrLCwsDJVvUupk7HXDp/07L2swUXdNzeU4b1q1Y9tRblzt/kthrzma9E20+mWcFaNmbHKu+22rW85J/CzUMSsdFBFdC3yU/8Q8ABzIzHvbj28FLhxhe5KkGoYu8Mz8D+DrEXF+e9ErWD+dIkmagFF/F3wH8JH2K1C+Cvza6JEkSVWMVOCZeT9wzHkZSdL4eSWmJBXKApekQlngklQoC1ySCmWBS1KhLHBJKtRsXxMstU36knk1w6/beHkELkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSrUyAUeESdExL9GxB1NBJIkVdPEEfi1wL4GtiNJqmGkAo+Ic4HLgL9sJo4kqarIzOFXjrgV+D3gVOC6zHxNlzFLwBLA/Pz89uXl5crbX1tbY25ubuh8k1An4+rBI12Xb9uyudY+e22nn/lNcOjJamP75Rlm31XVyTgtZmxOt5x1fxbGbVY6aHFxcW9mHvP2lUPfzCoiXgMczsy9EbHQa1xm7gJ2AbRarVxY6Dn0GCsrK9QZPw11Ml7d48Y++6+stv6g7fSzY9tRblyt9uXul2eYfVdVJ+O0mLE53XLW/VkYt1nvoFFOoVwMvDYi9gPLwMsj4sONpJIkDTR0gWfmOzPz3MzcCrwB+MfMfGNjySRJffk6cEkqVCMnyjJzBVhpYluSpGo8ApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFmv3rbY8DW3tdYr/zsgknWdcrj1S6WftZG5VH4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCDV3gEXFeRNwVEfsi4qGIuLbJYJKk/ka5EvMosCMz74uIU4G9EXFnZn6hoWySpD5GeU/MRzPzvvbn3wH2AVuaCiZJ6i8yc/SNRGwF7gYuyMzHO55bApYA5ufnty8vL1fe7traGnNzcyPnG6c6GVcPHhlzmt7mN8GhJ6e2+0rM2IwSMkIzObdt2VxrfN2fwV4Z6+53VIuLi3szs9W5fOQCj4g54J+A92bmbf3Gtlqt3LNnT+Vtr6yssLCwMFK+cauTcZo3idqx7Sg3rs72vcvM2IwSMkIzOevehKruz2CvjJO++VVEdC3wkV6FEhHPAD4KfGRQeUuSmjXKq1AC+CCwLzPf31wkSVIVoxyBXwy8CXh5RNzf/ri0oVySpAGGPgGVmf8MRINZJEk1eCWmJBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFmv0bJrT1uodBk/dC6LWtfuvs2HaUqzuen/R9EqTjVVO90NR++xlHJo/AJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYUa9T0xL4mIL0XElyPi+qZCSZIGG+U9MU8A/hT4eeD5wBUR8fymgkmS+hvlCPyngC9n5lcz8ylgGbi8mViSpEEiM4dbMeL1wCWZ+Zb24zcBP52Zb+8YtwQstR+eD3ypxm7OAr45VMDJKSEjlJHTjM0oISOUkXNWMv5YZj6zc+EoN7Pq9obGx/xrkJm7gF1D7SBiT2a2hll3UkrICGXkNGMzSsgIZeSc9YyjnEI5AJy34fG5wDdGiyNJqmqUAv8c8NyIeHZEnAS8Abi9mViSpEGGPoWSmUcj4u3A3wMnALsz86HGkq0b6tTLhJWQEcrIacZmlJARysg50xmH/k9MSdJ0eSWmJBXKApekQk29wCPizIi4MyIeaf95Rpcx50XEXRGxLyIeiohr66w/iYztcbsj4nBEPNix/N0RcTAi7m9/XDqDGcc+jzVzdr1NwzjnctCtIWLdH7ef/3xEXFh13RnJuD8iVtvztmeKGX8iIu6JiO9GxHV11p2RjBOZx0oyc6ofwPuA69ufXw/8fpcx5wAXtj8/FXgYeH7V9SeRsf3cy4ALgQc7lr8buG7a8zgg49jnscbX+wTgK8BzgJOABzZ8vccyl/32uWHMpcCnWL8G4iLg3qrrTjtj+7n9wFlj/j6skvFs4CXAezd+LWdsHrtmnNQ8Vv2Y+hE465ff39z+/GbgdZ0DMvPRzLyv/fl3gH3AlqrrTyJjO9vdwGNj2H8Vo2acxDxW3c80btNQZZ+XA3+V6z4LnB4R50ww7ygZJ2Vgxsw8nJmfA75Xd90ZyDhTZqHA5zPzUVgvatb/5espIrYCPwncO8z6k8jYw9vbv9LuHtPpiVEzTmIeq+5nC/D1DY8P8P1/sGE8czlon/3GVFl32hlh/UrpT0fE3li/xcU4jDIXszSP/UxiHisZ5VL6yiLiH4BndXnqhprbmQM+CvxmZj7eRLYN224kYw9/BryH9S/8e4AbgTfX3ciYMzamgZz9btPQyFzW3OegMZVuK9GAUTICXJyZ34iIs4E7I+KL7d/ImjTKXMzSPPYziXmsZCIFnpmv7PVcRByKiHMy89H2r3qHe4x7Buvl/ZHMvG3DU5XWn0TGPts+tGFbfwHcMWsZaWgeG8rZ8zYNTc1lnX1WGHNShXWnnZHMfPrPwxHxMdZPJTRdPKPcYmNSt+cYaT8TmsdKZuEUyu3AVe3PrwI+3jkgIgL4ILAvM99fd/1JZOyn4xzkLwAP9ho7glHnYRLzWHU/PW/TMMa5rHJriNuBX22/0uMi4Ej7NNCkbisxdMaIOCUiTgWIiFOAVzOe78NR5mKW5rGrCc5jNdP+X1TgR4DPAI+0/zyzvfxHgU+2P/9Z1n/F+Txwf/vj0n7rTzpj+/EtwKOs/8fHAeCa9vIPAavt/LcD58xgxrHPY82cl7L+aqOvADdsWD62uey2T+CtwFvbnwfrb2LylXaG1qC8Y5i/oTKy/oqLB9ofD00547Pa33uPA99uf37ajM1j14yTnMcqH15KL0mFmoVTKJKkIVjgklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVD/B/kl5fqSbOM/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## test this function distribution\n",
    "weights_glorot = my_glorot_initializer((300, 100), dtype=tf.float32)\n",
    "pd.DataFrame(weights_glorot[:,0]).hist(bins=50);  ## test one faeture ==> Normal Dist. ==> ok!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddb3644f",
   "metadata": {},
   "source": [
    "#### `L1 regularizer `"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f71a3de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## define custom function implemetns l1_regularizer, adding loss (alpha * abs(weights) + loss)\n",
    "## np.sum() => tf.reduce_sum()\n",
    "def my_l1_regularizer(weights):\n",
    "    ''' this Function tries to add loss to the Main Loss function => (alpha * weights)\n",
    "            let alpha=0.01\n",
    "    '''\n",
    "    new_loss = tf.reduce_sum(tf.abs(weights) * 0.01)\n",
    "    return new_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52b8fe93",
   "metadata": {},
   "source": [
    "#### `Positive Constraint for weights `"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0408ba48",
   "metadata": {},
   "outputs": [],
   "source": [
    "## define custom function to make constraints for returning only postive weights\n",
    "def my_postive_weights(weights):\n",
    "    ''' this Function tries to return only postive weights\n",
    "    '''\n",
    "    postive_only_else_zero = tf.where(weights < 0, tf.zeros_like(weights), weights)\n",
    "    return postive_only_else_zero"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec391718",
   "metadata": {},
   "source": [
    "`Now you can build your Model, pass these Function to the Model, compiling, fitting and saving the Model, but also when you load the Model again, you should specify custom_objects dictionary`\n",
    "\n",
    "``` python\n",
    "        ## when building the Model\n",
    "        model = tf.keras.models.Sequential()\n",
    "        model.add(tf.keras.layers.Dense(30, activation=my_softplus,  ## custom activation\n",
    "                         kernel_initializer=my_glorot_initializer,   ## custom initializer\n",
    "                         kernel_regularizer=my_l1_regularizer,  ## custom regularizer\n",
    "                         kernel_constraint=my_postive_weights)) ## custom constraint\n",
    "        model.compile(____)\n",
    "        model.fit(____)\n",
    "        model.save('my_model_with_customs.h5')\n",
    "                   \n",
    "        ## when loading the Model\n",
    "        my_model = tf.keras.models.load_model('my_model_with_customs.h5',\n",
    "                              custom_objects={'soft_plus':my_softplus,\n",
    "                                              'my_glorot_initializer': my_glorot_initializer,\n",
    "                                              'my_l1_regularizer' : my_l1_regularizer,\n",
    "                                              'my_postive_weights': my_postive_weights})\n",
    "        \n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76e9f533",
   "metadata": {},
   "source": [
    "## Computing Gradients using autodiff\n",
    "`using reverse-mode autodiff`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7835ec5",
   "metadata": {},
   "source": [
    "$$f(x) = 3 * w_1^{2} + 2 * w_1 * w_2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d74ec125",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## define the function contains the equation\n",
    "def f(w1, w2):\n",
    "    return 3 * w1**2 + 2 * w1 * w2\n",
    "\n",
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "## when (w1, w2) = (5, 3)\n",
    "## partial_derivative regard to w1 ==> 36\n",
    "## partial_derivative regard to w1 ==> 10\n",
    "## Jacobian vector = [36, 10]\n",
    "\n",
    "## using GradientTape\n",
    "## persistent=True  ==> to be able to call it more times without raising errors\n",
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "jacob_vector = tape.gradient(z, [w1, w2])\n",
    "jacob_vector ### [36., 10] ===> it is True ==> magic!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8fac922",
   "metadata": {},
   "source": [
    "## Done !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de3ed690",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}