diff --git a/ESM_01_introduction_to_pytorch.ipynb b/ESM_01_introduction_to_pytorch.ipynb new file mode 100644 index 0000000..e96863b --- /dev/null +++ b/ESM_01_introduction_to_pytorch.ipynb @@ -0,0 +1,5884 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "jupytext": { + "cell_metadata_filter": "id,colab,colab_type,-all", + "formats": "ipynb,py:percent", + "main_language": "python" + }, + "papermill": { + "default_parameters": {}, + "duration": 21.925345, + "end_time": "2021-09-16T12:33:06.344225", + "environment_variables": {}, + "exception": null, + "input_path": "course_UvA-DL/01-introduction-to-pytorch/Introduction_to_PyTorch.ipynb", + "output_path": ".notebooks/course_UvA-DL/01-introduction-to-pytorch.ipynb", + "parameters": {}, + "start_time": "2021-09-16T12:32:44.418880", + "version": "2.3.3" + }, + "colab": { + "name": "ESM 01-introduction-to-pytorch.ipynb", + "provenance": [], + "include_colab_link": true + }, + "language_info": { + "name": "python" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.050411, + "end_time": "2021-09-16T12:32:45.750290", + "exception": false, + "start_time": "2021-09-16T12:32:45.699879", + "status": "completed" + }, + "tags": [], + "id": "eaced3d8" + }, + "source": [ + "\n", + "# Tutorial 1: Introduction to PyTorch\n", + "\n", + "* **Author:** Phillip Lippe\n", + "* **License:** CC BY-SA\n", + "* **Generated:** 2021-09-16T14:32:16.770882\n", + "\n", + "This tutorial will give a short introduction to PyTorch basics, and get you setup for writing your own neural networks.\n", + "This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.\n", + "The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.\n", + "\n", + "\n", + "---\n", + "Open in [![Open In Colab](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAHUAAAAUCAYAAACzrHJDAAAIuUlEQVRoQ+1ZaVRURxb+qhdolmbTUVSURpZgmLhHbQVFZIlGQBEXcMvJhKiTEzfigjQg7oNEJ9GMGidnjnNMBs2czIzajksEFRE1xklCTKJiQLRFsUGkoUWw+82pamn79etGYoKek1B/4NW99/tu3e/dquJBAGD27NkHALxKf39WY39gyrOi+i3xqGtUoePJrFmznrmgtModorbTu8YRNZk5cybXTvCtwh7o6NR2KzuZMWNGh6jtVt7nA0ymT5/eJlF9POrh7PAQl6s8bGYa3PUum//htmebVtLRqW0q01M5keTk5FZFzU0oRle3+zxwg5Hgtb+PZiL/ZVohxCI+hL5JgjmfjPxZ26+33BG3dA+ealHPM4gQAo5rU59gsI8bRvl54t3Ca62mvHyUAhtOlLd5WSQpKcluBjumnoCLs1EARkVd9E8l3p9y2i7RbQ1B6pFwu/YDgW8KbHJHMTQrwnjz2oZm9M4pavOCfo5jWrgCaaMVcMs6/pNhDr0+AMN93XlxV7R6DNpyzi7W/OE+yIrsjU6rTrbKV5cd/pNyItOmTbMp6sbBB+EqaYJY4cWE3VUciNt1TpgfcRFv71Fi54xT5kSoyLvOBEJMOMxWXkFlBeBSX4u6Zkcs+3KszYRtiapbNRqF31UgetVuc8z9vBXIv1qD+F1f83B6uDlCUyfsZGepGPpmg01OB7EITQbhS9ribKy+DmP1DUiClLz4bnIHVOqa7BY+Z1wg5g3zgUvyehiNpnJKxSLc/ts76LKm0BzX3c0RNy1yXjDcB5lWoro4iNHQxM+f1kWeWQARAWQS++trISJTp061Kep25X/MycwtjuctSC5rxo7ppi7VNUox5+PhPHtrsS2O1qJ6yx1QujQUzm9sh6hbkBlvvGcN8hYnwjUjH6kjfZEd5c/jitz5Jc5U3ENnFynKl4eB7nyEgP2UZ+Yz3/rVEbyYr27qELrtC4FIC0J7sc7xWnmccdHfRRTs0VB+cA4lt+oFcRR/wUeH8FG5w2Mbx8FQ8TXEvv1xYf4wBP3O2WyL3/UVjpXWgIqaFeUPr+wTmDvUB7njH6/bOv+HRg4SqioAg5GDe1aB3ZeMTJkyRSBqkLsWqSEm0fZVBEN94zEZnYvrdx1JL5cxe+a+AbhSJecRRHW/ikTFRTa38dtQlNZ5CRKwFvUtZU/kvBoEF9Uxni/XqIM+dwKbTw3rhcxIf7gmr2M+H6SMwx8iBzJbw5oxeG3Lv5FX9B3AGaHPS8e8z77H7v9VMpvPG5ug1enh7eGK8h0LBTwUb+GInqzInlRUK65DmTPQu4c3+uQKjwKK77zwUxBX4Tq7yR1RuiwUsqlrABCM6esHdXoy47fk4+prYKy8ZF574x4V5BnHQBuf4g9Z9ld8U36L2aktZNNplNfw7zotwWTy5MkCUft4aLEopJj5/OPHl1BQqeAVOnHgNSQOqmBzq9V9cfEm/yx5ubMGKS9cYPZ3vx2OS/c6PVHUuUO7Y1Pci3BO/1zgq18byebfGemLtNF+6JRtOvMk926ibussZqM+1mNz4TWkH7rCbM5phwGRGDAaoF8fY5OHFnlldAA8sgoEXKnDukA1NgSeNjqkJT9brbN4pC9WRweYXyLugR73c+MYvyWfu0yC6+mjzN1Isfw3FKJS98CU/zI1IHFkFPR52cHL2FJk0sB6kMTERIGo9GzcPkLNfA0cwdwi/hfEYO86ZMd9w+y1egfM2T2Eh/vesMNwljSzuZRT420SW3eqy8N6aHMmwmnFUZ7/PGVPbIoNZvNU1BURdHs0bT2+HjL8sDSM2e6vi4Lj5NW8WOLVA6RTT2azxLV+bglaFNqLieqemS/gWkw7NyoAHo+2dEsiivengjKsPFoqWOvbSh/kxPaxyW/JRzH2Fl3EzD9/xjAefJqB3usKUFn/0Gb+S/d/jy3FN2yLOmnSJJtn6oehByEiHPSeXnDxFGPRnoFoaBJjcdQlbDwcjL1zTNuQpoxD7R0OG0uUTMi0fkVwdzBdYIwcwZunxrVJVLplNm54BZp7jfDfYLoNyqQi1K6KxIdHzmN+QQ2WjFIwUT2zTGdlRXo4NFXVUO4sgX5dFC7f0aP/ZlNeUjFBuL8Xjl6uRuP6aMjSjpjzsH62FDU7JhBuGccEXIvDfJFFBc/gHw80dklfCVYnRaDfpiJcutPA4F7qJsfJeUPQI+1fqMlNhFx1FM0GDqkjFVg7NojlQ0Vt4aM5ReSqcbpaCg8nCW5lRsBvbT4T1TLfFptsfh7gItzuKTdJSEiwKSrt1vcmnEXXrsLbYnWDA1bu+z2WKy9Arq+1KRqdfKsoBo0GcdtEpS/B1bO4v0cFiUhkjskvKcMrWwtAPHuwQq8Z+4LZ1vTQANfXt4J0DwZX9gWa9qh4XDM/voC9JXfwYEMMHJcfNtusn82ihvliVUwg5KrPGVf6GH94ZJpEZBen6EC4qYTHA1dXhW0JIex8txzv//c8lhzXIi/BFxOH9jGbQhZsRalTIBZZ8KkGyZAxeRQvXkFF1TWz/Hm46jNYUnjPbt3JxIkT7f6dSj8qfJJyVvBxgaIlblOyjtysNHWN9fjjqWi7glJfW3/S0Hlj2XnA8PhKT9w6g3Qx3XiXhvuxQsuT1proxBKI/AaZqY1Xz5muvY8G8XkRRCaHsfQsRAFDH/tZPbcYuHotOG0FRIqB4HR3wNVoIPLtz8ycTguu+jpEigE218vd1YCr5m+HpHMvEI9u4LTXwNWaLjl0iPwGAmIpeHx1VeCqTJdPs1/vweweQPO3HC24NhOhnTphwoQnfv6QSY2ICbkNmdSA4h87oaLaiYfn5diIEd4att2erOwJXbPUHp953p6orQVSUVWRAXBT8c/dJ5L9xhzaJGp71GR/wFP8P5V2z10NSC9T93QM2xUg8fHxT+zU9ijeU4naHon8CjFJXFzc8/kn+dN06q9QgF98SYSo2Xen2NjYZy5sR6f+4nLSK5Iam2PH/x87a1YN/t5sBgAAAABJRU5ErkJggg==){height=\"20px\" width=\"117px\"}](https://colab.research.google.com/github/PytorchLightning/lightning-tutorials/blob/publication/.notebooks/course_UvA-DL/01-introduction-to-pytorch.ipynb)\n", + "\n", + "Give us a ⭐ [on Github](https://www.github.com/PytorchLightning/pytorch-lightning/)\n", + "| Check out [the documentation](https://pytorch-lightning.readthedocs.io/en/latest/)\n", + "| Join us [on Slack](https://join.slack.com/t/pytorch-lightning/shared_invite/zt-pw5v393p-qRaDgEk24~EjiZNBpSQFgQ)" + ], + "id": "eaced3d8" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.042559, + "end_time": "2021-09-16T12:32:45.835806", + "exception": false, + "start_time": "2021-09-16T12:32:45.793247", + "status": "completed" + }, + "tags": [], + "id": "fa480fc6" + }, + "source": [ + "## Setup\n", + "This notebook requires some packages besides pytorch-lightning." + ], + "id": "fa480fc6" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.043036, + "end_time": "2021-09-16T12:32:46.013859", + "exception": false, + "start_time": "2021-09-16T12:32:45.970823", + "status": "completed" + }, + "tags": [], + "id": "9473f942" + }, + "source": [ + "
\n", + "Welcome to our PyTorch tutorial for the Deep Learning course 2020 at the University of Amsterdam!\n", + "The following notebook is meant to give a short introduction to PyTorch basics, and get you setup for writing your own neural networks.\n", + "PyTorch is an open source machine learning framework that allows you to write your own neural networks and optimize them efficiently.\n", + "However, PyTorch is not the only framework of its kind.\n", + "Alternatives to PyTorch include [TensorFlow](https://www.tensorflow.org/), [JAX](https://github.com/google/jax#quickstart-colab-in-the-cloud) and [Caffe](http://caffe.berkeleyvision.org/).\n", + "We choose to teach PyTorch at the University of Amsterdam because it is well established, has a huge developer community (originally developed by Facebook), is very flexible and especially used in research.\n", + "Many current papers publish their code in PyTorch, and thus it is good to be familiar with PyTorch as well.\n", + "Meanwhile, TensorFlow (developed by Google) is usually known for being a production-grade deep learning library.\n", + "Still, if you know one machine learning framework in depth, it is very easy to learn another one because many of them use the same concepts and ideas.\n", + "For instance, TensorFlow's version 2 was heavily inspired by the most popular features of PyTorch, making the frameworks even more similar.\n", + "If you are already familiar with PyTorch and have created your own neural network projects, feel free to just skim this notebook.\n", + "\n", + "We are of course not the first ones to create a PyTorch tutorial.\n", + "There are many great tutorials online, including the [\"60-min blitz\"](https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html) on the official [PyTorch website](https://pytorch.org/tutorials/).\n", + "Yet, we choose to create our own tutorial which is designed to give you the basics particularly necessary for the practicals, but still understand how PyTorch works under the hood.\n", + "Over the next few weeks, we will also keep exploring new PyTorch features in the series of Jupyter notebook tutorials about deep learning.\n", + "\n", + "We will use a set of standard libraries that are often used in machine learning projects.\n", + "If you are running this notebook on Google Colab, all libraries should be pre-installed.\n", + "If you are running this notebook locally, make sure you have installed our `dl2020` environment ([link](https://github.com/uvadlc/uvadlc_practicals_2020/blob/master/environment.yml)) and have activated it." + ], + "id": "9473f942" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:45.924885Z", + "iopub.status.busy": "2021-09-16T12:32:45.924409Z", + "iopub.status.idle": "2021-09-16T12:32:45.927196Z", + "shell.execute_reply": "2021-09-16T12:32:45.926697Z" + }, + "id": "a1f58dc1", + "lines_to_next_cell": 0, + "papermill": { + "duration": 0.048784, + "end_time": "2021-09-16T12:32:45.927310", + "exception": false, + "start_time": "2021-09-16T12:32:45.878526", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# ! pip install --quiet \"torchmetrics>=0.3\" \"matplotlib\" \"torch>=1.6, <1.9\" \"pytorch-lightning>=1.3\"" + ], + "id": "a1f58dc1", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:46.106552Z", + "iopub.status.busy": "2021-09-16T12:32:46.106081Z", + "iopub.status.idle": "2021-09-16T12:32:46.889364Z", + "shell.execute_reply": "2021-09-16T12:32:46.889833Z" + }, + "papermill": { + "duration": 0.833305, + "end_time": "2021-09-16T12:32:46.889977", + "exception": false, + "start_time": "2021-09-16T12:32:46.056672", + "status": "completed" + }, + "tags": [], + "id": "edcfda76" + }, + "source": [ + "import time\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.utils.data as data\n", + "\n", + "# %matplotlib inline\n", + "from IPython.display import set_matplotlib_formats\n", + "from matplotlib.colors import to_rgba\n", + "from tqdm.notebook import tqdm # Progress bar\n", + "\n", + "#EM HAD TO UPDATE and import the following 2 new lines to handle the deprecated \"set_matplotlib_inline\" by importing new packages (line 16) and attributes (line 17), showm immediately below, and THEN running it properly on the next line \n", + "\n", + "import matplotlib_inline\n", + "import matplotlib_inline.backend_inline\n", + "\n", + "matplotlib_inline.backend_inline.set_matplotlib_formats(\"svg\",\"pdf\")\n", + "#EM DEPRECATED: set_matplotlib_formats(\"svg\", \"pdf\")" + ], + "id": "edcfda76", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.042985, + "end_time": "2021-09-16T12:32:46.977014", + "exception": false, + "start_time": "2021-09-16T12:32:46.934029", + "status": "completed" + }, + "tags": [], + "id": "d9625e8f" + }, + "source": [ + "## The Basics of PyTorch\n", + "\n", + "We will start with reviewing the very basic concepts of PyTorch.\n", + "As a prerequisite, we recommend to be familiar with the `numpy` package as most machine learning frameworks are based on very similar concepts.\n", + "If you are not familiar with numpy yet, don't worry: here is a [tutorial](https://numpy.org/devdocs/user/quickstart.html) to go through.\n", + "\n", + "So, let's start with importing PyTorch.\n", + "The package is called `torch`, based on its original framework [Torch](http://torch.ch/).\n", + "As a first step, we can check its version:" + ], + "id": "d9625e8f" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:47.065771Z", + "iopub.status.busy": "2021-09-16T12:32:47.065287Z", + "iopub.status.idle": "2021-09-16T12:32:47.067941Z", + "shell.execute_reply": "2021-09-16T12:32:47.067546Z" + }, + "papermill": { + "duration": 0.048411, + "end_time": "2021-09-16T12:32:47.068040", + "exception": false, + "start_time": "2021-09-16T12:32:47.019629", + "status": "completed" + }, + "tags": [], + "id": "eb6179df", + "outputId": "b5feae1c-ec97-4692-81f8-2421ea095835" + }, + "source": [ + "print(\"Using torch\", torch.__version__)" + ], + "id": "eb6179df", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using torch 1.8.1+cu102\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.04343, + "end_time": "2021-09-16T12:32:47.154839", + "exception": false, + "start_time": "2021-09-16T12:32:47.111409", + "status": "completed" + }, + "tags": [], + "id": "8a8171cd" + }, + "source": [ + "At the time of writing this tutorial (mid of August 2021), the current stable version is 1.9.\n", + "You should therefore see the output `Using torch 1.9.0`, eventually with some extension for the CUDA version on Colab.\n", + "In case you use the `dl2020` environment, you should see `Using torch 1.6.0` since the environment was provided in October 2020.\n", + "It is recommended to update the PyTorch version to the newest one.\n", + "If you see a lower version number than 1.6, make sure you have installed the correct the environment, or ask one of your TAs.\n", + "In case PyTorch 1.10 or newer will be published during the time of the course, don't worry.\n", + "The interface between PyTorch versions doesn't change too much, and hence all code should also be runnable with newer versions.\n", + "\n", + "As in every machine learning framework, PyTorch provides functions that are stochastic like generating random numbers.\n", + "However, a very good practice is to setup your code to be reproducible with the exact same random numbers.\n", + "This is why we set a seed below." + ], + "id": "8a8171cd" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:47.245406Z", + "iopub.status.busy": "2021-09-16T12:32:47.244938Z", + "iopub.status.idle": "2021-09-16T12:32:47.249667Z", + "shell.execute_reply": "2021-09-16T12:32:47.250066Z" + }, + "papermill": { + "duration": 0.050609, + "end_time": "2021-09-16T12:32:47.250178", + "exception": false, + "start_time": "2021-09-16T12:32:47.199569", + "status": "completed" + }, + "tags": [], + "id": "5d3e8fcb", + "outputId": "207fbd40-2db7-4673-9090-9531f80e1042" + }, + "source": [ + "torch.manual_seed(42) # Setting the seed" + ], + "id": "5d3e8fcb", + "execution_count": null, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.043829, + "end_time": "2021-09-16T12:32:47.337473", + "exception": false, + "start_time": "2021-09-16T12:32:47.293644", + "status": "completed" + }, + "tags": [], + "id": "6f28366d" + }, + "source": [ + "### Tensors\n", + "\n", + "Tensors are the PyTorch equivalent to Numpy arrays, with the addition to also have support for GPU acceleration (more on that later).\n", + "The name \"tensor\" is a generalization of concepts you already know.\n", + "For instance, a vector is a 1-D tensor, and a matrix a 2-D tensor.\n", + "When working with neural networks, we will use tensors of various shapes and number of dimensions.\n", + "\n", + "Most common functions you know from numpy can be used on tensors as well.\n", + "Actually, since numpy arrays are so similar to tensors, we can convert most tensors to numpy arrays (and back) but we don't need it too often.\n", + "\n", + "#### Initialization\n", + "\n", + "Let's first start by looking at different ways of creating a tensor.\n", + "There are many possible options, the most simple one is to call\n", + "`torch.Tensor` passing the desired shape as input argument:" + ], + "id": "6f28366d" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:47.428074Z", + "iopub.status.busy": "2021-09-16T12:32:47.427607Z", + "iopub.status.idle": "2021-09-16T12:32:47.431411Z", + "shell.execute_reply": "2021-09-16T12:32:47.430883Z" + }, + "papermill": { + "duration": 0.050551, + "end_time": "2021-09-16T12:32:47.431515", + "exception": false, + "start_time": "2021-09-16T12:32:47.380964", + "status": "completed" + }, + "tags": [], + "id": "7d6bab63", + "outputId": "c758273b-e053-4d33-b4f1-8972fe8cd99f" + }, + "source": [ + "x = torch.Tensor(2, 3, 4)\n", + "print(x)" + ], + "id": "7d6bab63", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[7.3697e+28, 2.7869e+29, 4.3059e+21, 6.9768e+22],\n", + " [6.8612e+22, 4.6114e+24, 3.0186e+32, 4.5434e+30],\n", + " [1.9519e-19, 7.4934e+28, 8.9068e-15, 5.6284e-14]],\n", + "\n", + " [[2.0618e-19, 1.0901e+27, 2.0532e-19, 1.7440e+28],\n", + " [1.2997e+34, 6.8608e+22, 4.7473e+27, 2.0532e-19],\n", + " [3.1771e+30, 7.2442e+22, 1.6931e+22, 1.1022e+24]]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.043882, + "end_time": "2021-09-16T12:32:47.519551", + "exception": false, + "start_time": "2021-09-16T12:32:47.475669", + "status": "completed" + }, + "tags": [], + "id": "2b19df67" + }, + "source": [ + "The function `torch.Tensor` allocates memory for the desired tensor, but reuses any values that have already been in the memory.\n", + "To directly assign values to the tensor during initialization, there are many alternatives including:\n", + "\n", + "* `torch.zeros`: Creates a tensor filled with zeros\n", + "* `torch.ones`: Creates a tensor filled with ones\n", + "* `torch.rand`: Creates a tensor with random values uniformly sampled between 0 and 1\n", + "* `torch.randn`: Creates a tensor with random values sampled from a normal distribution with mean 0 and variance 1\n", + "* `torch.arange`: Creates a tensor containing the values $N,N+1,N+2,...,M$\n", + "* `torch.Tensor` (input list): Creates a tensor from the list elements you provide" + ], + "id": "2b19df67" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:47.611131Z", + "iopub.status.busy": "2021-09-16T12:32:47.610668Z", + "iopub.status.idle": "2021-09-16T12:32:47.623382Z", + "shell.execute_reply": "2021-09-16T12:32:47.622915Z" + }, + "papermill": { + "duration": 0.060116, + "end_time": "2021-09-16T12:32:47.623485", + "exception": false, + "start_time": "2021-09-16T12:32:47.563369", + "status": "completed" + }, + "tags": [], + "id": "45fe1b7e", + "outputId": "f9c44389-af89-485f-9136-1993b41f98b4" + }, + "source": [ + "# Create a tensor from a (nested) list\n", + "x = torch.Tensor([[1, 2], [3, 4]])\n", + "print(x)" + ], + "id": "45fe1b7e", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[1., 2.],\n", + " [3., 4.]])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:47.717600Z", + "iopub.status.busy": "2021-09-16T12:32:47.717128Z", + "iopub.status.idle": "2021-09-16T12:32:47.720139Z", + "shell.execute_reply": "2021-09-16T12:32:47.719670Z" + }, + "papermill": { + "duration": 0.052738, + "end_time": "2021-09-16T12:32:47.720244", + "exception": false, + "start_time": "2021-09-16T12:32:47.667506", + "status": "completed" + }, + "tags": [], + "id": "76f8e1f5", + "outputId": "5e507b04-8239-4761-ecdc-3209b6d26279" + }, + "source": [ + "# Create a tensor with random values between 0 and 1 with the shape [2, 3, 4]\n", + "x = torch.rand(2, 3, 4)\n", + "print(x)" + ], + "id": "76f8e1f5", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[[0.8823, 0.9150, 0.3829, 0.9593],\n", + " [0.3904, 0.6009, 0.2566, 0.7936],\n", + " [0.9408, 0.1332, 0.9346, 0.5936]],\n", + "\n", + " [[0.8694, 0.5677, 0.7411, 0.4294],\n", + " [0.8854, 0.5739, 0.2666, 0.6274],\n", + " [0.2696, 0.4414, 0.2969, 0.8317]]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.044337, + "end_time": "2021-09-16T12:32:47.809374", + "exception": false, + "start_time": "2021-09-16T12:32:47.765037", + "status": "completed" + }, + "tags": [], + "id": "f2c84d7c" + }, + "source": [ + "You can obtain the shape of a tensor in the same way as in numpy (`x.shape`), or using the `.size` method:" + ], + "id": "f2c84d7c" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:47.902716Z", + "iopub.status.busy": "2021-09-16T12:32:47.900874Z", + "iopub.status.idle": "2021-09-16T12:32:47.906006Z", + "shell.execute_reply": "2021-09-16T12:32:47.905588Z" + }, + "papermill": { + "duration": 0.05197, + "end_time": "2021-09-16T12:32:47.906110", + "exception": false, + "start_time": "2021-09-16T12:32:47.854140", + "status": "completed" + }, + "tags": [], + "id": "b9738fb0", + "outputId": "eccc9bc4-660a-40bd-d539-1757d4caef32" + }, + "source": [ + "shape = x.shape\n", + "print(\"Shape:\", x.shape)\n", + "\n", + "size = x.size()\n", + "print(\"Size:\", size)\n", + "\n", + "dim1, dim2, dim3 = x.size()\n", + "print(\"Size:\", dim1, dim2, dim3)" + ], + "id": "b9738fb0", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Shape: torch.Size([2, 3, 4])\n", + "Size: torch.Size([2, 3, 4])\n", + "Size: 2 3 4\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.045873, + "end_time": "2021-09-16T12:32:47.996974", + "exception": false, + "start_time": "2021-09-16T12:32:47.951101", + "status": "completed" + }, + "tags": [], + "id": "0e9401d0" + }, + "source": [ + "#### Tensor to Numpy, and Numpy to Tensor\n", + "\n", + "Tensors can be converted to numpy arrays, and numpy arrays back to tensors.\n", + "To transform a numpy array into a tensor, we can use the function `torch.from_numpy`:" + ], + "id": "0e9401d0" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:48.091606Z", + "iopub.status.busy": "2021-09-16T12:32:48.091139Z", + "iopub.status.idle": "2021-09-16T12:32:48.093695Z", + "shell.execute_reply": "2021-09-16T12:32:48.094094Z" + }, + "papermill": { + "duration": 0.052501, + "end_time": "2021-09-16T12:32:48.094216", + "exception": false, + "start_time": "2021-09-16T12:32:48.041715", + "status": "completed" + }, + "tags": [], + "id": "e0670ce6", + "outputId": "02c97d56-0c83-460a-83bd-9f992491095d" + }, + "source": [ + "np_arr = np.array([[1, 2], [3, 4]])\n", + "tensor = torch.from_numpy(np_arr)\n", + "\n", + "print(\"Numpy array:\", np_arr)\n", + "print(\"PyTorch tensor:\", tensor)" + ], + "id": "e0670ce6", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Numpy array: [[1 2]\n", + " [3 4]]\n", + "PyTorch tensor: tensor([[1, 2],\n", + " [3, 4]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.045581, + "end_time": "2021-09-16T12:32:49.246779", + "exception": false, + "start_time": "2021-09-16T12:32:49.201198", + "status": "completed" + }, + "tags": [], + "id": "98f64f88" + }, + "source": [ + "To transform a PyTorch tensor back to a numpy array, we can use the function `.numpy()` on tensors:" + ], + "id": "98f64f88" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:49.340766Z", + "iopub.status.busy": "2021-09-16T12:32:49.340292Z", + "iopub.status.idle": "2021-09-16T12:32:49.343538Z", + "shell.execute_reply": "2021-09-16T12:32:49.343139Z" + }, + "papermill": { + "duration": 0.05169, + "end_time": "2021-09-16T12:32:49.343640", + "exception": false, + "start_time": "2021-09-16T12:32:49.291950", + "status": "completed" + }, + "tags": [], + "id": "fb2c4b46", + "outputId": "f62d0660-5b03-4785-d05e-97500270a22d" + }, + "source": [ + "tensor = torch.arange(4)\n", + "np_arr = tensor.numpy()\n", + "\n", + "print(\"PyTorch tensor:\", tensor)\n", + "print(\"Numpy array:\", np_arr)" + ], + "id": "fb2c4b46", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PyTorch tensor: tensor([0, 1, 2, 3])\n", + "Numpy array: [0 1 2 3]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051428, + "end_time": "2021-09-16T12:32:49.440442", + "exception": false, + "start_time": "2021-09-16T12:32:49.389014", + "status": "completed" + }, + "tags": [], + "id": "d31dc267" + }, + "source": [ + "The conversion of tensors to numpy require the tensor to be on the CPU, and not the GPU (more on GPU support in a later section).\n", + "In case you have a tensor on GPU, you need to call `.cpu()` on the tensor beforehand.\n", + "Hence, you get a line like `np_arr = tensor.cpu().numpy()`." + ], + "id": "d31dc267" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.045401, + "end_time": "2021-09-16T12:32:49.530975", + "exception": false, + "start_time": "2021-09-16T12:32:49.485574", + "status": "completed" + }, + "tags": [], + "id": "46a6ef01" + }, + "source": [ + "#### Operations\n", + "\n", + "Most operations that exist in numpy, also exist in PyTorch.\n", + "A full list of operations can be found in the [PyTorch documentation](https://pytorch.org/docs/stable/tensors.html#), but we will review the most important ones here.\n", + "\n", + "The simplest operation is to add two tensors:" + ], + "id": "46a6ef01" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:49.625900Z", + "iopub.status.busy": "2021-09-16T12:32:49.625408Z", + "iopub.status.idle": "2021-09-16T12:32:49.629396Z", + "shell.execute_reply": "2021-09-16T12:32:49.629792Z" + }, + "papermill": { + "duration": 0.053783, + "end_time": "2021-09-16T12:32:49.629915", + "exception": false, + "start_time": "2021-09-16T12:32:49.576132", + "status": "completed" + }, + "tags": [], + "id": "13f957c9", + "outputId": "503387cb-4c45-4a9b-ea9e-212268a017ff" + }, + "source": [ + "x1 = torch.rand(2, 3)\n", + "x2 = torch.rand(2, 3)\n", + "y = x1 + x2\n", + "\n", + "print(\"X1\", x1)\n", + "print(\"X2\", x2)\n", + "print(\"Y\", y)" + ], + "id": "13f957c9", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X1 tensor([[0.1053, 0.2695, 0.3588],\n", + " [0.1994, 0.5472, 0.0062]])\n", + "X2 tensor([[0.9516, 0.0753, 0.8860],\n", + " [0.5832, 0.3376, 0.8090]])\n", + "Y tensor([[1.0569, 0.3448, 1.2448],\n", + " [0.7826, 0.8848, 0.8151]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.047584, + "end_time": "2021-09-16T12:32:49.724517", + "exception": false, + "start_time": "2021-09-16T12:32:49.676933", + "status": "completed" + }, + "tags": [], + "id": "4fbd2538" + }, + "source": [ + "Calling `x1 + x2` creates a new tensor containing the sum of the two inputs.\n", + "However, we can also use in-place operations that are applied directly on the memory of a tensor.\n", + "We therefore change the values of `x2` without the chance to re-accessing the values of `x2` before the operation.\n", + "An example is shown below:" + ], + "id": "4fbd2538" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:49.823070Z", + "iopub.status.busy": "2021-09-16T12:32:49.822399Z", + "iopub.status.idle": "2021-09-16T12:32:49.828109Z", + "shell.execute_reply": "2021-09-16T12:32:49.827637Z" + }, + "papermill": { + "duration": 0.055272, + "end_time": "2021-09-16T12:32:49.828214", + "exception": false, + "start_time": "2021-09-16T12:32:49.772942", + "status": "completed" + }, + "tags": [], + "id": "0e6a7497", + "outputId": "d0aab7ca-4ac1-42c9-94d7-214c4d52d30a" + }, + "source": [ + "x1 = torch.rand(2, 3)\n", + "x2 = torch.rand(2, 3)\n", + "print(\"X1 (before)\", x1)\n", + "print(\"X2 (before)\", x2)\n", + "\n", + "x2.add_(x1)\n", + "print(\"X1 (after)\", x1)\n", + "print(\"X2 (after)\", x2)" + ], + "id": "0e6a7497", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X1 (before) tensor([[0.5779, 0.9040, 0.5547],\n", + " [0.3423, 0.6343, 0.3644]])\n", + "X2 (before) tensor([[0.7104, 0.9464, 0.7890],\n", + " [0.2814, 0.7886, 0.5895]])\n", + "X1 (after) tensor([[0.5779, 0.9040, 0.5547],\n", + " [0.3423, 0.6343, 0.3644]])\n", + "X2 (after) tensor([[1.2884, 1.8504, 1.3437],\n", + " [0.6237, 1.4230, 0.9539]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.046964, + "end_time": "2021-09-16T12:32:49.921617", + "exception": false, + "start_time": "2021-09-16T12:32:49.874653", + "status": "completed" + }, + "tags": [], + "id": "5cbaa92f" + }, + "source": [ + "In-place operations are usually marked with a underscore postfix (e.g. \"add_\" instead of \"add\").\n", + "\n", + "Another common operation aims at changing the shape of a tensor.\n", + "A tensor of size (2,3) can be re-organized to any other shape with the same number of elements (e.g. a tensor of size (6), or (3,2), ...).\n", + "In PyTorch, this operation is called `view`:" + ], + "id": "5cbaa92f" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.026878Z", + "iopub.status.busy": "2021-09-16T12:32:50.026353Z", + "iopub.status.idle": "2021-09-16T12:32:50.029094Z", + "shell.execute_reply": "2021-09-16T12:32:50.028625Z" + }, + "papermill": { + "duration": 0.06096, + "end_time": "2021-09-16T12:32:50.029199", + "exception": false, + "start_time": "2021-09-16T12:32:49.968239", + "status": "completed" + }, + "tags": [], + "id": "6907851d", + "outputId": "06cd570d-7f0b-478f-c72a-6cf1eb1f4bc2" + }, + "source": [ + "x = torch.arange(6)\n", + "print(\"X\", x)" + ], + "id": "6907851d", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([0, 1, 2, 3, 4, 5])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.136129Z", + "iopub.status.busy": "2021-09-16T12:32:50.135661Z", + "iopub.status.idle": "2021-09-16T12:32:50.138426Z", + "shell.execute_reply": "2021-09-16T12:32:50.137963Z" + }, + "papermill": { + "duration": 0.054742, + "end_time": "2021-09-16T12:32:50.138528", + "exception": false, + "start_time": "2021-09-16T12:32:50.083786", + "status": "completed" + }, + "tags": [], + "id": "252fa33f", + "outputId": "223dccac-9220-47cb-88f2-0f5fd135618a" + }, + "source": [ + "x = x.view(2, 3)\n", + "print(\"X\", x)" + ], + "id": "252fa33f", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([[0, 1, 2],\n", + " [3, 4, 5]])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.239003Z", + "iopub.status.busy": "2021-09-16T12:32:50.238537Z", + "iopub.status.idle": "2021-09-16T12:32:50.241342Z", + "shell.execute_reply": "2021-09-16T12:32:50.240878Z" + }, + "papermill": { + "duration": 0.053431, + "end_time": "2021-09-16T12:32:50.241439", + "exception": false, + "start_time": "2021-09-16T12:32:50.188008", + "status": "completed" + }, + "tags": [], + "id": "72e32ecb", + "outputId": "e9acaf48-2821-4d8e-b89b-2ae76d7f0d2e" + }, + "source": [ + "x = x.permute(1, 0) # Swapping dimension 0 and 1\n", + "print(\"X\", x)" + ], + "id": "72e32ecb", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([[0, 3],\n", + " [1, 4],\n", + " [2, 5]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.04706, + "end_time": "2021-09-16T12:32:50.335409", + "exception": false, + "start_time": "2021-09-16T12:32:50.288349", + "status": "completed" + }, + "tags": [], + "id": "cde67e9c" + }, + "source": [ + "Other commonly used operations include matrix multiplications, which are essential for neural networks.\n", + "Quite often, we have an input vector $\\mathbf{x}$, which is transformed using a learned weight matrix $\\mathbf{W}$.\n", + "There are multiple ways and functions to perform matrix multiplication, some of which we list below:\n", + "\n", + "* `torch.matmul`: Performs the matrix product over two tensors, where the specific behavior depends on the dimensions.\n", + "If both inputs are matrices (2-dimensional tensors), it performs the standard matrix product.\n", + "For higher dimensional inputs, the function supports broadcasting (for details see the [documentation](https://pytorch.org/docs/stable/generated/torch.matmul.html?highlight=matmul#torch.matmul)).\n", + "Can also be written as `a @ b`, similar to numpy.\n", + "* `torch.mm`: Performs the matrix product over two matrices, but doesn't support broadcasting (see [documentation](https://pytorch.org/docs/stable/generated/torch.mm.html?highlight=torch%20mm#torch.mm))\n", + "* `torch.bmm`: Performs the matrix product with a support batch dimension.\n", + "If the first tensor $T$ is of shape ($b\\times n\\times m$), and the second tensor $R$ ($b\\times m\\times p$), the output $O$ is of shape ($b\\times n\\times p$), and has been calculated by performing $b$ matrix multiplications of the submatrices of $T$ and $R$: $O_i = T_i @ R_i$\n", + "* `torch.einsum`: Performs matrix multiplications and more (i.e. sums of products) using the Einstein summation convention.\n", + "Explanation of the Einstein sum can be found in assignment 1.\n", + "\n", + "Usually, we use `torch.matmul` or `torch.bmm`. We can try a matrix multiplication with `torch.matmul` below." + ], + "id": "cde67e9c" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.432080Z", + "iopub.status.busy": "2021-09-16T12:32:50.431615Z", + "iopub.status.idle": "2021-09-16T12:32:50.434705Z", + "shell.execute_reply": "2021-09-16T12:32:50.434244Z" + }, + "papermill": { + "duration": 0.052861, + "end_time": "2021-09-16T12:32:50.434804", + "exception": false, + "start_time": "2021-09-16T12:32:50.381943", + "status": "completed" + }, + "tags": [], + "id": "ff386c27", + "outputId": "5fc6dccf-55bd-4ed8-b06d-17f0567da5ac" + }, + "source": [ + "x = torch.arange(6)\n", + "x = x.view(2, 3)\n", + "print(\"X\", x)" + ], + "id": "ff386c27", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([[0, 1, 2],\n", + " [3, 4, 5]])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.533427Z", + "iopub.status.busy": "2021-09-16T12:32:50.532966Z", + "iopub.status.idle": "2021-09-16T12:32:50.535803Z", + "shell.execute_reply": "2021-09-16T12:32:50.535338Z" + }, + "papermill": { + "duration": 0.054221, + "end_time": "2021-09-16T12:32:50.535901", + "exception": false, + "start_time": "2021-09-16T12:32:50.481680", + "status": "completed" + }, + "tags": [], + "id": "8c1795af", + "outputId": "0e937cff-487b-467b-890e-8b91cccf4913" + }, + "source": [ + "W = torch.arange(9).view(3, 3) # We can also stack multiple operations in a single line\n", + "print(\"W\", W)" + ], + "id": "8c1795af", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "W tensor([[0, 1, 2],\n", + " [3, 4, 5],\n", + " [6, 7, 8]])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.633621Z", + "iopub.status.busy": "2021-09-16T12:32:50.633158Z", + "iopub.status.idle": "2021-09-16T12:32:50.635999Z", + "shell.execute_reply": "2021-09-16T12:32:50.635506Z" + }, + "papermill": { + "duration": 0.052906, + "end_time": "2021-09-16T12:32:50.636097", + "exception": false, + "start_time": "2021-09-16T12:32:50.583191", + "status": "completed" + }, + "tags": [], + "id": "4dddd17e", + "outputId": "e0123356-0b94-487b-b600-d02f396d5075" + }, + "source": [ + "h = torch.matmul(x, W) # Verify the result by calculating it by hand too!\n", + "print(\"h\", h)" + ], + "id": "4dddd17e", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "h tensor([[15, 18, 21],\n", + " [42, 54, 66]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.048142, + "end_time": "2021-09-16T12:32:50.732093", + "exception": false, + "start_time": "2021-09-16T12:32:50.683951", + "status": "completed" + }, + "tags": [], + "id": "72d026f9" + }, + "source": [ + "#### Indexing\n", + "\n", + "We often have the situation where we need to select a part of a tensor.\n", + "Indexing works just like in numpy, so let's try it:" + ], + "id": "72d026f9" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.831818Z", + "iopub.status.busy": "2021-09-16T12:32:50.831358Z", + "iopub.status.idle": "2021-09-16T12:32:50.834223Z", + "shell.execute_reply": "2021-09-16T12:32:50.833827Z" + }, + "papermill": { + "duration": 0.054078, + "end_time": "2021-09-16T12:32:50.834321", + "exception": false, + "start_time": "2021-09-16T12:32:50.780243", + "status": "completed" + }, + "tags": [], + "id": "b44382eb", + "outputId": "aed72501-e9e9-4996-dab0-66f2211f390c" + }, + "source": [ + "x = torch.arange(12).view(3, 4)\n", + "print(\"X\", x)" + ], + "id": "b44382eb", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([[ 0, 1, 2, 3],\n", + " [ 4, 5, 6, 7],\n", + " [ 8, 9, 10, 11]])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:50.933203Z", + "iopub.status.busy": "2021-09-16T12:32:50.932738Z", + "iopub.status.idle": "2021-09-16T12:32:50.935142Z", + "shell.execute_reply": "2021-09-16T12:32:50.934747Z" + }, + "papermill": { + "duration": 0.05308, + "end_time": "2021-09-16T12:32:50.935240", + "exception": false, + "start_time": "2021-09-16T12:32:50.882160", + "status": "completed" + }, + "tags": [], + "id": "e797f3da", + "outputId": "c53f9120-9e75-4b30-907a-206679e82791" + }, + "source": [ + "print(x[:, 1]) # Second column" + ], + "id": "e797f3da", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([1, 5, 9])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:51.035597Z", + "iopub.status.busy": "2021-09-16T12:32:51.035133Z", + "iopub.status.idle": "2021-09-16T12:32:51.037860Z", + "shell.execute_reply": "2021-09-16T12:32:51.037378Z" + }, + "papermill": { + "duration": 0.053815, + "end_time": "2021-09-16T12:32:51.037961", + "exception": false, + "start_time": "2021-09-16T12:32:50.984146", + "status": "completed" + }, + "tags": [], + "id": "832fa534", + "outputId": "578ac154-c19e-4acd-9e5d-8a2d2dd5b8a2" + }, + "source": [ + "print(x[0]) # First row" + ], + "id": "832fa534", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([0, 1, 2, 3])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:51.138719Z", + "iopub.status.busy": "2021-09-16T12:32:51.138254Z", + "iopub.status.idle": "2021-09-16T12:32:51.140664Z", + "shell.execute_reply": "2021-09-16T12:32:51.140201Z" + }, + "papermill": { + "duration": 0.053829, + "end_time": "2021-09-16T12:32:51.140762", + "exception": false, + "start_time": "2021-09-16T12:32:51.086933", + "status": "completed" + }, + "tags": [], + "id": "554196e9", + "outputId": "10730a61-9499-4cb4-966b-7beb994a547d" + }, + "source": [ + "print(x[:2, -1]) # First two rows, last column" + ], + "id": "554196e9", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([3, 7])\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:51.242836Z", + "iopub.status.busy": "2021-09-16T12:32:51.242376Z", + "iopub.status.idle": "2021-09-16T12:32:51.245113Z", + "shell.execute_reply": "2021-09-16T12:32:51.244657Z" + }, + "papermill": { + "duration": 0.054275, + "end_time": "2021-09-16T12:32:51.245210", + "exception": false, + "start_time": "2021-09-16T12:32:51.190935", + "status": "completed" + }, + "tags": [], + "id": "2efaee3a", + "outputId": "4589b195-ef4d-4fb1-a51a-64c374e68484" + }, + "source": [ + "print(x[1:3, :]) # Middle two rows" + ], + "id": "2efaee3a", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 4, 5, 6, 7],\n", + " [ 8, 9, 10, 11]])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.049087, + "end_time": "2021-09-16T12:32:51.343540", + "exception": false, + "start_time": "2021-09-16T12:32:51.294453", + "status": "completed" + }, + "tags": [], + "id": "e1a9591f" + }, + "source": [ + "### Dynamic Computation Graph and Backpropagation\n", + "\n", + "One of the main reasons for using PyTorch in Deep Learning projects is that we can automatically get **gradients/derivatives** of functions that we define.\n", + "We will mainly use PyTorch for implementing neural networks, and they are just fancy functions.\n", + "If we use weight matrices in our function that we want to learn, then those are called the **parameters** or simply the **weights**.\n", + "\n", + "If our neural network would output a single scalar value, we would talk about taking the **derivative**, but you will see that quite often we will have **multiple** output variables (\"values\"); in that case we talk about **gradients**.\n", + "It's a more general term.\n", + "\n", + "Given an input $\\mathbf{x}$, we define our function by **manipulating** that input, usually by matrix-multiplications with weight matrices and additions with so-called bias vectors.\n", + "As we manipulate our input, we are automatically creating a **computational graph**.\n", + "This graph shows how to arrive at our output from our input.\n", + "PyTorch is a **define-by-run** framework; this means that we can just do our manipulations, and PyTorch will keep track of that graph for us.\n", + "Thus, we create a dynamic computation graph along the way.\n", + "\n", + "So, to recap: the only thing we have to do is to compute the **output**, and then we can ask PyTorch to automatically get the **gradients**.\n", + "\n", + "> **Note: Why do we want gradients?\n", + "** Consider that we have defined a function, a neural net, that is supposed to compute a certain output $y$ for an input vector $\\mathbf{x}$.\n", + "We then define an **error measure** that tells us how wrong our network is; how bad it is in predicting output $y$ from input $\\mathbf{x}$.\n", + "Based on this error measure, we can use the gradients to **update** the weights $\\mathbf{W}$ that were responsible for the output, so that the next time we present input $\\mathbf{x}$ to our network, the output will be closer to what we want.\n", + "\n", + "The first thing we have to do is to specify which tensors require gradients.\n", + "By default, when we create a tensor, it does not require gradients." + ], + "id": "e1a9591f" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:51.444570Z", + "iopub.status.busy": "2021-09-16T12:32:51.443320Z", + "iopub.status.idle": "2021-09-16T12:32:51.447126Z", + "shell.execute_reply": "2021-09-16T12:32:51.446665Z" + }, + "papermill": { + "duration": 0.054607, + "end_time": "2021-09-16T12:32:51.447227", + "exception": false, + "start_time": "2021-09-16T12:32:51.392620", + "status": "completed" + }, + "tags": [], + "id": "9f94399d", + "outputId": "9a315d48-5385-4e97-953d-574848a420e4" + }, + "source": [ + "x = torch.ones((3,))\n", + "print(x.requires_grad)" + ], + "id": "9f94399d", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.049292, + "end_time": "2021-09-16T12:32:51.546032", + "exception": false, + "start_time": "2021-09-16T12:32:51.496740", + "status": "completed" + }, + "tags": [], + "id": "12697ab1" + }, + "source": [ + "We can change this for an existing tensor using the function `requires_grad_()` (underscore indicating that this is a in-place operation).\n", + "Alternatively, when creating a tensor, you can pass the argument\n", + "`requires_grad=True` to most initializers we have seen above." + ], + "id": "12697ab1" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:51.647892Z", + "iopub.status.busy": "2021-09-16T12:32:51.647430Z", + "iopub.status.idle": "2021-09-16T12:32:51.649913Z", + "shell.execute_reply": "2021-09-16T12:32:51.649498Z" + }, + "papermill": { + "duration": 0.05454, + "end_time": "2021-09-16T12:32:51.650014", + "exception": false, + "start_time": "2021-09-16T12:32:51.595474", + "status": "completed" + }, + "tags": [], + "id": "7d565264", + "outputId": "d1359e2c-57c8-4e76-9632-dedeb388e9cc" + }, + "source": [ + "x.requires_grad_(True)\n", + "print(x.requires_grad)" + ], + "id": "7d565264", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.050272, + "end_time": "2021-09-16T12:32:51.750030", + "exception": false, + "start_time": "2021-09-16T12:32:51.699758", + "status": "completed" + }, + "tags": [], + "id": "a16f495e" + }, + "source": [ + "In order to get familiar with the concept of a computation graph, we will create one for the following function:\n", + "\n", + "$$y = \\frac{1}{|x|}\\sum_i \\left[(x_i + 2)^2 + 3\\right]$$\n", + "\n", + "You could imagine that $x$ are our parameters, and we want to optimize (either maximize or minimize) the output $y$.\n", + "For this, we want to obtain the gradients $\\partial y / \\partial \\mathbf{x}$.\n", + "For our example, we'll use $\\mathbf{x}=[0,1,2]$ as our input." + ], + "id": "a16f495e" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:51.853091Z", + "iopub.status.busy": "2021-09-16T12:32:51.852629Z", + "iopub.status.idle": "2021-09-16T12:32:51.855635Z", + "shell.execute_reply": "2021-09-16T12:32:51.855175Z" + }, + "papermill": { + "duration": 0.055874, + "end_time": "2021-09-16T12:32:51.855735", + "exception": false, + "start_time": "2021-09-16T12:32:51.799861", + "status": "completed" + }, + "tags": [], + "id": "abd9c738", + "outputId": "e7e68aea-7afb-4d35-9e83-eb98cbc0c1cc" + }, + "source": [ + "x = torch.arange(3, dtype=torch.float32, requires_grad=True) # Only float tensors can have gradients\n", + "print(\"X\", x)" + ], + "id": "abd9c738", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([0., 1., 2.], requires_grad=True)\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.05566, + "end_time": "2021-09-16T12:32:51.961765", + "exception": false, + "start_time": "2021-09-16T12:32:51.906105", + "status": "completed" + }, + "tags": [], + "id": "548b420c" + }, + "source": [ + "Now let's build the computation graph step by step.\n", + "You can combine multiple operations in a single line, but we will\n", + "separate them here to get a better understanding of how each operation\n", + "is added to the computation graph." + ], + "id": "548b420c" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:52.072120Z", + "iopub.status.busy": "2021-09-16T12:32:52.071647Z", + "iopub.status.idle": "2021-09-16T12:32:52.074610Z", + "shell.execute_reply": "2021-09-16T12:32:52.074989Z" + }, + "papermill": { + "duration": 0.056246, + "end_time": "2021-09-16T12:32:52.075114", + "exception": false, + "start_time": "2021-09-16T12:32:52.018868", + "status": "completed" + }, + "tags": [], + "id": "50d91bf7", + "outputId": "9851aea1-fb05-482b-b55f-1ab70bc0bb57" + }, + "source": [ + "a = x + 2\n", + "b = a ** 2\n", + "c = b + 3\n", + "y = c.mean()\n", + "print(\"Y\", y)" + ], + "id": "50d91bf7", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Y tensor(12.6667, grad_fn=)\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.049612, + "end_time": "2021-09-16T12:32:52.175001", + "exception": false, + "start_time": "2021-09-16T12:32:52.125389", + "status": "completed" + }, + "tags": [], + "id": "e0d2f0ae" + }, + "source": [ + "Using the statements above, we have created a computation graph that looks similar to the figure below:\n", + "\n", + "
\n", + "\n", + "We calculate $a$ based on the inputs $x$ and the constant $2$, $b$ is $a$ squared, and so on.\n", + "The visualization is an abstraction of the dependencies between inputs and outputs of the operations we have applied.\n", + "Each node of the computation graph has automatically defined a function for calculating the gradients with respect to its inputs, `grad_fn`.\n", + "You can see this when we printed the output tensor $y$.\n", + "This is why the computation graph is usually visualized in the reverse direction (arrows point from the result to the inputs).\n", + "We can perform backpropagation on the computation graph by calling the\n", + "function `backward()` on the last output, which effectively calculates\n", + "the gradients for each tensor that has the property\n", + "`requires_grad=True`:" + ], + "id": "e0d2f0ae" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:52.278438Z", + "iopub.status.busy": "2021-09-16T12:32:52.277977Z", + "iopub.status.idle": "2021-09-16T12:32:52.363356Z", + "shell.execute_reply": "2021-09-16T12:32:52.362899Z" + }, + "papermill": { + "duration": 0.137892, + "end_time": "2021-09-16T12:32:52.363476", + "exception": false, + "start_time": "2021-09-16T12:32:52.225584", + "status": "completed" + }, + "tags": [], + "id": "d7c2de18" + }, + "source": [ + "y.backward()" + ], + "id": "d7c2de18", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.050494, + "end_time": "2021-09-16T12:32:52.465208", + "exception": false, + "start_time": "2021-09-16T12:32:52.414714", + "status": "completed" + }, + "tags": [], + "id": "44d67068" + }, + "source": [ + "`x.grad` will now contain the gradient $\\partial y/ \\partial \\mathcal{x}$, and this gradient indicates how a change in $\\mathbf{x}$ will affect output $y$ given the current input $\\mathbf{x}=[0,1,2]$:" + ], + "id": "44d67068" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:52.569520Z", + "iopub.status.busy": "2021-09-16T12:32:52.569055Z", + "iopub.status.idle": "2021-09-16T12:32:52.572034Z", + "shell.execute_reply": "2021-09-16T12:32:52.571551Z" + }, + "papermill": { + "duration": 0.056348, + "end_time": "2021-09-16T12:32:52.572135", + "exception": false, + "start_time": "2021-09-16T12:32:52.515787", + "status": "completed" + }, + "tags": [], + "id": "58d14f6c", + "outputId": "97e37bad-a05c-4028-d7b5-99cfc931f671" + }, + "source": [ + "print(x.grad)" + ], + "id": "58d14f6c", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([1.3333, 2.0000, 2.6667])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.050295, + "end_time": "2021-09-16T12:32:52.673692", + "exception": false, + "start_time": "2021-09-16T12:32:52.623397", + "status": "completed" + }, + "tags": [], + "id": "a180ccfc" + }, + "source": [ + "We can also verify these gradients by hand.\n", + "We will calculate the gradients using the chain rule, in the same way as PyTorch did it:\n", + "\n", + "$$\\frac{\\partial y}{\\partial x_i} = \\frac{\\partial y}{\\partial c_i}\\frac{\\partial c_i}{\\partial b_i}\\frac{\\partial b_i}{\\partial a_i}\\frac{\\partial a_i}{\\partial x_i}$$\n", + "\n", + "Note that we have simplified this equation to index notation, and by using the fact that all operation besides the mean do not combine the elements in the tensor.\n", + "The partial derivatives are:\n", + "\n", + "$$\n", + "\\frac{\\partial a_i}{\\partial x_i} = 1,\\hspace{1cm}\n", + "\\frac{\\partial b_i}{\\partial a_i} = 2\\cdot a_i\\hspace{1cm}\n", + "\\frac{\\partial c_i}{\\partial b_i} = 1\\hspace{1cm}\n", + "\\frac{\\partial y}{\\partial c_i} = \\frac{1}{3}\n", + "$$\n", + "\n", + "Hence, with the input being $\\mathbf{x}=[0,1,2]$, our gradients are $\\partial y/\\partial \\mathbf{x}=[4/3,2,8/3]$.\n", + "The previous code cell should have printed the same result." + ], + "id": "a180ccfc" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051077, + "end_time": "2021-09-16T12:32:52.777753", + "exception": false, + "start_time": "2021-09-16T12:32:52.726676", + "status": "completed" + }, + "tags": [], + "id": "804f38e2" + }, + "source": [ + "### GPU support\n", + "\n", + "A crucial feature of PyTorch is the support of GPUs, short for Graphics Processing Unit.\n", + "A GPU can perform many thousands of small operations in parallel, making it very well suitable for performing large matrix operations in neural networks.\n", + "When comparing GPUs to CPUs, we can list the following main differences (credit: [Kevin Krewell, 2009](https://blogs.nvidia.com/blog/2009/12/16/whats-the-difference-between-a-cpu-and-a-gpu/))\n", + "\n", + "
\n", + "\n", + "CPUs and GPUs have both different advantages and disadvantages, which is why many computers contain both components and use them for different tasks.\n", + "In case you are not familiar with GPUs, you can read up more details in this [NVIDIA blog post](https://blogs.nvidia.com/blog/2009/12/16/whats-the-difference-between-a-cpu-and-a-gpu/) or [here](https://www.intel.com/content/www/us/en/products/docs/processors/what-is-a-gpu.html).\n", + "\n", + "GPUs can accelerate the training of your network up to a factor of $100$ which is essential for large neural networks.\n", + "PyTorch implements a lot of functionality for supporting GPUs (mostly those of NVIDIA due to the libraries [CUDA](https://developer.nvidia.com/cuda-zone) and [cuDNN](https://developer.nvidia.com/cudnn)).\n", + "First, let's check whether you have a GPU available:" + ], + "id": "804f38e2" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:52.886128Z", + "iopub.status.busy": "2021-09-16T12:32:52.885628Z", + "iopub.status.idle": "2021-09-16T12:32:52.888249Z", + "shell.execute_reply": "2021-09-16T12:32:52.887851Z" + }, + "papermill": { + "duration": 0.059327, + "end_time": "2021-09-16T12:32:52.888348", + "exception": false, + "start_time": "2021-09-16T12:32:52.829021", + "status": "completed" + }, + "tags": [], + "id": "be576156", + "outputId": "35e4b3dd-c780-4b96-81e7-d3b590c6322d" + }, + "source": [ + "gpu_avail = torch.cuda.is_available()\n", + "print(f\"Is the GPU available? {gpu_avail}\")" + ], + "id": "be576156", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Is the GPU available? True\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051392, + "end_time": "2021-09-16T12:32:52.990937", + "exception": false, + "start_time": "2021-09-16T12:32:52.939545", + "status": "completed" + }, + "tags": [], + "id": "44ba6d0f" + }, + "source": [ + "If you have a GPU on your computer but the command above returns False, make sure you have the correct CUDA-version installed.\n", + "The `dl2020` environment comes with the CUDA-toolkit 10.1, which is selected for the Lisa supercomputer.\n", + "Please change it if necessary (CUDA 10.2 is currently common).\n", + "On Google Colab, make sure that you have selected a GPU in your runtime setup (in the menu, check under `Runtime -> Change runtime type`).\n", + "\n", + "By default, all tensors you create are stored on the CPU.\n", + "We can push a tensor to the GPU by using the function `.to(...)`, or `.cuda()`.\n", + "However, it is often a good practice to define a `device` object in your code which points to the GPU if you have one, and otherwise to the CPU.\n", + "Then, you can write your code with respect to this device object, and it allows you to run the same code on both a CPU-only system, and one with a GPU.\n", + "Let's try it below.\n", + "We can specify the device as follows:" + ], + "id": "44ba6d0f" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:53.096938Z", + "iopub.status.busy": "2021-09-16T12:32:53.096464Z", + "iopub.status.idle": "2021-09-16T12:32:53.099120Z", + "shell.execute_reply": "2021-09-16T12:32:53.098658Z" + }, + "papermill": { + "duration": 0.057283, + "end_time": "2021-09-16T12:32:53.099221", + "exception": false, + "start_time": "2021-09-16T12:32:53.041938", + "status": "completed" + }, + "tags": [], + "id": "c1821da3", + "outputId": "bc003b3f-93d1-486a-a580-dd094dd0df95" + }, + "source": [ + "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n", + "print(\"Device\", device)" + ], + "id": "c1821da3", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Device cuda\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.052772, + "end_time": "2021-09-16T12:32:53.204148", + "exception": false, + "start_time": "2021-09-16T12:32:53.151376", + "status": "completed" + }, + "tags": [], + "id": "c7b99a5d" + }, + "source": [ + "Now let's create a tensor and push it to the device:" + ], + "id": "c7b99a5d" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:53.312496Z", + "iopub.status.busy": "2021-09-16T12:32:53.312034Z", + "iopub.status.idle": "2021-09-16T12:32:55.885460Z", + "shell.execute_reply": "2021-09-16T12:32:55.884980Z" + }, + "papermill": { + "duration": 2.629406, + "end_time": "2021-09-16T12:32:55.885574", + "exception": false, + "start_time": "2021-09-16T12:32:53.256168", + "status": "completed" + }, + "tags": [], + "id": "be1ce082", + "outputId": "dd451975-df98-4f52-8803-5310cca38bf6" + }, + "source": [ + "x = torch.zeros(2, 3)\n", + "x = x.to(device)\n", + "print(\"X\", x)" + ], + "id": "be1ce082", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X tensor([[0., 0., 0.],\n", + " [0., 0., 0.]], device='cuda:0')\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.052118, + "end_time": "2021-09-16T12:32:55.989872", + "exception": false, + "start_time": "2021-09-16T12:32:55.937754", + "status": "completed" + }, + "tags": [], + "id": "e1bb4237" + }, + "source": [ + "In case you have a GPU, you should now see the attribute `device='cuda:0'` being printed next to your tensor.\n", + "The zero next to cuda indicates that this is the zero-th GPU device on your computer.\n", + "PyTorch also supports multi-GPU systems, but this you will only need once you have very big networks to train (if interested, see the [PyTorch documentation](https://pytorch.org/docs/stable/distributed.html#distributed-basics)).\n", + "We can also compare the runtime of a large matrix multiplication on the CPU with a operation on the GPU:" + ], + "id": "e1bb4237" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:56.099380Z", + "iopub.status.busy": "2021-09-16T12:32:56.098905Z", + "iopub.status.idle": "2021-09-16T12:32:56.633065Z", + "shell.execute_reply": "2021-09-16T12:32:56.632672Z" + }, + "papermill": { + "duration": 0.59052, + "end_time": "2021-09-16T12:32:56.633183", + "exception": false, + "start_time": "2021-09-16T12:32:56.042663", + "status": "completed" + }, + "tags": [], + "id": "097f28c5", + "outputId": "afb6477a-8658-4a72-adcf-83cc58bda39e" + }, + "source": [ + "x = torch.randn(5000, 5000)\n", + "\n", + "# CPU version\n", + "start_time = time.time()\n", + "_ = torch.matmul(x, x)\n", + "end_time = time.time()\n", + "print(f\"CPU time: {(end_time - start_time):6.5f}s\")\n", + "\n", + "# GPU version\n", + "x = x.to(device)\n", + "# The first operation on a CUDA device can be slow as it has to establish a CPU-GPU communication first.\n", + "# Hence, we run an arbitrary command first without timing it for a fair comparison.\n", + "if torch.cuda.is_available():\n", + " _ = torch.matmul(x * 0.0, x)\n", + "start_time = time.time()\n", + "_ = torch.matmul(x, x)\n", + "end_time = time.time()\n", + "print(f\"GPU time: {(end_time - start_time):6.5f}s\")" + ], + "id": "097f28c5", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU time: 0.25468s\n", + "GPU time: 0.00011s\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.054603, + "end_time": "2021-09-16T12:32:56.740740", + "exception": false, + "start_time": "2021-09-16T12:32:56.686137", + "status": "completed" + }, + "tags": [], + "id": "e6502b2e" + }, + "source": [ + "Depending on the size of the operation and the CPU/GPU in your system, the speedup of this operation can be >500x.\n", + "As `matmul` operations are very common in neural networks, we can already see the great benefit of training a NN on a GPU.\n", + "The time estimate can be relatively noisy here because we haven't run it for multiple times.\n", + "Feel free to extend this, but it also takes longer to run.\n", + "\n", + "When generating random numbers, the seed between CPU and GPU is not synchronized.\n", + "Hence, we need to set the seed on the GPU separately to ensure a reproducible code.\n", + "Note that due to different GPU architectures, running the same code on different GPUs does not guarantee the same random numbers.\n", + "Still, we don't want that our code gives us a different output every time we run it on the exact same hardware.\n", + "Hence, we also set the seed on the GPU:" + ], + "id": "e6502b2e" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:56.849316Z", + "iopub.status.busy": "2021-09-16T12:32:56.848847Z", + "iopub.status.idle": "2021-09-16T12:32:56.850935Z", + "shell.execute_reply": "2021-09-16T12:32:56.850475Z" + }, + "papermill": { + "duration": 0.057334, + "end_time": "2021-09-16T12:32:56.851032", + "exception": false, + "start_time": "2021-09-16T12:32:56.793698", + "status": "completed" + }, + "tags": [], + "id": "5b767a95" + }, + "source": [ + "# GPU operations have a separate seed we also want to set\n", + "if torch.cuda.is_available():\n", + " torch.cuda.manual_seed(42)\n", + " torch.cuda.manual_seed_all(42)\n", + "\n", + "# Additionally, some operations on a GPU are implemented stochastic for efficiency\n", + "# We want to ensure that all operations are deterministic on GPU (if used) for reproducibility\n", + "torch.backends.cudnn.determinstic = True\n", + "torch.backends.cudnn.benchmark = False" + ], + "id": "5b767a95", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051866, + "end_time": "2021-09-16T12:32:56.955066", + "exception": false, + "start_time": "2021-09-16T12:32:56.903200", + "status": "completed" + }, + "tags": [], + "id": "f4ca3f5b" + }, + "source": [ + "## Learning by example: Continuous XOR\n", + "
\n", + "\n", + "If we want to build a neural network in PyTorch, we could specify all our parameters (weight matrices, bias vectors) using `Tensors` (with `requires_grad=True`), ask PyTorch to calculate the gradients and then adjust the parameters.\n", + "But things can quickly get cumbersome if we have a lot of parameters.\n", + "In PyTorch, there is a package called `torch.nn` that makes building neural networks more convenient.\n", + "\n", + "We will introduce the libraries and all additional parts you might need to train a neural network in PyTorch, using a simple example classifier on a simple yet well known example: XOR.\n", + "Given two binary inputs $x_1$ and $x_2$, the label to predict is $1$ if either $x_1$ or $x_2$ is $1$ while the other is $0$, or the label is $0$ in all other cases.\n", + "The example became famous by the fact that a single neuron, i.e. a linear classifier, cannot learn this simple function.\n", + "Hence, we will learn how to build a small neural network that can learn this function.\n", + "To make it a little bit more interesting, we move the XOR into continuous space and introduce some gaussian noise on the binary inputs.\n", + "Our desired separation of an XOR dataset could look as follows:\n", + "\n", + "
" + ], + "id": "f4ca3f5b" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051714, + "end_time": "2021-09-16T12:32:57.058731", + "exception": false, + "start_time": "2021-09-16T12:32:57.007017", + "status": "completed" + }, + "tags": [], + "id": "e23f8eac" + }, + "source": [ + "### The model\n", + "\n", + "The package `torch.nn` defines a series of useful classes like linear networks layers, activation functions, loss functions etc.\n", + "A full list can be found [here](https://pytorch.org/docs/stable/nn.html).\n", + "In case you need a certain network layer, check the documentation of the package first before writing the layer yourself as the package likely contains the code for it already.\n", + "We import it below:" + ], + "id": "e23f8eac" + }, + { + "cell_type": "code", + "metadata": { + "lines_to_next_cell": 0, + "papermill": { + "duration": 0.052216, + "end_time": "2021-09-16T12:32:57.162758", + "exception": false, + "start_time": "2021-09-16T12:32:57.110542", + "status": "completed" + }, + "tags": [], + "id": "8592c856" + }, + "source": [ + "" + ], + "id": "8592c856", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "papermill": { + "duration": 0.052415, + "end_time": "2021-09-16T12:32:57.268259", + "exception": false, + "start_time": "2021-09-16T12:32:57.215844", + "status": "completed" + }, + "tags": [], + "id": "bf8c706a" + }, + "source": [ + "" + ], + "id": "bf8c706a", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051617, + "end_time": "2021-09-16T12:32:57.371727", + "exception": false, + "start_time": "2021-09-16T12:32:57.320110", + "status": "completed" + }, + "tags": [], + "id": "ba549835" + }, + "source": [ + "Additionally to `torch.nn`, there is also `torch.nn.functional`.\n", + "It contains functions that are used in network layers.\n", + "This is in contrast to `torch.nn` which defines them as `nn.Modules` (more on it below), and `torch.nn` actually uses a lot of functionalities from `torch.nn.functional`.\n", + "Hence, the functional package is useful in many situations, and so we import it as well here." + ], + "id": "ba549835" + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.052024, + "end_time": "2021-09-16T12:32:57.475971", + "exception": false, + "start_time": "2021-09-16T12:32:57.423947", + "status": "completed" + }, + "tags": [], + "id": "acc7d527" + }, + "source": [ + "#### nn.Module\n", + "\n", + "In PyTorch, a neural network is build up out of modules.\n", + "Modules can contain other modules, and a neural network is considered to be a module itself as well.\n", + "The basic template of a module is as follows:" + ], + "id": "acc7d527" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:57.583596Z", + "iopub.status.busy": "2021-09-16T12:32:57.583131Z", + "iopub.status.idle": "2021-09-16T12:32:57.585190Z", + "shell.execute_reply": "2021-09-16T12:32:57.584806Z" + }, + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.057057, + "end_time": "2021-09-16T12:32:57.585292", + "exception": false, + "start_time": "2021-09-16T12:32:57.528235", + "status": "completed" + }, + "tags": [], + "id": "34d1a3d7" + }, + "source": [ + "class MyModule(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " # Some init for my module\n", + "\n", + " def forward(self, x):\n", + " # Function for performing the calculation of the module.\n", + " pass" + ], + "id": "34d1a3d7", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.051843, + "end_time": "2021-09-16T12:32:57.689235", + "exception": false, + "start_time": "2021-09-16T12:32:57.637392", + "status": "completed" + }, + "tags": [], + "id": "04b470dc" + }, + "source": [ + "The forward function is where the computation of the module is taken place, and is executed when you call the module (`nn = MyModule(); nn(x)`).\n", + "In the init function, we usually create the parameters of the module, using `nn.Parameter`, or defining other modules that are used in the forward function.\n", + "The backward calculation is done automatically, but could be overwritten as well if wanted.\n", + "\n", + "#### Simple classifier\n", + "We can now make use of the pre-defined modules in the `torch.nn` package, and define our own small neural network.\n", + "We will use a minimal network with a input layer, one hidden layer with tanh as activation function, and a output layer.\n", + "In other words, our networks should look something like this:\n", + "\n", + "
\n", + "\n", + "The input neurons are shown in blue, which represent the coordinates $x_1$ and $x_2$ of a data point.\n", + "The hidden neurons including a tanh activation are shown in white, and the output neuron in red.\n", + "In PyTorch, we can define this as follows:" + ], + "id": "04b470dc" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:57.800277Z", + "iopub.status.busy": "2021-09-16T12:32:57.799807Z", + "iopub.status.idle": "2021-09-16T12:32:57.801874Z", + "shell.execute_reply": "2021-09-16T12:32:57.801393Z" + }, + "papermill": { + "duration": 0.057783, + "end_time": "2021-09-16T12:32:57.801973", + "exception": false, + "start_time": "2021-09-16T12:32:57.744190", + "status": "completed" + }, + "tags": [], + "id": "2606872c" + }, + "source": [ + "class SimpleClassifier(nn.Module):\n", + " def __init__(self, num_inputs, num_hidden, num_outputs):\n", + " super().__init__()\n", + " # Initialize the modules we need to build the network\n", + " self.linear1 = nn.Linear(num_inputs, num_hidden)\n", + " self.act_fn = nn.Tanh()\n", + " self.linear2 = nn.Linear(num_hidden, num_outputs)\n", + "\n", + " def forward(self, x):\n", + " # Perform the calculation of the model to determine the prediction\n", + " x = self.linear1(x)\n", + " x = self.act_fn(x)\n", + " x = self.linear2(x)\n", + " return x" + ], + "id": "2606872c", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.051977, + "end_time": "2021-09-16T12:32:57.905865", + "exception": false, + "start_time": "2021-09-16T12:32:57.853888", + "status": "completed" + }, + "tags": [], + "id": "708b426d" + }, + "source": [ + "For the examples in this notebook, we will use a tiny neural network with two input neurons and four hidden neurons.\n", + "As we perform binary classification, we will use a single output neuron.\n", + "Note that we do not apply a sigmoid on the output yet.\n", + "This is because other functions, especially the loss, are more efficient and precise to calculate on the original outputs instead of the sigmoid output.\n", + "We will discuss the detailed reason later." + ], + "id": "708b426d" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:58.014018Z", + "iopub.status.busy": "2021-09-16T12:32:58.013529Z", + "iopub.status.idle": "2021-09-16T12:32:58.016608Z", + "shell.execute_reply": "2021-09-16T12:32:58.016147Z" + }, + "papermill": { + "duration": 0.058322, + "end_time": "2021-09-16T12:32:58.016706", + "exception": false, + "start_time": "2021-09-16T12:32:57.958384", + "status": "completed" + }, + "tags": [], + "id": "f8c99074", + "outputId": "472ac145-a50c-42e7-f25f-c05cc72be78c" + }, + "source": [ + "model = SimpleClassifier(num_inputs=2, num_hidden=4, num_outputs=1)\n", + "# Printing a module shows all its submodules\n", + "print(model)" + ], + "id": "f8c99074", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SimpleClassifier(\n", + " (linear1): Linear(in_features=2, out_features=4, bias=True)\n", + " (act_fn): Tanh()\n", + " (linear2): Linear(in_features=4, out_features=1, bias=True)\n", + ")\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.0523, + "end_time": "2021-09-16T12:32:58.121280", + "exception": false, + "start_time": "2021-09-16T12:32:58.068980", + "status": "completed" + }, + "tags": [], + "id": "7b432a95" + }, + "source": [ + "Printing the model lists all submodules it contains.\n", + "The parameters of a module can be obtained by using its `parameters()` functions, or `named_parameters()` to get a name to each parameter object.\n", + "For our small neural network, we have the following parameters:" + ], + "id": "7b432a95" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:58.230366Z", + "iopub.status.busy": "2021-09-16T12:32:58.229902Z", + "iopub.status.idle": "2021-09-16T12:32:58.232813Z", + "shell.execute_reply": "2021-09-16T12:32:58.232352Z" + }, + "papermill": { + "duration": 0.059317, + "end_time": "2021-09-16T12:32:58.232913", + "exception": false, + "start_time": "2021-09-16T12:32:58.173596", + "status": "completed" + }, + "tags": [], + "id": "52c7230d", + "outputId": "2333aa10-d564-4873-a505-3f73dbf6f76e" + }, + "source": [ + "for name, param in model.named_parameters():\n", + " print(f\"Parameter {name}, shape {param.shape}\")" + ], + "id": "52c7230d", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parameter linear1.weight, shape torch.Size([4, 2])\n", + "Parameter linear1.bias, shape torch.Size([4])\n", + "Parameter linear2.weight, shape torch.Size([1, 4])\n", + "Parameter linear2.bias, shape torch.Size([1])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.053627, + "end_time": "2021-09-16T12:32:58.340801", + "exception": false, + "start_time": "2021-09-16T12:32:58.287174", + "status": "completed" + }, + "tags": [], + "id": "b2650ef9" + }, + "source": [ + "Each linear layer has a weight matrix of the shape `[output, input]`, and a bias of the shape `[output]`.\n", + "The tanh activation function does not have any parameters.\n", + "Note that parameters are only registered for `nn.Module` objects that are direct object attributes, i.e. `self.a = ...`.\n", + "If you define a list of modules, the parameters of those are not registered for the outer module and can cause some issues when you try to optimize your module.\n", + "There are alternatives, like `nn.ModuleList`, `nn.ModuleDict` and `nn.Sequential`, that allow you to have different data structures of modules.\n", + "We will use them in a few later tutorials and explain them there." + ], + "id": "b2650ef9" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.052923, + "end_time": "2021-09-16T12:32:58.446527", + "exception": false, + "start_time": "2021-09-16T12:32:58.393604", + "status": "completed" + }, + "tags": [], + "id": "463b7836" + }, + "source": [ + "### The data\n", + "\n", + "PyTorch also provides a few functionalities to load the training and\n", + "test data efficiently, summarized in the package `torch.utils.data`." + ], + "id": "463b7836" + }, + { + "cell_type": "code", + "metadata": { + "papermill": { + "duration": 0.052877, + "end_time": "2021-09-16T12:32:58.552525", + "exception": false, + "start_time": "2021-09-16T12:32:58.499648", + "status": "completed" + }, + "tags": [], + "id": "0ab84d11" + }, + "source": [ + "" + ], + "id": "0ab84d11", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.052744, + "end_time": "2021-09-16T12:32:58.658400", + "exception": false, + "start_time": "2021-09-16T12:32:58.605656", + "status": "completed" + }, + "tags": [], + "id": "21c14544" + }, + "source": [ + "The data package defines two classes which are the standard interface for handling data in PyTorch: `data.Dataset`, and `data.DataLoader`.\n", + "The dataset class provides an uniform interface to access the\n", + "training/test data, while the data loader makes sure to efficiently load\n", + "and stack the data points from the dataset into batches during training." + ], + "id": "21c14544" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.055595, + "end_time": "2021-09-16T12:32:58.767233", + "exception": false, + "start_time": "2021-09-16T12:32:58.711638", + "status": "completed" + }, + "tags": [], + "id": "fb7ac0a3" + }, + "source": [ + "#### The dataset class\n", + "\n", + "The dataset class summarizes the basic functionality of a dataset in a natural way.\n", + "To define a dataset in PyTorch, we simply specify two functions: `__getitem__`, and `__len__`.\n", + "The get-item function has to return the $i$-th data point in the dataset, while the len function returns the size of the dataset.\n", + "For the XOR dataset, we can define the dataset class as follows:" + ], + "id": "fb7ac0a3" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:58.880519Z", + "iopub.status.busy": "2021-09-16T12:32:58.880040Z", + "iopub.status.idle": "2021-09-16T12:32:58.881657Z", + "shell.execute_reply": "2021-09-16T12:32:58.882056Z" + }, + "papermill": { + "duration": 0.061328, + "end_time": "2021-09-16T12:32:58.882175", + "exception": false, + "start_time": "2021-09-16T12:32:58.820847", + "status": "completed" + }, + "tags": [], + "id": "85adf0a4" + }, + "source": [ + "\n", + "\n", + "class XORDataset(data.Dataset):\n", + " def __init__(self, size, std=0.1):\n", + " \"\"\"\n", + " Inputs:\n", + " size - Number of data points we want to generate\n", + " std - Standard deviation of the noise (see generate_continuous_xor function)\n", + " \"\"\"\n", + " super().__init__()\n", + " self.size = size\n", + " self.std = std\n", + " self.generate_continuous_xor()\n", + "\n", + " def generate_continuous_xor(self):\n", + " # Each data point in the XOR dataset has two variables, x and y, that can be either 0 or 1\n", + " # The label is their XOR combination, i.e. 1 if only x or only y is 1 while the other is 0.\n", + " # If x=y, the label is 0.\n", + " data = torch.randint(low=0, high=2, size=(self.size, 2), dtype=torch.float32)\n", + " label = (data.sum(dim=1) == 1).to(torch.long)\n", + " # To make it slightly more challenging, we add a bit of gaussian noise to the data points.\n", + " data += self.std * torch.randn(data.shape)\n", + "\n", + " self.data = data\n", + " self.label = label\n", + "\n", + " def __len__(self):\n", + " # Number of data point we have. Alternatively self.data.shape[0], or self.label.shape[0]\n", + " return self.size\n", + "\n", + " def __getitem__(self, idx):\n", + " # Return the idx-th data point of the dataset\n", + " # If we have multiple things to return (data point and label), we can return them as tuple\n", + " data_point = self.data[idx]\n", + " data_label = self.label[idx]\n", + " return data_point, data_label" + ], + "id": "85adf0a4", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.053132, + "end_time": "2021-09-16T12:32:58.988271", + "exception": false, + "start_time": "2021-09-16T12:32:58.935139", + "status": "completed" + }, + "tags": [], + "id": "82143473" + }, + "source": [ + "Let's try to create such a dataset and inspect it:" + ], + "id": "82143473" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:59.100298Z", + "iopub.status.busy": "2021-09-16T12:32:59.099829Z", + "iopub.status.idle": "2021-09-16T12:32:59.103186Z", + "shell.execute_reply": "2021-09-16T12:32:59.103565Z" + }, + "papermill": { + "duration": 0.059959, + "end_time": "2021-09-16T12:32:59.103683", + "exception": false, + "start_time": "2021-09-16T12:32:59.043724", + "status": "completed" + }, + "tags": [], + "id": "d35a9331", + "outputId": "6af17e28-2bc7-4324-b13d-8a92bfd9508c" + }, + "source": [ + "dataset = XORDataset(size=200)\n", + "print(\"Size of dataset:\", len(dataset))\n", + "print(\"Data point 0:\", dataset[0])" + ], + "id": "d35a9331", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size of dataset: 200\n", + "Data point 0: (tensor([0.9632, 0.1117]), tensor(1))\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.053101, + "end_time": "2021-09-16T12:32:59.210237", + "exception": false, + "start_time": "2021-09-16T12:32:59.157136", + "status": "completed" + }, + "tags": [], + "id": "f8eeb814" + }, + "source": [ + "To better relate to the dataset, we visualize the samples below." + ], + "id": "f8eeb814" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:59.324080Z", + "iopub.status.busy": "2021-09-16T12:32:59.323610Z", + "iopub.status.idle": "2021-09-16T12:32:59.325640Z", + "shell.execute_reply": "2021-09-16T12:32:59.325245Z" + }, + "papermill": { + "duration": 0.060548, + "end_time": "2021-09-16T12:32:59.325755", + "exception": false, + "start_time": "2021-09-16T12:32:59.265207", + "status": "completed" + }, + "tags": [], + "id": "40b4cbff" + }, + "source": [ + "def visualize_samples(data, label):\n", + " if isinstance(data, torch.Tensor):\n", + " data = data.cpu().numpy()\n", + " if isinstance(label, torch.Tensor):\n", + " label = label.cpu().numpy()\n", + " data_0 = data[label == 0]\n", + " data_1 = data[label == 1]\n", + "\n", + " plt.figure(figsize=(4, 4))\n", + " plt.scatter(data_0[:, 0], data_0[:, 1], edgecolor=\"#333\", label=\"Class 0\")\n", + " plt.scatter(data_1[:, 0], data_1[:, 1], edgecolor=\"#333\", label=\"Class 1\")\n", + " plt.title(\"Dataset samples\")\n", + " plt.ylabel(r\"$x_2$\")\n", + " plt.xlabel(r\"$x_1$\")\n", + " plt.legend()" + ], + "id": "40b4cbff", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:32:59.448747Z", + "iopub.status.busy": "2021-09-16T12:32:59.448284Z", + "iopub.status.idle": "2021-09-16T12:32:59.938181Z", + "shell.execute_reply": "2021-09-16T12:32:59.938567Z" + }, + "papermill": { + "duration": 0.560114, + "end_time": "2021-09-16T12:32:59.938710", + "exception": false, + "start_time": "2021-09-16T12:32:59.378596", + "status": "completed" + }, + "tags": [], + "id": "44e7f18f", + "outputId": "f35672d9-f0e6-43e0-b163-e35e481a29a5" + }, + "source": [ + "visualize_samples(dataset.data, dataset.label)\n", + "plt.show()" + ], + "id": "44e7f18f", + "execution_count": null, + "outputs": [ + { + "data": { + "application/pdf": "JVBERi0xLjQKJazcIKu6CjEgMCBvYmoKPDwgL1BhZ2VzIDIgMCBSIC9UeXBlIC9DYXRhbG9nID4+CmVuZG9iago4IDAgb2JqCjw8IC9FeHRHU3RhdGUgNCAwIFIgL0ZvbnQgMyAwIFIgL1BhdHRlcm4gNSAwIFIKL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0gL1NoYWRpbmcgNiAwIFIKL1hPYmplY3QgNyAwIFIgPj4KZW5kb2JqCjExIDAgb2JqCjw8IC9Bbm5vdHMgMTAgMCBSIC9Db250ZW50cyA5IDAgUgovR3JvdXAgPDwgL0NTIC9EZXZpY2VSR0IgL1MgL1RyYW5zcGFyZW5jeSAvVHlwZSAvR3JvdXAgPj4KL01lZGlhQm94IFsgMCAwIDI4OC45Nzc0NDU1MTg0IDI3Ny4zMDg3NSBdIC9QYXJlbnQgMiAwIFIgL1Jlc291cmNlcyA4IDAgUgovVHlwZSAvUGFnZSA+PgplbmRvYmoKOSAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDEyIDAgUiA+PgpzdHJlYW0KeJytm0uvHLcRhffzK3ppL26LxTeXVhQbMJCFEyFZBFkIiqxYkJTYsuP8/HyHPfdO91XNSAacIIEvzSaL9Th1qsix5c3pyVe2vP6whOUN//t1seWb5cmzV//94eWrP3/zdHn54RQYf3eKva+jtZwrf77d/xlbW1PorTAcDn/963R6f2J1vviGhV+fTiWuVpLFsqS2lpyYxtKtrOnR6Nv9aCx5HX0bvqywH2Wn708/Ls7yMaY1LtHamvPy06vlb8v75clXcR55tWil9spW/JHr+T/tFNYWSu8xlTiWn14j+xv+9yuTIie5+t3y6LvTj5xe6rNlVLSSem2jJVt6Qnm1pFBKqMvLd8uTP4Xl2b8fZlsqa8tFc3LkL1uthzjKYGNneqzr4IS92AhxubO1ppIt5TC8xe+shDXUGrBeroO/c1hrHGbVsjcfVWLPGizXstwxN6Wc6hi5e7NTWFNGA4kJabGy5h5LLz2V5p202YrQOEyKmM34mu/iqBayd9Q7C2uvvaeYQ6pLwxSlmulrd3bqa0gNPcaaddSSVrNgtTR2c8TJaeWkIYVRel2kqdF6TaOO7s1nA0ypuXmMlKTLujY27K1iRG8HfDSYjd4xUV+s9tVCDy3hT+4GOO5oiCO/2rTbG+qxYa5AZe38+2yZQy95Re2xjpyjaysjTIY8NofaJD0h1Rs2ybh38pcPKWJfYwNifdW3o2Ji17iVsyYkx2NwzJFXonG0kWr0zorqcw8F6UPGsqEi2VAUeILEvjYMma30MJaYV5yX8FKUuCfNeQ094vEjjX4fAzmg+JHdEJRjlhBTbTqqtbqWJjdLPbua6WuPMZpCHOkAQaFgaxbdmJKjt9hHTJG4uDOT3awmmdUVx1AeIYQrELibn+JG3WJwdXlnKAgnbHJc04GxRRYi1Mge3g58wFpWwwhpzB1iiynknq7ACLAxCO0YRgLF2BFn6q0n8G8M1x3S2nNp6IUIn7He0BfgQMB5O2QpkQVNkRuJKyYXAs11iLpG47hoEmGImm5gCg5VfGDIzEi9ICzYdMbANmK2kcz1ZVJLLYAqsSr3IzmlCvS3VvzQApqIogZmE0msHnoGYYsPVKawLiHXjDsS5UD4YKb+9GFqZe3RQA+LSkEFP6i1u4oxHMHAO2Zg16Jj4BUlZveYAzdH7MjJirwGNxsjBDKL65YBH0DnVhVXqD2uHBjrFozthfggHQB8QzEYlb9zIcqLH7NllcxZ2N05pebmyKGvRjjBB5ThsxOJUWIJHW+35lt0rCCR8gfaIcJI0dZIsoEvvCyb1kR6B2FIhUuKzCZmJgb58oDsKKOalKnT4DJ4P/Fdk+sDeHiAEZBsgW6QFnIAxHURho9n97VWshL6IIXLefPooE/0nRf/IvHgIZFwsFAkWR8s755UwZaETMwiawrmiXEYA5DmT68rzhdAlhL1dzVcMgOULOEeFYGJ1mpiBnlRGkqRPMvn7mHvRIFIs0hhecgvUX/JbaZxHy+BL8hTSKmMwZEraRTK0APZ6iog4BFsQWYem7mAktZQkvuFRMgDHtyTqI0JIHA+HCK4rm+GFjGBidos+J4R6rkhnY+uIgbsXtgiTZIGc2kRLgCv9U9cUBHBBxNlA80HcAbplyTnhSKQQ0YrKAhmLJTIUEixsO6boALHbBAjSahsJoG3WR8Ejasg4k/5h7RsYi5pDeAnkZ+ylx0gloUAr+QSEBXpQohRcNh8cVAP+QCODybrYxPp5gNzU08vIsUs3s1gpYlsSiaJGev5LBZaZ5AiyhmrSu5YFc8pgJ0HDaheaA//INRZPKAjSKnA3M0NsBs0H4Lyp2yLdDLWEFxdIVJQtI5KMiQ5xLXL0AP491GZcCIusvIw9J1Q5ssrsDapCDQRaox3jo05II14WnJJ8t0gVZJPOhAiK0exO2Knuma1tLYSUiZSq3jUSpZrAs3s6UYoj9ZKhOQvglhtQ53pohouLkBGc4iyiIp3mCmHFSX0YY2En2JXGGFjnKDnOl34SvUAfy2imA31qPYApyqFGbp0Y4oNVFUBsLIjahR+Q3b9JE6mVc5WRE8rQcFJKdXFWBmpEqkYSQBIEllLhI7KBVwfC/Jf8Eb5j616qqRwMvOVXGV4DFgGGpTpoKge9IRqdh/MqKOoqyjy4JiiW0H+2TJQcgU92ipSkZhC/En1nFY13mie+KLDgDcADPFeKOJw/c5ho1tnANZjbYq9TvE2tr9TA5gIMDeestYnI8JvKUxEHQbGCJgu+RuoICd7V4Iqb1hYRGIGYeDJP7AmboA9KQo7OAmKYWlzsSaCddP0uDq6RLPUb8L64PN0ajCVzNax55Y7G3kTLyr+fEzJciBrzUsFyAhWknNzXT4SIiAjTgZcUJHBYoDh4vcdZtmiLM5/SQdb9Ytexbtj8HyetKcOCUgUNjupJsGuFMy+avKEYZacxCIqwJvE8QsM3MrUACpEKkECmUWppMve/QLD4HRQAnUn6izDAnal4Metq9/dgCQQUYO0Q5hWGALAAA77rAXCR4VXEH3MGk+gT0KpYr+ez1TZiaq3TtSGzxEiRLwLHpgVB2udbGlboyIloAAk655h0QT+Cz2GrCxiuqRhgO9aPMFdqyW4t+qADQ+Aqa7egFtPqbTDhgU4RYAJxYNSnr9xhd38707fLb+1TaekALmlKMVv1ZsDkoGyIaD8qDd3nLw8mrxryEVlvYp1iCMAVf05uAC1Ws1nfdujKBoUbzWrLSd0jY1aZLB6d2bL1lBt8qIK2Il+jTqa3GP3nvJofo5asRFJKaUJr2ArCRnzJ0+cCr5nOAnBkZSZyAdRPZygzpC7Ax5CUgVTR5MLqDGWaxQH9Q9M8GBMShWAqG78uKECnSp5H9x1gV5KjZJUnL6Y8j6BNNT78Tao60hNjlgmpS9NsTeSnCx7G6TZAgA8SPl5UY2cMAAZLnjTwRYLZJeg9pPsRz4D+YIl32JAOs49O65tNkOA09DVbPMMgDapXChwprDKtrVGbXfP5u0xxM8uXwYQ2uzikctERYenSmKZ2WjOINZgWqagRTUpur4ThSsU0tSOhH5S36yraDBfK2Qjshkcss1+JcxS9SnMO15x5SY94zoNBrFE9WeDOm25eCcFxCqVnJymCK/VZyZ15+JaFOJGgEIHYflDWsftW+LcublxInxOYoO4rvokImgqwMj77gZqCRDoKlo4HCwefWPaFIp71L4WUA9HbJT04CuAqZrRd3fSMMU0BmV+Ua+KMgv4IxN78Ye3ptyHehHszbeQuYqlxvBi407dx8xKlFZJ7hgIb9yn4WP3bfRHeIDsQAeIPbKuAALBSDam9KIc9BEHS2JINe/EOsi06g8g0RU8wM3AMnIxbh631AnjYIMe+pWIogSHYOeeZy89qqkF2kNbmhdSsaoj2oQ44gZkLEpZm2WUKw8FYFVBh7ePDUGLlB9seBrCuFTbFIspyI2LKgFdeVwJWPVbwSb+dVY3mjJkpYKBE1HTueJQiyjl4IfQgU09KqF0ZZOuGCCpHwQmwCgAKPBKRVunDKjeDnfihWRXHKckhZaqKhH25jsE5hHzaBVCdr4IIaio6bqvn0nNRPuS7vRwH8LE4LsujGztEjJgEkyNzX0wNPotLninMLvJsyU0pYcciN740idiT8SKYIwqGcvoYhYt+J4ADyNSImR8HrVCcnNpaoY0X/UqD0g0oL0sR+pHLHmyuZ6flRk4F7ElxFTg48gsn31NqrEI/KkZl7saizMTUpUqQTgf6A4GQC0AZ9q6jORlEHr4YbWqqIGKxaiuCl4NB8zwUBcwqSY412xx6l5i9s1EttWw9ZkIlQ54Ogq5uwhx1QyouvSBu3hRG3RhlnQPlKbf4D7IXnQN4gG46lbKQDXzJoCDm1kF+UPd+BEmVLEgZZR56ZN0swZsNh83kb4TIORD4ntRI3JEWDdlW/Tk0a0sfqXbTAWt6oWhNg9A4kpPeuuDbFjDvGjVNY/hBsSxqxxllEAdqy6wboXW2bKB02df+qAsgqqpFGuft8Aix6CuL8+dPDMAsbr6mjWDwWiHUoy3PsYpZBExRLkObhfVf+/JzZ3K+5nCiZpX/UpdxouxUDT46VDXZVR2BFUfkwBj69461MIzrO4VhAdjXvWhEMkxhisI4az71FYsb02cjPyjm+9hABPlNsyS+mmcPUwt90a0ukpUNw4HMZVqaiOp11ZhcpBwN3Va62LSkHmSTdq6d4RTU6O++iFLbVNAXtJbV8VOZmuZqIp+CKoza1uuUlGtO5NEEqIQNm95dSoTLD7VyaJX5Vhcx0dW1AcVVSM8NjEFNUZHQJLG//t5X0U3OarJltuVMVmIKoi4unbaJEI6C3WdlvDALUq4Ig/KISp0X637cBMaqujtNbpOTMWeVPmC9Xm7FiRjDj0YcEOWvEdaY8E+CzHyVIYW4Q1XaIL6/mKsseGTbXb5FfFEguv3Jrc3tdq2mw+USxmsW8fhp54V6NA1bikCbyIAVVIbuq4siqM0SA6ftp2tHI7C19X7oKmLhZ0wflQLKuoyND/crh3mdnCywCbFcCbJJ6TIC3i9nxZm7xTH0ZVFnU2o3lQMBtXDjigdrwQ8gFI1//WwgOyWVZq7BJZqUSpOumKqFEAotdnEGzdCupKkGr8G91QrMkUVs747KmngAbqiVOsfnqPmFtF+hdjragBR8ffZSi94A0TEX37yLLydOpBC9vzABNYkh7+iyaoSTc8/SGV5tiKbOu+5tisRm9Xvq9J10JOOAS3GB6JbVefZh4HTRMGjoAYYmW+TvJOSXQO5TKXgJGVDyWZcgQ7doVH1CWqqal6VkmkS4hhcin6X9fhG3p3U+dfzArQ03d93MXIG4EVaaIJJgJz8kPXYxEf6UvEZ0qVum8+lLBmEgVwO1GlrOt0/+lq2lhH1u5pGes61vD7xqRhMzft3abtBsZTtpdnpKQL8em4k3U2Vq3UPdTS9eVqAy1nTn54+X558LQdann9/kr/2aUS+ev7P09+XLyDE5cvlH8vzb09/fH76Ua2q7WQP/6ATfG13z169efHXX/7y4v2Hu3c/vP/lA0daONJpnuA0tDtxFfeS7wZvSd4hzjg2Fqc69gUPO3lDuMh7v/28mM3E+X773eCt7ZWK+rxhbbqX+LQAe4U9CADbp8jVCfYS7EZvilD0Pi3B7vUo6TNEKI4OYjBSWFTU7F817kZvijB0Nz8aPAn0+wwRmqMF6oV1qHo/Pqzcjd4SIcb5UKzpNVL+tAjmegJVO4QxV6Dg8LbzMnpTBEoyShhKJOqm/BkiHH3hYFLo2dZdJgXObuS2kD5ft3/BIk++judVv/jfl8vzNydd4ZLa+rzbRuYZrHMi2WzOszlvd+RLHxvUT2JEOrJ6549H3SMHvbXTfpmzk16gDNuVz+dgx+8AHRfx+9a/LP0g/mXUFV9F7lRXm3UHTAai128jyMducxFCpWaZ92AHKXbDt8VQ9QSJzHrTMwuJG0ByU45U9dxWdPogx2X4E3LoaVBXt6/E/LE593Lkm3LoTqWrPXSU4zL8CTkIqaEbiqincDflqDflGLbOpyqP7HIZ/oQcfTb29XTCJk+4Lke/JUekPA/BUOpBjt3wbTnifE9GOic13XJTu+2mrAaHV5l7FOMy/AkxMhWF7mNj255u3sC4I8RtLEncB0cfQlI1cVvVP/0+CBcvCBeXb5fwwJOOvwhwX/H7D/NPf3Gf97+79ryf+b/hNwKH2Zdlbq0e5rl214bL60P2gDIDwkWqibpk6w/a3YwUj0Z69uLnFx9e/bx8ePHuP29ffdgb7MlXabu+fP2gyPOvIGIg9LfMpHcdFCN9k7r2e6l3428ZV99JXbD9uFLrbnbTU7p4HI33c1+e9uOcK6d5xLfHccObgC3I4G5H9UPSWSEX+S6jLw/nuYy/PdlQEz+3+QDvYZxC6TL7YcfDaLisvR+/nOftcfzh9Psdd5py9f1Svxp5evrUj0Ee/6jjXD08/iHI6eYPQU5RbauzCGVOPZtcvaEGAc378aiaL82qX6ON1ErFpdEy5v2MbldTpsSc6tdVZ50PwTWuR2Rpg7uHHaOeIukZ6mE03e+3rfEwrqsgdfOXw8qF+k2N5KMcRTWSjW2NndR1ez2pHXcnrJcT7vRR93KATD1ubxp2s/WcSR3748py3drDYzmCSvyslzl7qRl1Trgf3etjP37R3n7lnaZ3chzsspN6Z8XdCYOrj3CRY7poOP+E6hFW6QnD7pueN7e+mlD+8PbFBxx9j1G3n0t83puKg2/rndR47Njnwb1/iEyZ+vgHb2K06pcwj5w66QmjfjOy91L9dGWMNOw42udWR4dO8Ouhy9aDQ+cJfI/iSreiNt9nH9w5mxbj0Ad3ZnQ71+X856GjIz/Mu/jDfsGL9+x337naRdKdV+4OtR99OP9+8KKq3ao7rV4kOFjgIu7eXJdzPdj1I/t/vuNq5VvkdHNa2zvtd6f/A2CYYAEKZW5kc3RyZWFtCmVuZG9iagoxMiAwIG9iago0ODQ0CmVuZG9iagoxMCAwIG9iagpbIF0KZW5kb2JqCjE5IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggOTUgPj4Kc3RyZWFtCnicPYxBDsAgCATvvGI/0AQRFf/TND3Y/1+7RtsLTHZhSjcoDiucVRXFG84kHz6SvcNax5CimUdDnN3cFg5LjRSrWBYWnmERpLQ1zPi8KGtgSinqaWf1v7vlegH/nxwsCmVuZHN0cmVhbQplbmRvYmoKMTcgMCBvYmoKPDwgL0Jhc2VGb250IC9EZWphVnVTYW5zLU9ibGlxdWUgL0NoYXJQcm9jcyAxOCAwIFIKL0VuY29kaW5nIDw8IC9EaWZmZXJlbmNlcyBbIDEyMCAveCBdIC9UeXBlIC9FbmNvZGluZyA+PiAvRmlyc3RDaGFyIDAKL0ZvbnRCQm94IFsgLTEwMTYgLTM1MSAxNjYwIDEwNjggXSAvRm9udERlc2NyaXB0b3IgMTYgMCBSCi9Gb250TWF0cml4IFsgMC4wMDEgMCAwIDAuMDAxIDAgMCBdIC9MYXN0Q2hhciAyNTUgL05hbWUgL0RlamFWdVNhbnMtT2JsaXF1ZQovU3VidHlwZSAvVHlwZTMgL1R5cGUgL0ZvbnQgL1dpZHRocyAxNSAwIFIgPj4KZW5kb2JqCjE2IDAgb2JqCjw8IC9Bc2NlbnQgOTI5IC9DYXBIZWlnaHQgMCAvRGVzY2VudCAtMjM2IC9GbGFncyA5NgovRm9udEJCb3ggWyAtMTAxNiAtMzUxIDE2NjAgMTA2OCBdIC9Gb250TmFtZSAvRGVqYVZ1U2Fucy1PYmxpcXVlCi9JdGFsaWNBbmdsZSAwIC9NYXhXaWR0aCAxMzUwIC9TdGVtViAwIC9UeXBlIC9Gb250RGVzY3JpcHRvciAvWEhlaWdodCAwID4+CmVuZG9iagoxNSAwIG9iagpbIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwCjYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgMzE4IDQwMSA0NjAgODM4IDYzNgo5NTAgNzgwIDI3NSAzOTAgMzkwIDUwMCA4MzggMzE4IDM2MSAzMTggMzM3IDYzNiA2MzYgNjM2IDYzNiA2MzYgNjM2IDYzNiA2MzYKNjM2IDYzNiAzMzcgMzM3IDgzOCA4MzggODM4IDUzMSAxMDAwIDY4NCA2ODYgNjk4IDc3MCA2MzIgNTc1IDc3NSA3NTIgMjk1CjI5NSA2NTYgNTU3IDg2MyA3NDggNzg3IDYwMyA3ODcgNjk1IDYzNSA2MTEgNzMyIDY4NCA5ODkgNjg1IDYxMSA2ODUgMzkwIDMzNwozOTAgODM4IDUwMCA1MDAgNjEzIDYzNSA1NTAgNjM1IDYxNSAzNTIgNjM1IDYzNCAyNzggMjc4IDU3OSAyNzggOTc0IDYzNCA2MTIKNjM1IDYzNSA0MTEgNTIxIDM5MiA2MzQgNTkyIDgxOCA1OTIgNTkyIDUyNSA2MzYgMzM3IDYzNiA4MzggNjAwIDYzNiA2MDAgMzE4CjM1MiA1MTggMTAwMCA1MDAgNTAwIDUwMCAxMzUwIDYzNSA0MDAgMTA3MCA2MDAgNjg1IDYwMCA2MDAgMzE4IDMxOCA1MTggNTE4CjU5MCA1MDAgMTAwMCA1MDAgMTAwMCA1MjEgNDAwIDEwMjggNjAwIDUyNSA2MTEgMzE4IDQwMSA2MzYgNjM2IDYzNiA2MzYgMzM3CjUwMCA1MDAgMTAwMCA0NzEgNjE3IDgzOCAzNjEgMTAwMCA1MDAgNTAwIDgzOCA0MDEgNDAxIDUwMCA2MzYgNjM2IDMxOCA1MDAKNDAxIDQ3MSA2MTcgOTY5IDk2OSA5NjkgNTMxIDY4NCA2ODQgNjg0IDY4NCA2ODQgNjg0IDk3NCA2OTggNjMyIDYzMiA2MzIgNjMyCjI5NSAyOTUgMjk1IDI5NSA3NzUgNzQ4IDc4NyA3ODcgNzg3IDc4NyA3ODcgODM4IDc4NyA3MzIgNzMyIDczMiA3MzIgNjExIDYwOAo2MzAgNjEzIDYxMyA2MTMgNjEzIDYxMyA2MTMgOTk1IDU1MCA2MTUgNjE1IDYxNSA2MTUgMjc4IDI3OCAyNzggMjc4IDYxMiA2MzQKNjEyIDYxMiA2MTIgNjEyIDYxMiA4MzggNjEyIDYzNCA2MzQgNjM0IDYzNCA1OTIgNjM1IDU5MiBdCmVuZG9iagoxOCAwIG9iago8PCAveCAxOSAwIFIgPj4KZW5kb2JqCjI0IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMjM1ID4+CnN0cmVhbQp4nDVRSW4AMQi75xX+QKWwJ++Zquqh/f+1hlEvAwPY2CTvwUYkPsSQ7ihXfMrqNMvwO1nkxc9K4eS9iAqkKsIKaQfPclYzDJ4bmQKXM/FZZj6ZFjsWUE3EcXbkNINBiGlcR8vpMNM86Am5PhhxY6dZrmJI691Svb7X8p8qykfW3Sy3TtnUSt2iZ+xJXHZeT21pXxh1FDcFkQ4fO7wH+SLmLC46kW72mymHlaQhOC2AH4mhVM8OrxEmfmYkeMqeTu+jNLz2QdP1vXtBR24mZCq3UEYqnqw0xoyh+o1oJqnv/4Ge9b2+/gBDTVS5CmVuZHN0cmVhbQplbmRvYmoKMjUgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAxNjQgPj4Kc3RyZWFtCnicPZDBEUMhCETvVrElgIBAPclkcvi//2tAk1xkHWD3qTuBkFGHM8Nn4smD07E0cG8VjGsIryP0CE0Ck8DEwZp4DAsBp2GRYy7fVZZVp5Wumo2e171jQdVplzUNbdqB8q2PP8I13qPwGuweQgexKHRuZVoLmVg8a5w7zKPM535O23c9GK2m1Kw3ctnXPTrL1FBeWvuEzmi0/SfXL7sxXh+FFDkICmVuZHN0cmVhbQplbmRvYmoKMjYgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAzMDcgPj4Kc3RyZWFtCnicPZJLbgMxDEP3PoUuEMD62Z7zpCi6mN5/2ycl6Yoc2RZFapa6TFlTHpA0k4R/6fBwsZ3yO2zPZmbgWqKXieWU59AVYu6ifNnMRl1ZJ8XqhGY6t+hRORcHNk2qn6sspd0ueA7XJp5b9hE/vNCgHtQ1Lgk3dFejZSk0Y6r7f9J7/Iwy4GpMXWxSq3sfPF5EVejoB0eJImOXF+fjQQnpSsJoWoiVd0UDQe7ytMp7Ce7b3mrIsgepmM47KWaw63RSLm4XhyEeyPKo8OWj2GtCz/iwKyX0SNiGM3In7mjG5tTI4pD+3o0ES4+uaCHz4K9u1i5gvFM6RWJkTnKsaYtVTvdQFNO5w70MEPVsRUMpc5HV6l/DzgtrlmwWeEr6BR6j3SZLDlbZ26hO76082dD3H1rXdB8KZW5kc3RyZWFtCmVuZG9iagoyNyAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDI0OSA+PgpzdHJlYW0KeJw9UDuORCEM6zmFL/Ak8iNwHkarLWbv364DmilQTH62MyTQEYFHDDGUr+MlraCugb+LQvFu4uuDwiCrQ1IgznoPiHTspjaREzodnDM/YTdjjsBFMQac6XSmPQcmOfvCCoRzG2XsVkgniaoijuozjimeKnufeBYs7cg2WyeSPeQg4VJSicmln5TKP23KlAo6ZtEELBK54GQTTTjLu0lSjBmUMuoepnYifaw8yKM66GRNzqwjmdnTT9uZ+Bxwt1/aZE6Vx3QezPictM6DORW69+OJNgdNjdro7PcTaSovUrsdWp1+dRKV3RjnGBKXZ38Z32T/+Qf+h1oiCmVuZHN0cmVhbQplbmRvYmoKMjggMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAzOTUgPj4Kc3RyZWFtCnicPVJLbsVACNvnFFyg0vCbz3lSVd28+29rQ1KpKryJMcYwfcqQueVLXRJxhcm3Xq5bPKZ8LltamXmIu4uNJT623JfuIbZddC6xOB1H8gsynSpEqM2q0aH4QpaFB5BO8KELwn05/uMvgMHXsA244T0yQbAk5ilCxm5RGZoSQRFh55EVqKRQn1nC31Hu6/cyBWpvjKULYxz0CbQFQm1IxALqQABE7JRUrZCOZyQTvxXdZ2IcYOfRsgGuGVRElnvsx4ipzqiMvETEPk9N+iiWTC1Wxm5TGV/8lIzUfHQFKqk08pTy0FWz0AtYiXkS9jn8SPjn1mwhhjpu1vKJ5R8zxTISzmBLOWChl+NH4NtZdRGuHbm4znSBH5XWcEy0637I9U/+dNtazXW8cgiiQOVNQfC7Dq5GscTEMj6djSl6oiywGpq8RjPBYRAR1vfDyAMa/XK8EDSnayK0WCKbtWJEjYpscz29BNZM78U51sMTwmzvndahsjMzKiGC2rqGautAdrO+83C2nz8z6KJtCmVuZHN0cmVhbQplbmRvYmoKMjkgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyNDkgPj4Kc3RyZWFtCnicTVFJigMwDLvnFfpAIV6TvKdDmUPn/9fKDoU5BAmvkpOWmFgLDzGEHyw9+JEhczf9G36i2btZepLJ2f+Y5yJTUfhSqC5iQl2IG8+hEfA9oWsSWbG98Tkso5lzvgcfhbgEM6EBY31JMrmo5pUhE04MdRwOWqTCuGtiw+Ja0TyN3G77RmZlJoQNj2RC3BiAiCDrArIYLJQ2NhMyWc4D7Q3JDVpg16kbUYuCK5TWCXSiVsSqzOCz5tZ2N0Mt8uCoffH6aFaXYIXRS/VYeF+FPpipmXbukkJ64U07IsweCqQyOy0rtXvE6m6B+j/LUvD9yff4Ha8PzfxcnAplbmRzdHJlYW0KZW5kb2JqCjMwIDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggOTQgPj4Kc3RyZWFtCnicRY3BEcAgCAT/VEEJCgraTyaTh/b/jRAyfGDnDu6EBQu2eUYfBZUmXhVYB0pj3FCPQL3hci3J3AUPcCd/2tBUnJbTd2mRSVUp3KQSef8OZyaQqHnRY533C2P7IzwKZW5kc3RyZWFtCmVuZG9iagozMSAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDQ3ID4+CnN0cmVhbQp4nDMyt1AwULA0ARKGFiYK5mYGCimGXJYQVi4XTCwHzALRlnAKIp7BlQYAuWcNJwplbmRzdHJlYW0KZW5kb2JqCjMyIDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMjU4ID4+CnN0cmVhbQp4nEWRS3IEIAhE956CI4D85DyTSmUxuf82Dc5kNnaXqP2ESiOmEiznFHkwfcnyzWS26Xc5VjsbBRRFKJjJVeixAqs7U8SZa4lq62Nl5LjTOwbFG85dOalkcaOMdVR1KnBMz5X1Ud35dlmUfUcOZQrYrHMcbODKbcMYJ0abre4O94kgTydTR8XtINnwByeNfZWrK3CdbPbRSzAOBP1CE5jki0DrDIHGzVP05BLs4+N254Fgb3kRSNkQyJEhGB2Cdp1c/+LW+b3/cYY7z7UZrhzv4neY1nbHX2KSFXMBi9wpqOdrLlrXGTrekzPH5Kb7hs65YJe7g0zv+T/Wz/r+Ax4pZvoKZW5kc3RyZWFtCmVuZG9iagozMyAwIG9iago8PCAvQkJveCBbIC0xMDIxIC00NjMgMTc5NCAxMjMzIF0gL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAzOQovU3VidHlwZSAvRm9ybSAvVHlwZSAvWE9iamVjdCA+PgpzdHJlYW0KeJzjMjQwUzA2NVXI5TI3NgKzcsAsI3MjIAski2BBZDO40gAV8wp8CmVuZHN0cmVhbQplbmRvYmoKMzQgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCA4MyA+PgpzdHJlYW0KeJxFjLsNwDAIRHumYAR+JvY+UZTC3r8NECVuuCfdPVwdCZkpbjPDQwaeDCyGXXGB9JYwC1xHUI6d7KNh1b7qBI31plLz7w+Unuys4obrAQJCGmYKZW5kc3RyZWFtCmVuZG9iagozNSAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDIzOSA+PgpzdHJlYW0KeJxNUMltBDEM+7sKNTDA6By7HgeLPLL9f0PKCZKXaEviofKUW5bKZfcjOW/JuuVDh06VafJu0M2vsf6jDAJ2/1BUEK0lsUrMXNJusTRJL9nDOI2Xa7WO56l7hFmjePDj2NMpgek9MsFms705MKs9zg6QTrjGr+rTO5UkA4m6kPNCpQrrHtQloo8r25hSnU4t5RiXn+h7fI4APcXejdzRx8sXjEa1LajRapU4DzATU9GVcauRgZQTBkNnR1c0C6XIynpCNcKNOaGZvcNwYAPLs4Skpa1SvA9lAegCXdo64zRKgo4Awt8ojPX6Bqr8XjcKZW5kc3RyZWFtCmVuZG9iagozNiAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDUxID4+CnN0cmVhbQp4nDM2tFAwUDA0MAeSRoZAlpGJQoohF0gAxMzlggnmgFkGQBqiOAeuJocrgysNAOG0DZgKZW5kc3RyZWFtCmVuZG9iagozNyAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDMzNCA+PgpzdHJlYW0KeJwtUktyxSAM23MKXaAz+AfkPOl0uni9/7aSk0VGDmD0MeWGiUp8WSC3o9bEt43MQIXhr6vMhc9I28g6iMuQi7iSLYV7RCzkMcQ8xILvq/EeHvmszMmzB8Yv2XcPK/bUhGUh48UZ2mEVx2EV5FiwdSGqe3hTpMOpJNjji/8+xXMtBC18RtCAX+Sfr47g+ZIWafeYbdOuerBMO6qksBxsT3NeJl9aZ7k6Hs8Hyfau2BFSuwIUhbkzznPhKNNWRrQWdjZIalxsb479WErQhW5cRoojkJ+pIjygpMnMJgrij5wecioDYeqarnRyG1Vxp57MNZuLtzNJZuu+SLGZwnldOLP+DFNmtXknz3Ki1KkI77FnS9DQOa6evZZZaHSbE7ykhM/GTk9Ovlcz6yE5FQmpYlpXwWkUmWIJ2xJfU1FTmnoZ/vvy7vE7fv4BLHN8cwplbmRzdHJlYW0KZW5kb2JqCjM4IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggNzAgPj4Kc3RyZWFtCnicMzM2UzBQsDACEqamhgrmRpYKKYZcQD6IlcsFE8sBs8wszIEsIwuQlhwuQwtjMG1ibKRgZmIGZFkgMSC6MrjSAJiaEwMKZW5kc3RyZWFtCmVuZG9iagozOSAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDMyMCA+PgpzdHJlYW0KeJw1UktuBTEI288puECl8E/O86qqi777b2sTvRVMMGDjKS9Z0ku+1CXbpcPkWx/3JbFC3o/tmsxSxfcWsxTPLa9HzxG3LQoEURM9WJkvFSLUz/ToOqhwSp+BVwi3FBu8g0kAg2r4Bx6lMyBQ50DGu2IyUgOCJNhzaXEIiXImiX+kvJ7fJ62kofQ9WZnL35NLpdAdTU7oAcXKxUmgXUn5oJmYSkSSl+t9sUL0hsCSPD5HMcmA7DaJbaIFJucepSXMxBQ6sMcCvGaa1VXoYMIehymMVwuzqB5s8lsTlaQdreMZ2TDeyzBTYqHhsAXU5mJlgu7l4zWvwojtUZNdw3Duls13CNFo/hsWyuBjFZKAR6exEg1pOMCIwJ5eOMVe8xM5DsCIY52aLAxjaCaneo6JwNCes6VhxsceWvXzD1TpfIcKZW5kc3RyZWFtCmVuZG9iago0MCAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDE4ID4+CnN0cmVhbQp4nDM2tFAwgMMUQ640AB3mA1IKZW5kc3RyZWFtCmVuZG9iago0MSAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDEzMyA+PgpzdHJlYW0KeJxFj0sOBCEIRPecoo7Axx/ncTLphXP/7YCdbhNjPYVUgbmCoT0uawOdFR8hGbbxt6mWjkVZPlR6UlYPyeCHrMbLIdygLPCCSSqGIVCLmBqRLWVut4DbNg2yspVTpY6wi6Mwj/a0bBUeX6JbInWSP4PEKi/c47odyKXWu96ii75/pAExCQplbmRzdHJlYW0KZW5kb2JqCjQyIDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMjUxID4+CnN0cmVhbQp4nC1RSXIDQQi7zyv0hGan32OXK4fk/9cIygcGDYtAdFrioIyfICxXvOWRq2jD3zMxgt8Fh34r121Y5EBUIEljUDWhdvF69B7YcZgJzJPWsAxmrA/8jCnc6MXhMRlnt9dl1BDsXa89mUHJrFzEJRMXTNVhI2cOP5kyLrRzPTcg50ZYl2GQblYaMxKONIVIIYWqm6TOBEESjK5GjTZyFPulL490hlWNqDHscy1tX89NOGvQ7Fis8uSUHl1xLicXL6wc9PU2AxdRaazyQEjA/W4P9XOyk994S+fOFtPje83J8sJUYMWb125ANtXi37yI4/uMr+fn+fwDX2BbiAplbmRzdHJlYW0KZW5kb2JqCjQzIDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMjE1ID4+CnN0cmVhbQp4nDVROQ4DIQzs9xX+QCSML3hPoijN/r/NjNFWHsFchrSUIZnyUpOoIeVTPnqZLpy63NfMajTnlrQtc4C4trwvrZLAiWaIg8FpmLgBmjwBQ9fRqFFDFx7Q1KVTKLDcBD6Kt24P3WO1gZe2IeeJIGIoGSxBzalFExZtzyekNb9eixvel+3dyFOlxpYYgQYBVjgc1+jX8JU9TybRdBUy1Ks1yxgJE0UiPPmOptUT61o00jIS1MYRrGoDvDv9ME4AABNxywJkn0qUs+TEb7H0swZX+v4Bn0dUlgplbmRzdHJlYW0KZW5kb2JqCjIyIDAgb2JqCjw8IC9CYXNlRm9udCAvRGVqYVZ1U2FucyAvQ2hhclByb2NzIDIzIDAgUgovRW5jb2RpbmcgPDwKL0RpZmZlcmVuY2VzIFsgMzIgL3NwYWNlIDQ2IC9wZXJpb2QgNDggL3plcm8gL29uZSAvdHdvIDUyIC9mb3VyIC9maXZlIC9zaXggL3NldmVuCi9laWdodCA2NyAvQyAvRCA5NyAvYSAxMDEgL2UgMTA4IC9sIC9tIDExMiAvcCAxMTUgL3MgL3QgXQovVHlwZSAvRW5jb2RpbmcgPj4KL0ZpcnN0Q2hhciAwIC9Gb250QkJveCBbIC0xMDIxIC00NjMgMTc5NCAxMjMzIF0gL0ZvbnREZXNjcmlwdG9yIDIxIDAgUgovRm9udE1hdHJpeCBbIDAuMDAxIDAgMCAwLjAwMSAwIDAgXSAvTGFzdENoYXIgMjU1IC9OYW1lIC9EZWphVnVTYW5zCi9TdWJ0eXBlIC9UeXBlMyAvVHlwZSAvRm9udCAvV2lkdGhzIDIwIDAgUiA+PgplbmRvYmoKMjEgMCBvYmoKPDwgL0FzY2VudCA5MjkgL0NhcEhlaWdodCAwIC9EZXNjZW50IC0yMzYgL0ZsYWdzIDMyCi9Gb250QkJveCBbIC0xMDIxIC00NjMgMTc5NCAxMjMzIF0gL0ZvbnROYW1lIC9EZWphVnVTYW5zIC9JdGFsaWNBbmdsZSAwCi9NYXhXaWR0aCAxMzQyIC9TdGVtViAwIC9UeXBlIC9Gb250RGVzY3JpcHRvciAvWEhlaWdodCAwID4+CmVuZG9iagoyMCAwIG9iagpbIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwCjYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgMzE4IDQwMSA0NjAgODM4IDYzNgo5NTAgNzgwIDI3NSAzOTAgMzkwIDUwMCA4MzggMzE4IDM2MSAzMTggMzM3IDYzNiA2MzYgNjM2IDYzNiA2MzYgNjM2IDYzNiA2MzYKNjM2IDYzNiAzMzcgMzM3IDgzOCA4MzggODM4IDUzMSAxMDAwIDY4NCA2ODYgNjk4IDc3MCA2MzIgNTc1IDc3NSA3NTIgMjk1CjI5NSA2NTYgNTU3IDg2MyA3NDggNzg3IDYwMyA3ODcgNjk1IDYzNSA2MTEgNzMyIDY4NCA5ODkgNjg1IDYxMSA2ODUgMzkwIDMzNwozOTAgODM4IDUwMCA1MDAgNjEzIDYzNSA1NTAgNjM1IDYxNSAzNTIgNjM1IDYzNCAyNzggMjc4IDU3OSAyNzggOTc0IDYzNCA2MTIKNjM1IDYzNSA0MTEgNTIxIDM5MiA2MzQgNTkyIDgxOCA1OTIgNTkyIDUyNSA2MzYgMzM3IDYzNiA4MzggNjAwIDYzNiA2MDAgMzE4CjM1MiA1MTggMTAwMCA1MDAgNTAwIDUwMCAxMzQyIDYzNSA0MDAgMTA3MCA2MDAgNjg1IDYwMCA2MDAgMzE4IDMxOCA1MTggNTE4CjU5MCA1MDAgMTAwMCA1MDAgMTAwMCA1MjEgNDAwIDEwMjMgNjAwIDUyNSA2MTEgMzE4IDQwMSA2MzYgNjM2IDYzNiA2MzYgMzM3CjUwMCA1MDAgMTAwMCA0NzEgNjEyIDgzOCAzNjEgMTAwMCA1MDAgNTAwIDgzOCA0MDEgNDAxIDUwMCA2MzYgNjM2IDMxOCA1MDAKNDAxIDQ3MSA2MTIgOTY5IDk2OSA5NjkgNTMxIDY4NCA2ODQgNjg0IDY4NCA2ODQgNjg0IDk3NCA2OTggNjMyIDYzMiA2MzIgNjMyCjI5NSAyOTUgMjk1IDI5NSA3NzUgNzQ4IDc4NyA3ODcgNzg3IDc4NyA3ODcgODM4IDc4NyA3MzIgNzMyIDczMiA3MzIgNjExIDYwNQo2MzAgNjEzIDYxMyA2MTMgNjEzIDYxMyA2MTMgOTgyIDU1MCA2MTUgNjE1IDYxNSA2MTUgMjc4IDI3OCAyNzggMjc4IDYxMiA2MzQKNjEyIDYxMiA2MTIgNjEyIDYxMiA4MzggNjEyIDYzNCA2MzQgNjM0IDYzNCA1OTIgNjM1IDU5MiBdCmVuZG9iagoyMyAwIG9iago8PCAvQyAyNCAwIFIgL0QgMjUgMCBSIC9hIDI2IDAgUiAvZSAyNyAwIFIgL2VpZ2h0IDI4IDAgUiAvZml2ZSAyOSAwIFIKL2ZvdXIgMzAgMCBSIC9sIDMxIDAgUiAvbSAzMiAwIFIgL29uZSAzNCAwIFIgL3AgMzUgMCBSIC9wZXJpb2QgMzYgMCBSCi9zIDM3IDAgUiAvc2V2ZW4gMzggMCBSIC9zaXggMzkgMCBSIC9zcGFjZSA0MCAwIFIgL3QgNDEgMCBSIC90d28gNDIgMCBSCi96ZXJvIDQzIDAgUiA+PgplbmRvYmoKMyAwIG9iago8PCAvRjEgMjIgMCBSIC9GMiAxNyAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0ExIDw8IC9DQSAwIC9UeXBlIC9FeHRHU3RhdGUgL2NhIDEgPj4KL0EyIDw8IC9DQSAxIC9UeXBlIC9FeHRHU3RhdGUgL2NhIDEgPj4KL0EzIDw8IC9DQSAwLjggL1R5cGUgL0V4dEdTdGF0ZSAvY2EgMC44ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8ID4+CmVuZG9iago2IDAgb2JqCjw8ID4+CmVuZG9iago3IDAgb2JqCjw8IC9GMS1EZWphVnVTYW5zLW1pbnVzIDMzIDAgUiAvTTAgMTMgMCBSIC9NMSAxNCAwIFIgPj4KZW5kb2JqCjEzIDAgb2JqCjw8IC9CQm94IFsgLTggLTggOCA4IF0gL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAxMzEgL1N1YnR5cGUgL0Zvcm0KL1R5cGUgL1hPYmplY3QgPj4Kc3RyZWFtCnicbZBBDoQgDEX3PUUv8ElLRWXr0mu4mUzi/bcDcUBM3TTQvjx+Uf6S8E6lwPgkCUtOs+R605DSukyMGObVsijHoFEt1s51OKjP0HBjdIuxFKbU1uh4o5vpNt6TP/qwWSFGPxwOr4R7FkMmXCkxBoffCy/bw/8Rnl7UwB+ijX5jWkP9CmVuZHN0cmVhbQplbmRvYmoKMTQgMCBvYmoKPDwgL0JCb3ggWyAtOCAtOCA4IDggXSAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDEzMSAvU3VidHlwZSAvRm9ybQovVHlwZSAvWE9iamVjdCA+PgpzdHJlYW0KeJxtkEEOhCAMRfc9RS/wSUtFZevSa7iZTOL9twNxQEzdNNC+PH5R/pLwTqXA+CQJS06z5HrTkNK6TIwY5tWyKMegUS3WznU4qM/QcGN0i7EUptTW6Hijm+k23pM/+rBZIUY/HA6vhHsWQyZcKTEGh98LL9vD/xGeXtTAH6KNfmNaQ/0KZW5kc3RyZWFtCmVuZG9iagoyIDAgb2JqCjw8IC9Db3VudCAxIC9LaWRzIFsgMTEgMCBSIF0gL1R5cGUgL1BhZ2VzID4+CmVuZG9iago0NCAwIG9iago8PCAvQ3JlYXRpb25EYXRlIChEOjIwMjEwOTE2MTQzMjU5KzAyJzAwJykKL0NyZWF0b3IgKE1hdHBsb3RsaWIgdjMuNC4zLCBodHRwczovL21hdHBsb3RsaWIub3JnKQovUHJvZHVjZXIgKE1hdHBsb3RsaWIgcGRmIGJhY2tlbmQgdjMuNC4zKSA+PgplbmRvYmoKeHJlZgowIDQ1CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAxNiAwMDAwMCBuIAowMDAwMDE1MTI1IDAwMDAwIG4gCjAwMDAwMTQzMTkgMDAwMDAgbiAKMDAwMDAxNDM2MiAwMDAwMCBuIAowMDAwMDE0NTA0IDAwMDAwIG4gCjAwMDAwMTQ1MjUgMDAwMDAgbiAKMDAwMDAxNDU0NiAwMDAwMCBuIAowMDAwMDAwMDY1IDAwMDAwIG4gCjAwMDAwMDA0MDUgMDAwMDAgbiAKMDAwMDAwNTM0NSAwMDAwMCBuIAowMDAwMDAwMjA4IDAwMDAwIG4gCjAwMDAwMDUzMjQgMDAwMDAgbiAKMDAwMDAxNDYxNyAwMDAwMCBuIAowMDAwMDE0ODcxIDAwMDAwIG4gCjAwMDAwMDYwNTYgMDAwMDAgbiAKMDAwMDAwNTg0OCAwMDAwMCBuIAowMDAwMDA1NTMyIDAwMDAwIG4gCjAwMDAwMDcxMDkgMDAwMDAgbiAKMDAwMDAwNTM2NSAwMDAwMCBuIAowMDAwMDEzMDIyIDAwMDAwIG4gCjAwMDAwMTI4MjIgMDAwMDAgbiAKMDAwMDAxMjQwNiAwMDAwMCBuIAowMDAwMDE0MDc1IDAwMDAwIG4gCjAwMDAwMDcxNDEgMDAwMDAgbiAKMDAwMDAwNzQ0OSAwMDAwMCBuIAowMDAwMDA3Njg2IDAwMDAwIG4gCjAwMDAwMDgwNjYgMDAwMDAgbiAKMDAwMDAwODM4OCAwMDAwMCBuIAowMDAwMDA4ODU2IDAwMDAwIG4gCjAwMDAwMDkxNzggMDAwMDAgbiAKMDAwMDAwOTM0NCAwMDAwMCBuIAowMDAwMDA5NDYzIDAwMDAwIG4gCjAwMDAwMDk3OTQgMDAwMDAgbiAKMDAwMDAwOTk2NiAwMDAwMCBuIAowMDAwMDEwMTIxIDAwMDAwIG4gCjAwMDAwMTA0MzMgMDAwMDAgbiAKMDAwMDAxMDU1NiAwMDAwMCBuIAowMDAwMDEwOTYzIDAwMDAwIG4gCjAwMDAwMTExMDUgMDAwMDAgbiAKMDAwMDAxMTQ5OCAwMDAwMCBuIAowMDAwMDExNTg4IDAwMDAwIG4gCjAwMDAwMTE3OTQgMDAwMDAgbiAKMDAwMDAxMjExOCAwMDAwMCBuIAowMDAwMDE1MTg1IDAwMDAwIG4gCnRyYWlsZXIKPDwgL0luZm8gNDQgMCBSIC9Sb290IDEgMCBSIC9TaXplIDQ1ID4+CnN0YXJ0eHJlZgoxNTM0MgolJUVPRgo=\n", + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2021-09-16T14:32:59.687476\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.4.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.069271, + "end_time": "2021-09-16T12:33:00.064326", + "exception": false, + "start_time": "2021-09-16T12:32:59.995055", + "status": "completed" + }, + "tags": [], + "id": "b8baab8e" + }, + "source": [ + "#### The data loader class\n", + "\n", + "The class `torch.utils.data.DataLoader` represents a Python iterable over a dataset with support for automatic batching, multi-process data loading and many more features.\n", + "The data loader communicates with the dataset using the function `__getitem__`, and stacks its outputs as tensors over the first dimension to form a batch.\n", + "In contrast to the dataset class, we usually don't have to define our own data loader class, but can just create an object of it with the dataset as input.\n", + "Additionally, we can configure our data loader with the following input arguments (only a selection, see full list [here](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)):\n", + "\n", + "* `batch_size`: Number of samples to stack per batch\n", + "* `shuffle`: If True, the data is returned in a random order.\n", + "This is important during training for introducing stochasticity.\n", + "* `num_workers`: Number of subprocesses to use for data loading.\n", + "The default, 0, means that the data will be loaded in the main process which can slow down training for datasets where loading a data point takes a considerable amount of time (e.g. large images).\n", + "More workers are recommended for those, but can cause issues on Windows computers.\n", + "For tiny datasets as ours, 0 workers are usually faster.\n", + "* `pin_memory`: If True, the data loader will copy Tensors into CUDA pinned memory before returning them.\n", + "This can save some time for large data points on GPUs.\n", + "Usually a good practice to use for a training set, but not necessarily for validation and test to save memory on the GPU.\n", + "* `drop_last`: If True, the last batch is dropped in case it is smaller than the specified batch size.\n", + "This occurs when the dataset size is not a multiple of the batch size.\n", + "Only potentially helpful during training to keep a consistent batch size.\n", + "\n", + "Let's create a simple data loader below:" + ], + "id": "b8baab8e" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:00.214548Z", + "iopub.status.busy": "2021-09-16T12:33:00.214081Z", + "iopub.status.idle": "2021-09-16T12:33:00.217620Z", + "shell.execute_reply": "2021-09-16T12:33:00.217155Z" + }, + "papermill": { + "duration": 0.094527, + "end_time": "2021-09-16T12:33:00.217739", + "exception": false, + "start_time": "2021-09-16T12:33:00.123212", + "status": "completed" + }, + "tags": [], + "id": "8641d26c" + }, + "source": [ + "data_loader = data.DataLoader(dataset, batch_size=8, shuffle=True)" + ], + "id": "8641d26c", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:00.334680Z", + "iopub.status.busy": "2021-09-16T12:33:00.334204Z", + "iopub.status.idle": "2021-09-16T12:33:00.339109Z", + "shell.execute_reply": "2021-09-16T12:33:00.338644Z" + }, + "papermill": { + "duration": 0.063837, + "end_time": "2021-09-16T12:33:00.339208", + "exception": false, + "start_time": "2021-09-16T12:33:00.275371", + "status": "completed" + }, + "tags": [], + "id": "5a119351", + "outputId": "ac0a3133-3cfd-45cd-c035-a08a14adf48e" + }, + "source": [ + "# next(iter(...)) catches the first batch of the data loader\n", + "# If shuffle is True, this will return a different batch every time we run this cell\n", + "# For iterating over the whole dataset, we can simple use \"for batch in data_loader: ...\"\n", + "data_inputs, data_labels = next(iter(data_loader))\n", + "\n", + "# The shape of the outputs are [batch_size, d_1,...,d_N] where d_1,...,d_N are the\n", + "# dimensions of the data point returned from the dataset class\n", + "print(\"Data inputs\", data_inputs.shape, \"\\n\", data_inputs)\n", + "print(\"Data labels\", data_labels.shape, \"\\n\", data_labels)" + ], + "id": "5a119351", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Data inputs torch.Size([8, 2]) \n", + " tensor([[ 1.2108, -0.1180],\n", + " [-0.1895, 0.0415],\n", + " [ 1.1542, -0.0989],\n", + " [ 1.1135, 0.1228],\n", + " [-0.0280, 0.0046],\n", + " [-0.0378, 1.0500],\n", + " [-0.0636, 0.9167],\n", + " [-0.0392, 0.8611]])\n", + "Data labels torch.Size([8]) \n", + " tensor([1, 0, 1, 1, 0, 1, 1, 1])\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.056177, + "end_time": "2021-09-16T12:33:00.451414", + "exception": false, + "start_time": "2021-09-16T12:33:00.395237", + "status": "completed" + }, + "tags": [], + "id": "e0dc65f2" + }, + "source": [ + "### Optimization\n", + "\n", + "After defining the model and the dataset, it is time to prepare the optimization of the model.\n", + "During training, we will perform the following steps:\n", + "\n", + "1. Get a batch from the data loader\n", + "2. Obtain the predictions from the model for the batch\n", + "3. Calculate the loss based on the difference between predictions and labels\n", + "4. Backpropagation: calculate the gradients for every parameter with respect to the loss\n", + "5. Update the parameters of the model in the direction of the gradients\n", + "\n", + "We have seen how we can do step 1, 2 and 4 in PyTorch. Now, we will look at step 3 and 5." + ], + "id": "e0dc65f2" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.059693, + "end_time": "2021-09-16T12:33:00.567796", + "exception": false, + "start_time": "2021-09-16T12:33:00.508103", + "status": "completed" + }, + "tags": [], + "id": "8385c05e" + }, + "source": [ + "#### Loss modules\n", + "\n", + "We can calculate the loss for a batch by simply performing a few tensor operations as those are automatically added to the computation graph.\n", + "For instance, for binary classification, we can use Binary Cross Entropy (BCE) which is defined as follows:\n", + "\n", + "$$\\mathcal{L}_{BCE} = -\\sum_i \\left[ y_i \\log x_i + (1 - y_i) \\log (1 - x_i) \\right]$$\n", + "\n", + "where $y$ are our labels, and $x$ our predictions, both in the range of $[0,1]$.\n", + "However, PyTorch already provides a list of predefined loss functions which we can use (see [here](https://pytorch.org/docs/stable/nn.html#loss-functions) for a full list).\n", + "For instance, for BCE, PyTorch has two modules: `nn.BCELoss()`, `nn.BCEWithLogitsLoss()`.\n", + "While `nn.BCELoss` expects the inputs $x$ to be in the range $[0,1]$, i.e. the output of a sigmoid, `nn.BCEWithLogitsLoss` combines a sigmoid layer and the BCE loss in a single class.\n", + "This version is numerically more stable than using a plain Sigmoid followed by a BCE loss because of the logarithms applied in the loss function.\n", + "Hence, it is adviced to use loss functions applied on \"logits\" where possible (remember to not apply a sigmoid on the output of the model in this case!).\n", + "For our model defined above, we therefore use the module `nn.BCEWithLogitsLoss`." + ], + "id": "8385c05e" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:00.684490Z", + "iopub.status.busy": "2021-09-16T12:33:00.683973Z", + "iopub.status.idle": "2021-09-16T12:33:00.686124Z", + "shell.execute_reply": "2021-09-16T12:33:00.685712Z" + }, + "papermill": { + "duration": 0.061359, + "end_time": "2021-09-16T12:33:00.686225", + "exception": false, + "start_time": "2021-09-16T12:33:00.624866", + "status": "completed" + }, + "tags": [], + "id": "72102e7a" + }, + "source": [ + "loss_module = nn.BCEWithLogitsLoss()" + ], + "id": "72102e7a", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.056071, + "end_time": "2021-09-16T12:33:00.801022", + "exception": false, + "start_time": "2021-09-16T12:33:00.744951", + "status": "completed" + }, + "tags": [], + "id": "00874cba" + }, + "source": [ + "#### Stochastic Gradient Descent\n", + "\n", + "For updating the parameters, PyTorch provides the package `torch.optim` that has most popular optimizers implemented.\n", + "We will discuss the specific optimizers and their differences later in the course, but will for now use the simplest of them: `torch.optim.SGD`.\n", + "Stochastic Gradient Descent updates parameters by multiplying the gradients with a small constant, called learning rate, and subtracting those from the parameters (hence minimizing the loss).\n", + "Therefore, we slowly move towards the direction of minimizing the loss.\n", + "A good default value of the learning rate for a small network as ours is 0.1." + ], + "id": "00874cba" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:00.917260Z", + "iopub.status.busy": "2021-09-16T12:33:00.916796Z", + "iopub.status.idle": "2021-09-16T12:33:00.918881Z", + "shell.execute_reply": "2021-09-16T12:33:00.918420Z" + }, + "papermill": { + "duration": 0.061061, + "end_time": "2021-09-16T12:33:00.918979", + "exception": false, + "start_time": "2021-09-16T12:33:00.857918", + "status": "completed" + }, + "tags": [], + "id": "3fde9ac7" + }, + "source": [ + "# Input to the optimizer are the parameters of the model: model.parameters()\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)" + ], + "id": "3fde9ac7", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.057881, + "end_time": "2021-09-16T12:33:01.033356", + "exception": false, + "start_time": "2021-09-16T12:33:00.975475", + "status": "completed" + }, + "tags": [], + "id": "cf05359f" + }, + "source": [ + "The optimizer provides two useful functions: `optimizer.step()`, and `optimizer.zero_grad()`.\n", + "The step function updates the parameters based on the gradients as explained above.\n", + "The function `optimizer.zero_grad()` sets the gradients of all parameters to zero.\n", + "While this function seems less relevant at first, it is a crucial pre-step before performing backpropagation.\n", + "If we would call the `backward` function on the loss while the parameter gradients are non-zero from the previous batch, the new gradients would actually be added to the previous ones instead of overwriting them.\n", + "This is done because a parameter might occur multiple times in a computation graph, and we need to sum the gradients in this case instead of replacing them.\n", + "Hence, remember to call `optimizer.zero_grad()` before calculating the gradients of a batch." + ], + "id": "cf05359f" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.056592, + "end_time": "2021-09-16T12:33:01.149423", + "exception": false, + "start_time": "2021-09-16T12:33:01.092831", + "status": "completed" + }, + "tags": [], + "id": "f5744461" + }, + "source": [ + "### Training\n", + "\n", + "Finally, we are ready to train our model.\n", + "As a first step, we create a slightly larger dataset and specify a data loader with a larger batch size." + ], + "id": "f5744461" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:01.267375Z", + "iopub.status.busy": "2021-09-16T12:33:01.266909Z", + "iopub.status.idle": "2021-09-16T12:33:01.268980Z", + "shell.execute_reply": "2021-09-16T12:33:01.269359Z" + }, + "papermill": { + "duration": 0.063516, + "end_time": "2021-09-16T12:33:01.269475", + "exception": false, + "start_time": "2021-09-16T12:33:01.205959", + "status": "completed" + }, + "tags": [], + "id": "d1b7f9c1" + }, + "source": [ + "train_dataset = XORDataset(size=1000)\n", + "train_data_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=True)" + ], + "id": "d1b7f9c1", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.056307, + "end_time": "2021-09-16T12:33:01.382623", + "exception": false, + "start_time": "2021-09-16T12:33:01.326316", + "status": "completed" + }, + "tags": [], + "id": "efee300f" + }, + "source": [ + "Now, we can write a small training function.\n", + "Remember our five steps: load a batch, obtain the predictions, calculate the loss, backpropagate, and update.\n", + "Additionally, we have to push all data and model parameters to the device of our choice (GPU if available).\n", + "For the tiny neural network we have, communicating the data to the GPU actually takes much more time than we could save from running the operation on GPU.\n", + "For large networks, the communication time is significantly smaller than the actual runtime making a GPU crucial in these cases.\n", + "Still, to practice, we will push the data to GPU here." + ], + "id": "efee300f" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:01.498773Z", + "iopub.status.busy": "2021-09-16T12:33:01.498306Z", + "iopub.status.idle": "2021-09-16T12:33:01.501891Z", + "shell.execute_reply": "2021-09-16T12:33:01.501403Z" + }, + "papermill": { + "duration": 0.062907, + "end_time": "2021-09-16T12:33:01.501989", + "exception": false, + "start_time": "2021-09-16T12:33:01.439082", + "status": "completed" + }, + "tags": [], + "id": "c08b3e3e", + "outputId": "aa854b75-adec-4b15-af86-b9537268b21c" + }, + "source": [ + "# Push model to device. Has to be only done once\n", + "model.to(device)" + ], + "id": "c08b3e3e", + "execution_count": null, + "outputs": [ + { + "data": { + "text/plain": [ + "SimpleClassifier(\n", + " (linear1): Linear(in_features=2, out_features=4, bias=True)\n", + " (act_fn): Tanh()\n", + " (linear2): Linear(in_features=4, out_features=1, bias=True)\n", + ")" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.058857, + "end_time": "2021-09-16T12:33:01.617784", + "exception": false, + "start_time": "2021-09-16T12:33:01.558927", + "status": "completed" + }, + "tags": [], + "id": "a07afbc8" + }, + "source": [ + "In addition, we set our model to training mode.\n", + "This is done by calling `model.train()`.\n", + "There exist certain modules that need to perform a different forward\n", + "step during training than during testing (e.g. BatchNorm and Dropout),\n", + "and we can switch between them using `model.train()` and `model.eval()`." + ], + "id": "a07afbc8" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:01.738599Z", + "iopub.status.busy": "2021-09-16T12:33:01.737571Z", + "iopub.status.idle": "2021-09-16T12:33:01.739548Z", + "shell.execute_reply": "2021-09-16T12:33:01.739099Z" + }, + "papermill": { + "duration": 0.065058, + "end_time": "2021-09-16T12:33:01.739646", + "exception": false, + "start_time": "2021-09-16T12:33:01.674588", + "status": "completed" + }, + "tags": [], + "id": "6633a5fc" + }, + "source": [ + "def train_model(model, optimizer, data_loader, loss_module, num_epochs=100):\n", + " # Set model to train mode\n", + " model.train()\n", + "\n", + " # Training loop\n", + " for epoch in tqdm(range(num_epochs)):\n", + " for data_inputs, data_labels in data_loader:\n", + "\n", + " # Step 1: Move input data to device (only strictly necessary if we use GPU)\n", + " data_inputs = data_inputs.to(device)\n", + " data_labels = data_labels.to(device)\n", + "\n", + " # Step 2: Run the model on the input data\n", + " preds = model(data_inputs)\n", + " preds = preds.squeeze(dim=1) # Output is [Batch size, 1], but we want [Batch size]\n", + "\n", + " # Step 3: Calculate the loss\n", + " loss = loss_module(preds, data_labels.float())\n", + "\n", + " # Step 4: Perform backpropagation\n", + " # Before calculating the gradients, we need to ensure that they are all zero.\n", + " # The gradients would not be overwritten, but actually added to the existing ones.\n", + " optimizer.zero_grad()\n", + " # Perform backpropagation\n", + " loss.backward()\n", + "\n", + " # Step 5: Update the parameters\n", + " optimizer.step()" + ], + "id": "6633a5fc", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:01.857296Z", + "iopub.status.busy": "2021-09-16T12:33:01.856831Z", + "iopub.status.idle": "2021-09-16T12:33:03.289262Z", + "shell.execute_reply": "2021-09-16T12:33:03.288844Z" + }, + "papermill": { + "duration": 1.492924, + "end_time": "2021-09-16T12:33:03.289376", + "exception": false, + "start_time": "2021-09-16T12:33:01.796452", + "status": "completed" + }, + "tags": [], + "id": "34a34a0a", + "outputId": "d9e6e9f5-e86a-4164-f565-4ea4f92c5bb4" + }, + "source": [ + "train_model(model, optimizer, train_data_loader, loss_module)" + ], + "id": "34a34a0a", + "execution_count": null, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8bfec2a0ce3e474ebac953f8e7415f83", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/100 [00:00 Don't drop the last batch although it is smaller than 128\n", + "test_data_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False, drop_last=False)" + ], + "id": "bb9d1b00", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.057647, + "end_time": "2021-09-16T12:33:04.501159", + "exception": false, + "start_time": "2021-09-16T12:33:04.443512", + "status": "completed" + }, + "tags": [], + "id": "3d9bd3db" + }, + "source": [ + "As metric, we will use accuracy which is calculated as follows:\n", + "\n", + "$$acc = \\frac{\\#\\text{correct predictions}}{\\#\\text{all predictions}} = \\frac{TP+TN}{TP+TN+FP+FN}$$\n", + "\n", + "where TP are the true positives, TN true negatives, FP false positives, and FN the fale negatives.\n", + "\n", + "When evaluating the model, we don't need to keep track of the computation graph as we don't intend to calculate the gradients.\n", + "This reduces the required memory and speed up the model.\n", + "In PyTorch, we can deactivate the computation graph using `with torch.no_grad(): ...`.\n", + "Remember to additionally set the model to eval mode." + ], + "id": "3d9bd3db" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:04.623119Z", + "iopub.status.busy": "2021-09-16T12:33:04.622644Z", + "iopub.status.idle": "2021-09-16T12:33:04.624728Z", + "shell.execute_reply": "2021-09-16T12:33:04.624331Z" + }, + "papermill": { + "duration": 0.065533, + "end_time": "2021-09-16T12:33:04.624827", + "exception": false, + "start_time": "2021-09-16T12:33:04.559294", + "status": "completed" + }, + "tags": [], + "id": "ee3254a7" + }, + "source": [ + "def eval_model(model, data_loader):\n", + " model.eval() # Set model to eval mode\n", + " true_preds, num_preds = 0.0, 0.0\n", + "\n", + " with torch.no_grad(): # Deactivate gradients for the following code\n", + " for data_inputs, data_labels in data_loader:\n", + "\n", + " # Determine prediction of model on dev set\n", + " data_inputs, data_labels = data_inputs.to(device), data_labels.to(device)\n", + " preds = model(data_inputs)\n", + " preds = preds.squeeze(dim=1)\n", + " preds = torch.sigmoid(preds) # Sigmoid to map predictions between 0 and 1\n", + " pred_labels = (preds >= 0.5).long() # Binarize predictions to 0 and 1\n", + "\n", + " # Keep records of predictions for the accuracy metric (true_preds=TP+TN, num_preds=TP+TN+FP+FN)\n", + " true_preds += (pred_labels == data_labels).sum()\n", + " num_preds += data_labels.shape[0]\n", + "\n", + " acc = true_preds / num_preds\n", + " print(f\"Accuracy of the model: {100.0*acc:4.2f}%\")" + ], + "id": "ee3254a7", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:04.746517Z", + "iopub.status.busy": "2021-09-16T12:33:04.746051Z", + "iopub.status.idle": "2021-09-16T12:33:04.754165Z", + "shell.execute_reply": "2021-09-16T12:33:04.754547Z" + }, + "papermill": { + "duration": 0.071433, + "end_time": "2021-09-16T12:33:04.754673", + "exception": false, + "start_time": "2021-09-16T12:33:04.683240", + "status": "completed" + }, + "tags": [], + "id": "5272400b", + "outputId": "68148616-f205-412d-d607-bb6fb4aea4a3" + }, + "source": [ + "eval_model(model, test_data_loader)" + ], + "id": "5272400b", + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy of the model: 100.00%\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.058108, + "end_time": "2021-09-16T12:33:04.871215", + "exception": false, + "start_time": "2021-09-16T12:33:04.813107", + "status": "completed" + }, + "tags": [], + "id": "70e6957c" + }, + "source": [ + "If we trained our model correctly, we should see a score close to 100% accuracy.\n", + "However, this is only possible because of our simple task, and\n", + "unfortunately, we usually don't get such high scores on test sets of\n", + "more complex tasks." + ], + "id": "70e6957c" + }, + { + "cell_type": "markdown", + "metadata": { + "lines_to_next_cell": 2, + "papermill": { + "duration": 0.057859, + "end_time": "2021-09-16T12:33:04.987289", + "exception": false, + "start_time": "2021-09-16T12:33:04.929430", + "status": "completed" + }, + "tags": [], + "id": "939b0237" + }, + "source": [ + "#### Visualizing classification boundaries\n", + "\n", + "To visualize what our model has learned, we can perform a prediction for every data point in a range of $[-0.5, 1.5]$, and visualize the predicted class as in the sample figure at the beginning of this section.\n", + "This shows where the model has created decision boundaries, and which points would be classified as $0$, and which as $1$.\n", + "We therefore get a background image out of blue (class 0) and orange (class 1).\n", + "The spots where the model is uncertain we will see a blurry overlap.\n", + "The specific code is less relevant compared to the output figure which\n", + "should hopefully show us a clear separation of classes:" + ], + "id": "939b0237" + }, + { + "cell_type": "code", + "metadata": { + "execution": { + "iopub.execute_input": "2021-09-16T12:33:05.128533Z", + "iopub.status.busy": "2021-09-16T12:33:05.120427Z", + "iopub.status.idle": "2021-09-16T12:33:05.504515Z", + "shell.execute_reply": "2021-09-16T12:33:05.504900Z" + }, + "papermill": { + "duration": 0.451291, + "end_time": "2021-09-16T12:33:05.505038", + "exception": false, + "start_time": "2021-09-16T12:33:05.053747", + "status": "completed" + }, + "tags": [], + "id": "9abcbe24", + "outputId": "88a89d9b-40e8-4e61-dbbe-f4a14c6e3347" + }, + "source": [ + "@torch.no_grad() # Decorator, same effect as \"with torch.no_grad(): ...\" over the whole function.\n", + "def visualize_classification(model, data, label):\n", + " if isinstance(data, torch.Tensor):\n", + " data = data.cpu().numpy()\n", + " if isinstance(label, torch.Tensor):\n", + " label = label.cpu().numpy()\n", + " data_0 = data[label == 0]\n", + " data_1 = data[label == 1]\n", + "\n", + " plt.figure(figsize=(4, 4))\n", + " plt.scatter(data_0[:, 0], data_0[:, 1], edgecolor=\"#333\", label=\"Class 0\")\n", + " plt.scatter(data_1[:, 0], data_1[:, 1], edgecolor=\"#333\", label=\"Class 1\")\n", + " plt.title(\"Dataset samples\")\n", + " plt.ylabel(r\"$x_2$\")\n", + " plt.xlabel(r\"$x_1$\")\n", + " plt.legend()\n", + "\n", + " # Let's make use of a lot of operations we have learned above\n", + " model.to(device)\n", + " c0 = torch.Tensor(to_rgba(\"C0\")).to(device)\n", + " c1 = torch.Tensor(to_rgba(\"C1\")).to(device)\n", + " x1 = torch.arange(-0.5, 1.5, step=0.01, device=device)\n", + " x2 = torch.arange(-0.5, 1.5, step=0.01, device=device)\n", + " xx1, xx2 = torch.meshgrid(x1, x2) # Meshgrid function as in numpy\n", + " model_inputs = torch.stack([xx1, xx2], dim=-1)\n", + " preds = model(model_inputs)\n", + " preds = torch.sigmoid(preds)\n", + " # Specifying \"None\" in a dimension creates a new one\n", + " output_image = preds * c0[None, None] + (1 - preds) * c1[None, None]\n", + " output_image = (\n", + " output_image.cpu().numpy()\n", + " ) # Convert to numpy array. This only works for tensors on CPU, hence first push to CPU\n", + " plt.imshow(output_image, origin=\"upper\", extent=(-0.5, 1.5, -0.5, 1.5))\n", + " plt.grid(False)\n", + "\n", + "\n", + "visualize_classification(model, dataset.data, dataset.label)\n", + "plt.show()" + ], + "id": "9abcbe24", + "execution_count": null, + "outputs": [ + { + "data": { + "application/pdf": "JVBERi0xLjQKJazcIKu6CjEgMCBvYmoKPDwgL1BhZ2VzIDIgMCBSIC9UeXBlIC9DYXRhbG9nID4+CmVuZG9iago4IDAgb2JqCjw8IC9FeHRHU3RhdGUgNCAwIFIgL0ZvbnQgMyAwIFIgL1BhdHRlcm4gNSAwIFIKL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0gL1NoYWRpbmcgNiAwIFIKL1hPYmplY3QgNyAwIFIgPj4KZW5kb2JqCjExIDAgb2JqCjw8IC9Bbm5vdHMgMTAgMCBSIC9Db250ZW50cyA5IDAgUgovR3JvdXAgPDwgL0NTIC9EZXZpY2VSR0IgL1MgL1RyYW5zcGFyZW5jeSAvVHlwZSAvR3JvdXAgPj4KL01lZGlhQm94IFsgMCAwIDI5MS4xMDU2MjUgMjc3LjMwODc1IF0gL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDggMCBSCi9UeXBlIC9QYWdlID4+CmVuZG9iago5IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMTIgMCBSID4+CnN0cmVhbQp4nK2bS5NcuXGF9/Ur7lJa9CWAxHM54/FMWBFejMWwFw4vGDRFa4IzNkXJ8s/3d3Cr6t6qzuqmFDNPFhoFJPJx8mQCHZefTm++icvHL0tYfuLfvy5x+WF5892H//3j+w//8sO3y/svp8D4z6c04hpDqanw8dPxY2pttdBbYTjcfPqv0+mXE6vzjR9Y+OPpVPpaIt+xtpZsTGLhVtZRjmOf9rFU8jr6Nnj57nGMHf5w+rw8WzbFtuZ8+d+fPiz/tvyyvPkm6aSRk0ZOGp6d9DNf6IvOq/8/W/T9z8ubf4rLd/+9/Hj6cfkbdw1rTLHUXjkbH3I9/9VOYW2h9J6spLH86eMu35qQ8OH3lrvvnT6jaskelxjTGuMo1mKMtoy8ppZbLNZLnaf458Ap9umhrjlEMytlDD7aain0UmIOxZmfZPoUYkWstDzFNfUeRop1dGf2U0Qd+AlTRm+aH5G8ptGD5eh9AdvnZpZGTkznzKMPZBwhebOTra1bLCHk2hedfGT+hIY8aaLlteWYK7MQXmeNLVtpobnzn2TjkXoKJY+lrDUmk2pGdo8aOts3FDEwEZ9TWMswq7mG6J1VlioJYXpro/KxryPmgvsRR/4OZQ1pIEwybRDLWpAfF0nV3A3GGou0hxw6fsLUNfQ+SvSthTpRD8tb2bQ7LBsHYRNvfkYlRAH2DLbY2gOR2WNpnqNN/YTBSojDdPQTCeXQQ461uRpl+5xzz6Wjfma3NKxnjOc6cmprCHhDjmG0peH30WK23KM3PeE4aCZ1qR49Vr5VcnCXfkoYKhhOklJeUKrhy00OPXyfT+iN+JCvFH2WY2S8r1n1DDv9so0UUjMME22soeHDGK26eq84SgkjWo+SLsqNcmuphgeax/C1d0t4Z16egO/UYsoNpPbk6QNMi+hSyIYXGV7HdgHfc8WJIa4htlCLhawgxxTEb7FcLLoHDmE1NAp64f0zEDqGxmAEm69SoZ+saehJOwa8rdrAgMV1/dTXnCooVQo6lwZGaQKiaq6OgL2BoxlAk7X+inRJHtLdSMkr1u8SAFeOmdkZdQYi1xe/MB9nb7luh+lsg7WKtzjaN/7uNUbNHkhOrh1CxQdBm20kQBzdGb4g02bw27PsCqyDdliq4zjk0ppIFEaMu/ix8pNW+Sco/5AiTOH9AP8Erj3VRvj1BZWAw2So5PokCsGWg3jIqc8zhGAoZJgLBQ0g48d4hyIoGlnCiD4gpHurW8D/UFjAY8pUIQkBmCndPScWIaDAXgLWBgfFbwAxDv0gp9lKfiRFESFDn1F6ncBfgpt3iA8jMwSFXF4IP1x5KCW67gJcAAN4L3AE4YFOjFgrJn3gWxXdEEKhhZkVACsjfG1EXM51L/wpGNk+VTky9icBlVa8dF/XNmT1SnolfZD3hdjmhVyvmDSQIwHWpRPRSV4GD3HFxiRlTBVWcSdUDqEZcEdX5QPMDThUStCNDWBkTcW/SwwA83UEBBfEidXg+ECFVWGID2EmgCGRARnSYsgyGRtyAteBlbyxYIcHC7Lj2msjaGEW/gbyeIK0JGC0bWbijwVUim4ylkSdKG4CAls6TsHqtYTiRsiQbZReKywuCe3IZ/hE9VOU5IW0sUHNW8bC35MSYvWpSgZkSNWE1BmwyZZtNJKCm11NfqPYrzNCMvmzkpnBMV87wDXHDJVNUtrMYSBDyqUObwPwIBdATv4FkFUAGOmT9e55sXhkakkph2yBcNVCwSLNHiEZNL7As2Bj+jKCwfqqHyIIotPFHiGHizgamKbs3Lu7OsKShsIgQG3mcuPYAjYXDCCRgLPyKNly4bsAQzDs6qcxKAGa54eE3iRZQmWZFS5YH9GmJDeGRGKbTiY3mA0R7/JzUTjSeg7CYcg61iFUkvT0gHgUvHD0OlreiAJ2Igj5vis/1VQvkBNAib16biBKSi6DU5JucDJCluy7KA1OyOwAvzNdqA5pgtwSoNP7WbY9KIrwb3I7HllR97RpxgCdKukBmIHkgGhVHn2aRNTyxLdHWFMCcc3ugFknOCoEThr3IztXNIxFTXjcKQCI0uwGEsQOlCgkShI25sHnh0HbXBSYtDAqrOGZ8gSjmMtdmDFcwdE3VIGj4SJsRaUFPI6QH+QlmQcKlvFBOSbSiPLNdPOgWJmlDf7d2uTEaWK8dPmA9+AqkDBQ2qbWTUGFp/lZVYeFYSABeQ+GUGrIFEbxAWcm8SYj8EhJEyL5zAciBZR18ZpIo97Iqhexuz7GBgJiaZejPKnyhrdZH/qvdhgTGMCO6AINgVpVH5KXEo4g7UPmqG/c1VG3GYZUBsPUyMZpWNglkYO4hpGAp2Fs6ZLcFFV7mScJyZvKAy4jMg7AJNToR9KT6n80TY3ObKouviLiCya4gUSUYnIq8pK3nMmXI0X0cGvKzWOpmiAZM6eS5eFDQWzGFQYNJ6KHkrLIZbAwCSwopNz4KCtBERs1dl0G7hblLkHlpSc7Ya8Mp6SksIaGdfEfKlD3rJwN/jMpTEGJUib1lmtMQoGwAN7Qg7oShHUknRYXF4FQ8gmhJoWTXihx+Rrmd0s9iBHn7BgTx1ClSuEHcYIEuTycQjZkAoDUr4iiDCXGqfp8dUNNM7urhG8iVYY0uOyganJZoWgABQAJRaxQ+ldhC9+uN9P/jg6c8D+r6zQSIp8o74AauEwlZT9ru91OXu4mH3pt2goGh4FhOmQVSCPxDjGycklv8dajREOrGCA+AZ6ShFAGaWM4s0krqbKvUsVE0FV8i0hK5i2+1SFRwUYCJSFCHQZmB47MlwY2QR0CLsBKJ8SZaBTBcCnz79dvcAsSZz9TalXAUB1Kd2f6EMtqQh8o3NkXiLQAYbgY83b5Cv5TBKhb2WY2ELgD2fXS27pdPgop+KF0KdmBW7JeLtATT5wn8BsCQLCZ+ncqfeNQydGbqxsYQoawqlNWwDtmdZvfdjWD31YVCih7bPVepArH0OpiPv+CjY3tVrUcZl7lE3YW0feWL5pA8ZgnzCG79aZWVzLvqLgVgNUxLSRKZGQQo41c6lu1StEAEelc+IxhrQj2XI/BwTENzLZPl0yqq1rhlNfq9G51Ux1I1oLtLNqK5Ctykr2DEkxAFQyEWFNruYqzI0xxDTrgllaDCl8sDvpawqCgqHmSQGwh6CAza7etvsN5xPKTtzpsBapTKAlLmiQd/cOh7dogvPNdkewCNxiEppoJZCBVJG5Y49oISeFSTda3lZhVs4/k4UxH5aUBNBAjyGUSZpCZCY3gmxPeRLhDPgceuXQKMViREmL35qsnzM+pAiiulbWopNKEuuTqZevEwXIqiEjgoXYqbEpHMVIvTMuqpjCGVKGoHGlqtlO6XUjU80giQ1MMyd/1dWUynKsNz6qCOAhWCmryUjwkMjAEqvlnDWoihtjhgLOJSAJLRNS5OnHNihwlZ5lSe0FIZvfDDeqnDoDhkJQjBNQyJkOjWpc6XYRJK2yrqNZpY6MP8BjKgJb9UIpq7DTM2QecD6rU1aLE533USKqHCL6x0WMClXiiinHtNCNZNxdUrbFszfqW1eKODxxtci9V1L1wPqUOii9qjetVwb30ZVV3GMKDNqffVFHISMh4hjVlF6p0EDJPLKBqIBuk5MYIdShe07K6GYIcopXpwc2TQ9cs4BWVy0QCSgHEaPr0QPFKFpQtitcqZk9lUNWNRBxP9tkwDwDC6EV3XmxIPBI01roPCCqLgQ9oDtWJOFmgdGAHX/Vk7SEe0OeFnRhbJG3W3FzF4yZoBugTQuMGDUoj8u6j5FjlsVL9bOSDkjnNzpk9wHeKmMkf8yxEum6kyB2qYNy0WhVDjSAscRo5qhWgms2bDXhleW/oaqrN6y9wLIP62UUPoYFSXe9NPjwbobgBHOEBodHdJ5rUF3QBR64kP3Xd9DTPzbJuNQYZJs78BLQV3fO0MbxkiS6QBu474izvQJ/EVPgWiPUAbah2BNYhz4CFK5Ofhu80Xbc+Im4F353XuFXJM1DSuCnK1GYAOTpFSZ/VR8YtIDTRgz6ZhvIP8mbqJIi3z6tGVxZFnKl73+YtrHjcRO3hZnmVhKKBIGtXU0hAAyOA12ZvcRIxBUmGORLbukaJqn2Kj6kAgbodGF0OA8vSxYgNi567TEiC1bV8bvniXsoNZEM/9UHbiJtEWGcgd95yUe2RPnrx3EXztXYGh7dyltRM3ufUzYUxdbRmv8Zm+MV545LUsQqeg83bbwiKwZeUQSzKIfUkwMcNQTQWSmqyzGKZY5M7os8inwaMug4IU+J0MRMqKQM1ez/8FlVVaII0VAOz40v+MPIqdVzw/VE8wjCBZQvyX4KlFNy3+O7O+jWTHyk50ljgfpQeUKh0LT1vVS8+HoE5MsH5Oo+4yOqburx2FjNgaJMsAmw0mkX3gy97igCHdV2HzFoIMkktASe/XqTdKgdKRhmmFm6Rw+t7TXN9TIUBifEkoeKTOqgiFLW7tcckNRm3JRtVwUKXZIk8OZrn9JgJgFGqx4zqKOEscA7z54In4J2eVEwqzwmr+mE+fnV1jXQvNgS/Qy16OJxyvOdfFUKOc5HT0fWmUl1Z8PdwqV5WNFDyZN1V6f2LmneUFr67kNSDzMOMpraTke1Z2ad5Mn+u21uDoIcY6viD1fMK37dQw95qaUHgOTWGV0Pf5e9bjYSYeG88PwuhMI6Qk0c5FWCBkDTdcPbZSITPqla53kXfU6s1YKIgMKp6hqGefJtPOBwQ2LqUXU3+2ZBQv4Uql0ByIUY9hCAS3Df6QLimXN0qSJdKLJwGoDWvXeFNrcw7PutuJjA1B/VwoOpaTx1TAIOoa67oT72tun2WJRV1kAm9NQNidQXnaZ50AL6IAU8Hnt0lvFHhcZRn6ydd3motWzsI3nR+J8aI93btOkRNfH66dvqWzf96bhA9Ta+t2/1q2e7k2PT07dvlzfdS/fL2D6cp0Qjby7q3/3n69+U3YS2/Xf5jefu70z++PX1W92k70PUPEvz7+PTdh5/e/etffv/uly9PP//xl798Wba3aacp+GmWF+1W4n3sJZGFZiMOeyhzOIgadlGvO9c2O+c3O1/HXty5DFGkr9u5PN85CUPG3QvD69hLO6eobtVX7Ry9Mz9/x3h82/jizmgGxPu6nW/OfFQcYNqkXumvx3ReRd+F0eozK7z5Pp2X/M3//XZ5+9OpqE8hslRmgZWnF86JcWlzXpzzDuf0wgAaHG9fcDqnDCJu2sngLHnu+XWhEH6FWLhIXXVVcSv1dehlqSswmL5e6vRrRPBF6hFX1ShHqa9DrtQ4waZhNfeDK/VNCDv+fNlbbdjQbzffx17efdbJ/dXtkxPI1+2BT0rt2+2vY69sn/Xo7dXty0unbxSgd26+j72yfR2qsF7bvr1wetjnWu/8dR97Zfuhm+JXto8v2j6Jhtw53j728vZJjeLXXC++aPv9pfdh+8Pr7xe3L3GNr7levLP959PGpsQd8rqloZjVm7f2awFq2gE1Lb9bwpVn3L5Ff4Ft3Kjg9y8nnru5X/n+/jDzaINHq4Z5jsMV2vLxJjXB0vRCfCY4wWi3W6ukW6t89+7P7758+PPy5d3P//Ppw5ejgd58Y9tV3ser4s6/WVDLWV7dNmfxDdKM9XYd+XSab2wTuGWXMb1IOM/peoNTzy8Lz2MK3+X9aR/J7XrswyClHQRWT0oOGzC6Td3luA6936W9jn06Vb0WGNCkw2i1fd55k32o7ctdxjaRPx0Gruc6LL8d/k5j7/XLFN+eXvt1hftfOzgT5ftfVTi9+KsKp9bOe5sqYflWk6A9hnEda3oCrT6MBiic9CyBsWrW0xxT/QrpP3U2HoSaWlZjzXoaszB21pJuS/TQ6jAU5/rzi9cRvfNl88NaJI/S9BZ13zSb3qCqu33apWNQ7z75yuEMOW9nuJzz/Jnv6Z6o9hH3OQoqo84/roRzDT0KvNkyg+ez43uQLT87wnHovGd+pozDWled7ZselLtLd7DCfoaLre7seXamcP7dnztAGGWfbeMFfP6HT+++4ItHBHj5Yv7rbu9390t6YzjB/MYD9+HdzHoySBWRjv6Q9Coj6qHVjReq6O3ZdPd28TA9HcpNxfk+VC+bHDxRr1pN7yWOnqguo+6synFr9StyzuXGF8l/Iekm4OiLDF5Ocz332Pc+WPMw82r2w5JX/7jZfPekg5z52Xn2ocO598Grfo7rXfV42Pqg8oOQu3EOp7Hn57Z976/yUPVX415OveCl8eilP57+H5bXH+wKZW5kc3RyZWFtCmVuZG9iagoxMiAwIG9iago0NjgwCmVuZG9iagoxMCAwIG9iagpbIF0KZW5kb2JqCjIwIDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggOTUgPj4Kc3RyZWFtCnicPYxBDsAgCATvvGI/0AQRFf/TND3Y/1+7RtsLTHZhSjcoDiucVRXFG84kHz6SvcNax5CimUdDnN3cFg5LjRSrWBYWnmERpLQ1zPi8KGtgSinqaWf1v7vlegH/nxwsCmVuZHN0cmVhbQplbmRvYmoKMTggMCBvYmoKPDwgL0Jhc2VGb250IC9EZWphVnVTYW5zLU9ibGlxdWUgL0NoYXJQcm9jcyAxOSAwIFIKL0VuY29kaW5nIDw8IC9EaWZmZXJlbmNlcyBbIDEyMCAveCBdIC9UeXBlIC9FbmNvZGluZyA+PiAvRmlyc3RDaGFyIDAKL0ZvbnRCQm94IFsgLTEwMTYgLTM1MSAxNjYwIDEwNjggXSAvRm9udERlc2NyaXB0b3IgMTcgMCBSCi9Gb250TWF0cml4IFsgMC4wMDEgMCAwIDAuMDAxIDAgMCBdIC9MYXN0Q2hhciAyNTUgL05hbWUgL0RlamFWdVNhbnMtT2JsaXF1ZQovU3VidHlwZSAvVHlwZTMgL1R5cGUgL0ZvbnQgL1dpZHRocyAxNiAwIFIgPj4KZW5kb2JqCjE3IDAgb2JqCjw8IC9Bc2NlbnQgOTI5IC9DYXBIZWlnaHQgMCAvRGVzY2VudCAtMjM2IC9GbGFncyA5NgovRm9udEJCb3ggWyAtMTAxNiAtMzUxIDE2NjAgMTA2OCBdIC9Gb250TmFtZSAvRGVqYVZ1U2Fucy1PYmxpcXVlCi9JdGFsaWNBbmdsZSAwIC9NYXhXaWR0aCAxMzUwIC9TdGVtViAwIC9UeXBlIC9Gb250RGVzY3JpcHRvciAvWEhlaWdodCAwID4+CmVuZG9iagoxNiAwIG9iagpbIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwCjYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgMzE4IDQwMSA0NjAgODM4IDYzNgo5NTAgNzgwIDI3NSAzOTAgMzkwIDUwMCA4MzggMzE4IDM2MSAzMTggMzM3IDYzNiA2MzYgNjM2IDYzNiA2MzYgNjM2IDYzNiA2MzYKNjM2IDYzNiAzMzcgMzM3IDgzOCA4MzggODM4IDUzMSAxMDAwIDY4NCA2ODYgNjk4IDc3MCA2MzIgNTc1IDc3NSA3NTIgMjk1CjI5NSA2NTYgNTU3IDg2MyA3NDggNzg3IDYwMyA3ODcgNjk1IDYzNSA2MTEgNzMyIDY4NCA5ODkgNjg1IDYxMSA2ODUgMzkwIDMzNwozOTAgODM4IDUwMCA1MDAgNjEzIDYzNSA1NTAgNjM1IDYxNSAzNTIgNjM1IDYzNCAyNzggMjc4IDU3OSAyNzggOTc0IDYzNCA2MTIKNjM1IDYzNSA0MTEgNTIxIDM5MiA2MzQgNTkyIDgxOCA1OTIgNTkyIDUyNSA2MzYgMzM3IDYzNiA4MzggNjAwIDYzNiA2MDAgMzE4CjM1MiA1MTggMTAwMCA1MDAgNTAwIDUwMCAxMzUwIDYzNSA0MDAgMTA3MCA2MDAgNjg1IDYwMCA2MDAgMzE4IDMxOCA1MTggNTE4CjU5MCA1MDAgMTAwMCA1MDAgMTAwMCA1MjEgNDAwIDEwMjggNjAwIDUyNSA2MTEgMzE4IDQwMSA2MzYgNjM2IDYzNiA2MzYgMzM3CjUwMCA1MDAgMTAwMCA0NzEgNjE3IDgzOCAzNjEgMTAwMCA1MDAgNTAwIDgzOCA0MDEgNDAxIDUwMCA2MzYgNjM2IDMxOCA1MDAKNDAxIDQ3MSA2MTcgOTY5IDk2OSA5NjkgNTMxIDY4NCA2ODQgNjg0IDY4NCA2ODQgNjg0IDk3NCA2OTggNjMyIDYzMiA2MzIgNjMyCjI5NSAyOTUgMjk1IDI5NSA3NzUgNzQ4IDc4NyA3ODcgNzg3IDc4NyA3ODcgODM4IDc4NyA3MzIgNzMyIDczMiA3MzIgNjExIDYwOAo2MzAgNjEzIDYxMyA2MTMgNjEzIDYxMyA2MTMgOTk1IDU1MCA2MTUgNjE1IDYxNSA2MTUgMjc4IDI3OCAyNzggMjc4IDYxMiA2MzQKNjEyIDYxMiA2MTIgNjEyIDYxMiA4MzggNjEyIDYzNCA2MzQgNjM0IDYzNCA1OTIgNjM1IDU5MiBdCmVuZG9iagoxOSAwIG9iago8PCAveCAyMCAwIFIgPj4KZW5kb2JqCjI1IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMjM1ID4+CnN0cmVhbQp4nDVRSW4AMQi75xX+QKWwJ++Zquqh/f+1hlEvAwPY2CTvwUYkPsSQ7ihXfMrqNMvwO1nkxc9K4eS9iAqkKsIKaQfPclYzDJ4bmQKXM/FZZj6ZFjsWUE3EcXbkNINBiGlcR8vpMNM86Am5PhhxY6dZrmJI691Svb7X8p8qykfW3Sy3TtnUSt2iZ+xJXHZeT21pXxh1FDcFkQ4fO7wH+SLmLC46kW72mymHlaQhOC2AH4mhVM8OrxEmfmYkeMqeTu+jNLz2QdP1vXtBR24mZCq3UEYqnqw0xoyh+o1oJqnv/4Ge9b2+/gBDTVS5CmVuZHN0cmVhbQplbmRvYmoKMjYgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAxNjQgPj4Kc3RyZWFtCnicPZDBEUMhCETvVrElgIBAPclkcvi//2tAk1xkHWD3qTuBkFGHM8Nn4smD07E0cG8VjGsIryP0CE0Ck8DEwZp4DAsBp2GRYy7fVZZVp5Wumo2e171jQdVplzUNbdqB8q2PP8I13qPwGuweQgexKHRuZVoLmVg8a5w7zKPM535O23c9GK2m1Kw3ctnXPTrL1FBeWvuEzmi0/SfXL7sxXh+FFDkICmVuZHN0cmVhbQplbmRvYmoKMjcgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAzMDcgPj4Kc3RyZWFtCnicPZJLbgMxDEP3PoUuEMD62Z7zpCi6mN5/2ycl6Yoc2RZFapa6TFlTHpA0k4R/6fBwsZ3yO2zPZmbgWqKXieWU59AVYu6ifNnMRl1ZJ8XqhGY6t+hRORcHNk2qn6sspd0ueA7XJp5b9hE/vNCgHtQ1Lgk3dFejZSk0Y6r7f9J7/Iwy4GpMXWxSq3sfPF5EVejoB0eJImOXF+fjQQnpSsJoWoiVd0UDQe7ytMp7Ce7b3mrIsgepmM47KWaw63RSLm4XhyEeyPKo8OWj2GtCz/iwKyX0SNiGM3In7mjG5tTI4pD+3o0ES4+uaCHz4K9u1i5gvFM6RWJkTnKsaYtVTvdQFNO5w70MEPVsRUMpc5HV6l/DzgtrlmwWeEr6BR6j3SZLDlbZ26hO76082dD3H1rXdB8KZW5kc3RyZWFtCmVuZG9iagoyOCAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDI0OSA+PgpzdHJlYW0KeJw9UDuORCEM6zmFL/Ak8iNwHkarLWbv364DmilQTH62MyTQEYFHDDGUr+MlraCugb+LQvFu4uuDwiCrQ1IgznoPiHTspjaREzodnDM/YTdjjsBFMQac6XSmPQcmOfvCCoRzG2XsVkgniaoijuozjimeKnufeBYs7cg2WyeSPeQg4VJSicmln5TKP23KlAo6ZtEELBK54GQTTTjLu0lSjBmUMuoepnYifaw8yKM66GRNzqwjmdnTT9uZ+Bxwt1/aZE6Vx3QezPictM6DORW69+OJNgdNjdro7PcTaSovUrsdWp1+dRKV3RjnGBKXZ38Z32T/+Qf+h1oiCmVuZHN0cmVhbQplbmRvYmoKMjkgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyNDkgPj4Kc3RyZWFtCnicTVFJigMwDLvnFfpAIV6TvKdDmUPn/9fKDoU5BAmvkpOWmFgLDzGEHyw9+JEhczf9G36i2btZepLJ2f+Y5yJTUfhSqC5iQl2IG8+hEfA9oWsSWbG98Tkso5lzvgcfhbgEM6EBY31JMrmo5pUhE04MdRwOWqTCuGtiw+Ja0TyN3G77RmZlJoQNj2RC3BiAiCDrArIYLJQ2NhMyWc4D7Q3JDVpg16kbUYuCK5TWCXSiVsSqzOCz5tZ2N0Mt8uCoffH6aFaXYIXRS/VYeF+FPpipmXbukkJ64U07IsweCqQyOy0rtXvE6m6B+j/LUvD9yff4Ha8PzfxcnAplbmRzdHJlYW0KZW5kb2JqCjMwIDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggNDcgPj4Kc3RyZWFtCnicMzK3UDBQsDQBEoYWJgrmZgYKKYZclhBWLhdMLAfMAtGWcAoinsGVBgC5Zw0nCmVuZHN0cmVhbQplbmRvYmoKMzEgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyNTggPj4Kc3RyZWFtCnicRZFLcgQgCET3noIjgPzkPJNKZTG5/zYNzmQ2dpeo/YRKI6YSLOcUeTB9yfLNZLbpdzlWOxsFFEUomMlV6LECqztTxJlriWrrY2XkuNM7BsUbzl05qWRxo4x1VHUqcEzPlfVR3fl2WZR9Rw5lCtiscxxs4MptwxgnRput7g73iSBPJ1NHxe0g2fAHJ419lasrcJ1s9tFLMA4E/UITmOSLQOsMgcbNU/TkEuzj43bngWBveRFI2RDIkSEYHYJ2nVz/4tb5vf9xhjvPtRmuHO/id5jWdsdfYpIVcwGL3Cmo52suWtcZOt6TM8fkpvuGzrlgl7uDTO/5P9bP+v4DHilm+gplbmRzdHJlYW0KZW5kb2JqCjMyIDAgb2JqCjw8IC9CQm94IFsgLTEwMjEgLTQ2MyAxNzk0IDEyMzMgXSAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDM5Ci9TdWJ0eXBlIC9Gb3JtIC9UeXBlIC9YT2JqZWN0ID4+CnN0cmVhbQp4nOMyNDBTMDY1VcjlMjc2ArNywCwjcyMgCySLYEFkM7jSABXzCnwKZW5kc3RyZWFtCmVuZG9iagozMyAwIG9iago8PCAvRmlsdGVyIC9GbGF0ZURlY29kZSAvTGVuZ3RoIDgzID4+CnN0cmVhbQp4nEWMuw3AMAhEe6ZgBH4m9j5RlMLevw0QJW64J909XB0JmSluM8NDBp4MLIZdcYH0ljALXEdQjp3so2HVvuoEjfWmUvPvD5Se7KzihusBAkIaZgplbmRzdHJlYW0KZW5kb2JqCjM0IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMjM5ID4+CnN0cmVhbQp4nE1QyW0EMQz7uwo1MMDoHLseB4s8sv1/Q8oJkpdoS+Kh8pRblspl9yM5b8m65UOHTpVp8m7Qza+x/qMMAnb/UFQQrSWxSsxc0m6xNEkv2cM4jZdrtY7nqXuEWaN48OPY0ymB6T0ywWazvTkwqz3ODpBOuMav6tM7lSQDibqQ80KlCuse1CWijyvbmFKdTi3lGJef6Ht8jgA9xd6N3NHHyxeMRrUtqNFqlTgPMBNT0ZVxq5GBlBMGQ2dHVzQLpcjKekI1wo05oZm9w3BgA8uzhKSlrVK8D2UB6AJd2jrjNEqCjgDC3yiM9foGqvxeNwplbmRzdHJlYW0KZW5kb2JqCjM1IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggNTEgPj4Kc3RyZWFtCnicMza0UDBQMDQwB5JGhkCWkYlCiiEXSADEzOWCCeaAWQZAGqI4B64mhyuDKw0A4bQNmAplbmRzdHJlYW0KZW5kb2JqCjM2IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMzM0ID4+CnN0cmVhbQp4nC1SS3LFIAzbcwpdoDP4B+Q86XS6eL3/tpKTRUYOYPQx5YaJSnxZILej1sS3jcxAheGvq8yFz0jbyDqIy5CLuJIthXtELOQxxDzEgu+r8R4e+azMybMHxi/Zdw8r9tSEZSHjxRnaYRXHYRXkWLB1Iap7eFOkw6kk2OOL/z7Fcy0ELXxG0IBf5J+vjuD5khZp95ht0656sEw7qqSwHGxPc14mX1pnuToezwfJ9q7YEVK7AhSFuTPOc+Eo01ZGtBZ2NkhqXGxvjv1YStCFblxGiiOQn6kiPKCkycwmCuKPnB5yKgNh6pqudHIbVXGnnsw1m4u3M0lm675IsZnCeV04s/4MU2a1eSfPcqLUqQjvsWdL0NA5rp69lllodJsTvKSEz8ZOT06+VzPrITkVCaliWlfBaRSZYgnbEl9TUVOaehn++/Lu8Tt+/gEsc3xzCmVuZHN0cmVhbQplbmRvYmoKMzcgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCA3MCA+PgpzdHJlYW0KeJwzMzZTMFCwMAISpqaGCuZGlgophlxAPoiVywUTywGzzCzMgSwjC5CWHC5DC2MwbWJspGBmYgZkWSAxILoyuNIAmJoTAwplbmRzdHJlYW0KZW5kb2JqCjM4IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMTggPj4Kc3RyZWFtCnicMza0UDCAwxRDrjQAHeYDUgplbmRzdHJlYW0KZW5kb2JqCjM5IDAgb2JqCjw8IC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMTMzID4+CnN0cmVhbQp4nEWPSw4EIQhE95yijsDHH+dxMumFc//tgJ1uE2M9hVSBuYKhPS5rA50VHyEZtvG3qZaORVk+VHpSVg/J4Iesxssh3KAs8IJJKoYhUIuYGpEtZW63gNs2DbKylVOljrCLozCP9rRsFR5folsidZI/g8QqL9zjuh3Ipda73qKLvn+kATEJCmVuZHN0cmVhbQplbmRvYmoKNDAgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyNTEgPj4Kc3RyZWFtCnicLVFJcgNBCLvPK/SEZqffY5crh+T/1wjKBwYNi0B0WuKgjJ8gLFe85ZGraMPfMzGC3wWHfivXbVjkQFQgSWNQNaF28Xr0HthxmAnMk9awDGasD/yMKdzoxeExGWe312XUEOxdrz2ZQcmsXMQlExdM1WEjZw4/mTIutHM9NyDnRliXYZBuVhozEo40hUghhaqbpM4EQRKMrkaNNnIU+6Uvj3SGVY2oMexzLW1fz004a9DsWKzy5JQeXXEuJxcvrBz09TYDF1FprPJASMD9bg/1c7KT33hL584W0+N7zcnywlRgxZvXbkA21eLfvIjj+4yv5+f5/ANfYFuICmVuZHN0cmVhbQplbmRvYmoKNDEgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyMTUgPj4Kc3RyZWFtCnicNVE5DgMhDOz3Ff5AJIwveE+iKM3+v82M0VYewVyGtJQhmfJSk6gh5VM+epkunLrc18xqNOeWtC1zgLi2vC+tksCJZoiDwWmYuAGaPAFD19GoUUMXHtDUpVMosNwEPoq3bg/dY7WBl7Yh54kgYigZLEHNqUUTFm3PJ6Q1v16LG96X7d3IU6XGlhiBBgFWOBzX6NfwlT1PJtF0FTLUqzXLGAkTRSI8+Y6m1RPrWjTSMhLUxhGsagO8O/0wTgAAE3HLAmSfSpSz5MRvsfSzBlf6/gGfR1SWCmVuZHN0cmVhbQplbmRvYmoKMjMgMCBvYmoKPDwgL0Jhc2VGb250IC9EZWphVnVTYW5zIC9DaGFyUHJvY3MgMjQgMCBSCi9FbmNvZGluZyA8PAovRGlmZmVyZW5jZXMgWyAzMiAvc3BhY2UgNDYgL3BlcmlvZCA0OCAvemVybyAvb25lIC90d28gNTMgL2ZpdmUgNTUgL3NldmVuIDY3IC9DIC9EIDk3Ci9hIDEwMSAvZSAxMDggL2wgL20gMTEyIC9wIDExNSAvcyAvdCBdCi9UeXBlIC9FbmNvZGluZyA+PgovRmlyc3RDaGFyIDAgL0ZvbnRCQm94IFsgLTEwMjEgLTQ2MyAxNzk0IDEyMzMgXSAvRm9udERlc2NyaXB0b3IgMjIgMCBSCi9Gb250TWF0cml4IFsgMC4wMDEgMCAwIDAuMDAxIDAgMCBdIC9MYXN0Q2hhciAyNTUgL05hbWUgL0RlamFWdVNhbnMKL1N1YnR5cGUgL1R5cGUzIC9UeXBlIC9Gb250IC9XaWR0aHMgMjEgMCBSID4+CmVuZG9iagoyMiAwIG9iago8PCAvQXNjZW50IDkyOSAvQ2FwSGVpZ2h0IDAgL0Rlc2NlbnQgLTIzNiAvRmxhZ3MgMzIKL0ZvbnRCQm94IFsgLTEwMjEgLTQ2MyAxNzk0IDEyMzMgXSAvRm9udE5hbWUgL0RlamFWdVNhbnMgL0l0YWxpY0FuZ2xlIDAKL01heFdpZHRoIDEzNDIgL1N0ZW1WIDAgL1R5cGUgL0ZvbnREZXNjcmlwdG9yIC9YSGVpZ2h0IDAgPj4KZW5kb2JqCjIxIDAgb2JqClsgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAKNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCA2MDAgNjAwIDYwMCAzMTggNDAxIDQ2MCA4MzggNjM2Cjk1MCA3ODAgMjc1IDM5MCAzOTAgNTAwIDgzOCAzMTggMzYxIDMxOCAzMzcgNjM2IDYzNiA2MzYgNjM2IDYzNiA2MzYgNjM2IDYzNgo2MzYgNjM2IDMzNyAzMzcgODM4IDgzOCA4MzggNTMxIDEwMDAgNjg0IDY4NiA2OTggNzcwIDYzMiA1NzUgNzc1IDc1MiAyOTUKMjk1IDY1NiA1NTcgODYzIDc0OCA3ODcgNjAzIDc4NyA2OTUgNjM1IDYxMSA3MzIgNjg0IDk4OSA2ODUgNjExIDY4NSAzOTAgMzM3CjM5MCA4MzggNTAwIDUwMCA2MTMgNjM1IDU1MCA2MzUgNjE1IDM1MiA2MzUgNjM0IDI3OCAyNzggNTc5IDI3OCA5NzQgNjM0IDYxMgo2MzUgNjM1IDQxMSA1MjEgMzkyIDYzNCA1OTIgODE4IDU5MiA1OTIgNTI1IDYzNiAzMzcgNjM2IDgzOCA2MDAgNjM2IDYwMCAzMTgKMzUyIDUxOCAxMDAwIDUwMCA1MDAgNTAwIDEzNDIgNjM1IDQwMCAxMDcwIDYwMCA2ODUgNjAwIDYwMCAzMTggMzE4IDUxOCA1MTgKNTkwIDUwMCAxMDAwIDUwMCAxMDAwIDUyMSA0MDAgMTAyMyA2MDAgNTI1IDYxMSAzMTggNDAxIDYzNiA2MzYgNjM2IDYzNiAzMzcKNTAwIDUwMCAxMDAwIDQ3MSA2MTIgODM4IDM2MSAxMDAwIDUwMCA1MDAgODM4IDQwMSA0MDEgNTAwIDYzNiA2MzYgMzE4IDUwMAo0MDEgNDcxIDYxMiA5NjkgOTY5IDk2OSA1MzEgNjg0IDY4NCA2ODQgNjg0IDY4NCA2ODQgOTc0IDY5OCA2MzIgNjMyIDYzMiA2MzIKMjk1IDI5NSAyOTUgMjk1IDc3NSA3NDggNzg3IDc4NyA3ODcgNzg3IDc4NyA4MzggNzg3IDczMiA3MzIgNzMyIDczMiA2MTEgNjA1CjYzMCA2MTMgNjEzIDYxMyA2MTMgNjEzIDYxMyA5ODIgNTUwIDYxNSA2MTUgNjE1IDYxNSAyNzggMjc4IDI3OCAyNzggNjEyIDYzNAo2MTIgNjEyIDYxMiA2MTIgNjEyIDgzOCA2MTIgNjM0IDYzNCA2MzQgNjM0IDU5MiA2MzUgNTkyIF0KZW5kb2JqCjI0IDAgb2JqCjw8IC9DIDI1IDAgUiAvRCAyNiAwIFIgL2EgMjcgMCBSIC9lIDI4IDAgUiAvZml2ZSAyOSAwIFIgL2wgMzAgMCBSCi9tIDMxIDAgUiAvb25lIDMzIDAgUiAvcCAzNCAwIFIgL3BlcmlvZCAzNSAwIFIgL3MgMzYgMCBSIC9zZXZlbiAzNyAwIFIKL3NwYWNlIDM4IDAgUiAvdCAzOSAwIFIgL3R3byA0MCAwIFIgL3plcm8gNDEgMCBSID4+CmVuZG9iagozIDAgb2JqCjw8IC9GMSAyMyAwIFIgL0YyIDE4IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvQTEgPDwgL0NBIDAgL1R5cGUgL0V4dEdTdGF0ZSAvY2EgMSA+PgovQTIgPDwgL0NBIDEgL1R5cGUgL0V4dEdTdGF0ZSAvY2EgMSA+PgovQTMgPDwgL0NBIDAuOCAvVHlwZSAvRXh0R1N0YXRlIC9jYSAwLjggPj4gPj4KZW5kb2JqCjUgMCBvYmoKPDwgPj4KZW5kb2JqCjYgMCBvYmoKPDwgPj4KZW5kb2JqCjcgMCBvYmoKPDwgL0YxLURlamFWdVNhbnMtbWludXMgMzIgMCBSIC9JMSAxMyAwIFIgL00wIDE0IDAgUiAvTTEgMTUgMCBSID4+CmVuZG9iagoxMyAwIG9iago8PCAvQml0c1BlckNvbXBvbmVudCA4IC9Db2xvclNwYWNlIC9EZXZpY2VSR0IKL0RlY29kZVBhcm1zIDw8IC9Db2xvcnMgMyAvQ29sdW1ucyAyMTggL1ByZWRpY3RvciAxMCA+PgovRmlsdGVyIC9GbGF0ZURlY29kZSAvSGVpZ2h0IDIxOCAvTGVuZ3RoIDQyIDAgUiAvU3VidHlwZSAvSW1hZ2UKL1R5cGUgL1hPYmplY3QgL1dpZHRoIDIxOCA+PgpzdHJlYW0KeJzVvWGW4yoTtJl4vInZ7Gx5VtBmfkgkkZGRgFxV9/2G08ctIYQQPIpIJNnV/t//5/82MzNr1seCpx5XzazH1ZnaKF+lasdfSasDW5MF+tgUa2hQ8l7uc9lzGn52+DRf7a1b+4zVj73GavtY+9hrLL989V9/fez1z+7Pf9bi6utfvxb+r38j5zNWr91Fpr3mIXo4HLWhmzfYz6h9+p0/Tn+erM1T9s/cUdT/2L33XjgELxq2OHJ5IFs18OPAJXV9A82XaVdtkwW+YtEUi4bDU7F4FUAWe2Cxfax9bhbbx9q/iewE9CpA//75Ln0Uu8tjDYLFHvjzywnPqPUwoF6gQf806BbuKIudnNhgVJrZy/fEEaLxe0KkOAyV+S0oD6rSTX3K4ihGMjCHbewSWMRxTdrTPuHf69NfEsGpdkP56N9nbM2kfhDKLijEVURw/uuI5hSzyIx3VyMAUrF9euEwKCIlWz8i0n4M5QmIv8giLARLguEB+AaLcbARRJQuZyVp4Yq2WBi2/kugQ5PYlzF4QDm8T6cjiLMHpE2nHvZRqDbp9KLdKiJTRT8l0sAaztNZeR0s2hMWYz1kTGEwgrT0IIoRxBeC0u9QEj1U/Ptn7YL13+3gIawcsvrK/j7r7Bgs8rWh/vm5N+qHbNPYq5U0Wsxfp9fYOQ8SLv8VkV6+QrMvt6r0mEXcF4yCWPQGLFgs3TkP/wc2JYymLn4gIrzE7x8KYc+74xFfUEBEC/GyiVHjmU0vpbHMWSSKHcNgxPQ1kc+m1M/5wxa2aq8Fi1kjFYs5fkcXa2Bt+h8b9JDGHoK8ydY/SWcIB0U+1VZECxQymhGXO5uuZjDLiUsxJjG95HhgdbTTIyKfyuQPUimKdsP0UxYpbJogJndeRGYdsQs2jdgNpwZti+RJc8fMV4/1F4ooQ8atTZfDKqVtkTIWr3pULBFZhgWLw38tk8eprQ2alF6GH9+xOJ26+3CurDlEjZ3uxdx6+S/e1oHp8x0+/kvTF2K6j5ARGvCKTZJEzs70cyxsmqXxizGr0ivV+HdE2m8T2eDyrY57wiLdysHyejoZRDHc1pn/UmgIAjanF3ijcVL1j0PGSF6PEeRkGqQxXABZqrm1KWQ0i92SZjCbKHw5cHNXKue3wZuEJo5N2PqUyF+VyQ2Io3lfszjJUzkgMGMqnUb9BWOPGkm3dZpgruMM+sXLfYol1tMD7rcoLtQ6evQMGS2dL/Zz/kzdK5lZpECkjh17ILK8Dh4RaQLKL7jcg2iJRYpiH7IYbEuxqIPFDyLI8dwLI8LA3H27uwkQk2pGphF3ns1UE/xB5AwZwbuDM9j8DAtVHFkNnhwuR+HVeYR4qNJyqPkpkcYktR2ajcpsQezlxGXL4gytLLDY1iwmCl+Zxc94JD0c9oUs/ksLHw8TkylDYcbdH04W82iOF0dnhpAx9uQc087dtU/HetPNerP+ovGIA6ZrPiDyFMp0iPxvUV7UuTboHYsip0+OV7oI/zKLwVXnHeyQ0yKv8DB67PJvPkjkOkdMCVzyPciA4OiNkTlCRgtnjR3IHUs5J6Zc1YJpxo4miNTTGtsTGfbdNvPk30klJ8GiDamzUxZ95IwGT0qjYvE1butQpBgftHRWxE+06eTRVH+4+5NV8BMRhDs74eJEm7bQMwaZ1MPP0oLIF4Wlv07kdy1+lNxuUqYpFr3AIYstsai9D/0RgPDbOvmWIQA3Xy2Lt3v4tk77F5ibc5cPHI4aUzn16B+MEe+uIypU334zj8Yd5D4vGoz1wb4g8lwmv0g5UjTozTMWc0hEgfyKxQ+Pvd/hA+yiblHUqJSPSA379oi185cuA36HAyJji2c0RREtG3ss0fkgRew6baKYLLxCEaWYBw/qe0Sk/YVMSneODThiEeeMyCIYdMViGFcc+6B/0aNJI/ENiVjMUeZAE+IBlMYA5VoOZ/vTnR20aSWNDyYxmBKRoo4LSjfrHE6ZXLBvifxFmZTubE9YjPl+tTCL8SgEomNhgkJ1zyXMWsJNRJ++tPAiY090Bl28LhKSRgMip6rFqHF2kSWbTp1TSmM1mk1tTNYMlzOkFx3V910EkXZA5Fomv+PSu1WCmIKhcNBRLAQkPewrDGs6dXqjliybfBO9W0mjmKwk8uQMmkLGiOb+37RmF3voor00quXTVMWLiOYrD1u8Vk6JzA1dyOQjLqGkPqPcd8jfjsUwHn2OBwzeZFFGjXP2QNIYLDULZOecgGlHWD1w1OjHiQtfMNCHkz/VVzRM3ycpkL6p1Wi+x2Bcz2ruhW7t+iZXN2vWoGovabS1z3a0sfEqsGyzWd8wueqX3IPxgmm0TNcVkqdYdDdkUSwixcqmlSIG2tygPeLMHo1TmTmbLmbQFv+1yF95oxF7qfLoYhx4mNoc2XIE8wZ/SDhDXRw/31F6n+8bV6me7Sm15b8yxWOVLPbYyw9ZnCBmT4xGCS8uDBbBiOMNRf6aVYDyH79UGw36OnSvQDyy6TQ61M85vsx9rnegzCGTp9EZPiQk/9KjmJbXRJ5D+SDJSDE2L/iyfc+iGNpPgCBp2BQ2vFPoU+kXPnqJ0siTG6oHptL6uQtxaQyi2/REEzsnSyOk7707QrkGob9tWPPlvMmsfZNFpy5de6xie2b+T86sCm4ov4h9OYh8xOJHsZjsOLzFiF9znvPl3pC/f2Gu3SKp/NbZjWD6vgExh2Y9Oopseoqfksaqwzm1UKC3KFK58EnF4UbP72qklEnfdC6WqrwW3b5nsf0NizjPnTyxHY+3dWAGQ36dpzV8oJ5uMX4Shcqpg02njgoAUCefD9QmujpIb5q+9EG9f46WuUaG/IVGGsthg7y4dJoqRbR0bQSzzhfYDz06hXT5Lcb2MR0gRnXkyU2640M2LaWR/1kAkX3ZXbuSxrXI9btIg9Wwa0sR53l6o0cjitG7b87igo2zCkSateaLIzN699PWhi7bgTjLZxYh55BF+wgWk16qtxjpFs+/rl+PUE8C0Z3hX28Vf0kLc9e1+KpE6EwljQ/GJo1Ih2MfVnjv4uo4IYsih0Te+7W79TqONK5hHi02jfbIp1k0ORaTLFJo8RWLU1dIHeH+H5kp3tNp+FIj3rgRsPKTQCWN6Y63Pw3yBpuOINmmc799labIKCLjmBwn/GJrMzFyhoFwPEIZRxqQQZlF61r6t91xjgE07xdZFMEZLiBMXcghqGBy4fCEmm8x8m0d8mU1g5kU2u3UmGOxW6R2btPcqRWZP4warxru12/92jJFZD8lkl2112zVaJ4UY1GsGvYTFiUEGDX2cFuHKfys3pAo8YVgAKYsHSnkV3UynXk2HaXRsK9k5jpFIhlKrShFGSr87vc9nduy07SmyVs/NsW6xdixWTLuHE0+Onk6nYUMk1kfsui9RCxmaQQObgPtibPwzcD0hgQHiOolCbqXqX70J1wwJIeol7mfo++dDkS245ijYrEjveTD3vcdbbKYiTT8tIJIODyHkraD8iAFu8E642k9YHHuu3w9IoJIAV8IHKUE6p/Gm1NpVMcMfes9rmqbtigxR9KoaFnz06mAzyXCgDxMdMhXH5XQyEHhHE0Kc4zNadJkwb4PIw0ULa4qs9iZxbZg8YZv+YNjcRINM4ztI+n5hgSL331LMm4d9cdJUg+NocDRBJ20bLF/Zi+lzKORkDtt3VmWrHZ5m7XeuzXSwmbLibYpjbR4A8jMskxaJLg6h0VnrUWxihq/Y5EVkVmkXw+Dm9j8O7bRlNmjyay5JWOB9TvbtEHnWJLGhx7ND1ouIrH/KX01m0FfHfcdzfpwZ3o8yEHkEyJtwFH+hPOja5Q6gvizoNl37/cwGJHLAxbJTIlFXCAXFr/RuLrp03ASAxN2VsStTW+lsV64yrYcJlICKO05gQG+nN5X4Gi9tTaJtBA+BiLtXjCgdrbPb0magvKL5lui0ABES1D2sKwU8cpULH4SkfHOIrMY7iwCbfGJS3juN74uSCY+ibz0GH7LWfwzAehcmH2upPGhR5dPosdhwsYeNp0mrGq+73hIJJj4lEYrZDI2zjulG2/iVF18MlTNoohELlisbZF9M7EYvi6dfyUif29VxZoYCeAd7xmz0uy+kMkWu6WSxtMUBbK32NvrHQ/rv2rOm96DvC+JNGXcNmRSQTnXzjsLSoo508ghFlvPOHaDuZGYIsDwh1lFz0oZJs74Po7nyKk0i2L+SkOhfMGmR1ewTc9VYUU5zgnbhlI0S0TaHNBniMcWiF2piXijx8ysA5EeJqIcEpEylJynFaHMh98mAhFzesiZChECRMgZD22PWFQhI89ggtSBC+uvv8Tvp8bpUQgZU8OmChqT6gGxoS/HDlxrW6/GJAWR5GkH3lZUW5VvcBt8QjlgnKopb4/XRN4HAF1sFjMXJ6ROJXfoQhRXLPqmbH+KRUTkFXVxYqd+w4lv7tDX/rPW3s0opy8YON4nDqsUuix6UvRs5+UJ6Bw2kU5VcsFf3vp2UQGDbr331swCi0zkmIXl6bZZlEkLUM7WLE+ochYKjJ6xeGXm6UueTXf362DBJIrDl8PP1Pon/5pt9Gi+GEAXLSFIkeK0aT8pP/fCqblzZecDkUZQqrEQuxep5M8LXPnvcSY9E2njfqMk0sImu7oMKLxCECYvcrlK6cxbD1tREtYs4gAbQZA+o3p1enlHOXVfaOTF2eszHv3F2l6Afg4Z704AL87ezX31KLC7++4edF/1egKUlldWKTRkjSCm9zhmcyKnKKrb4xWRliY3lqDkZsa21l1JIFotip6zZ1FxGb+iyo8BxewYH72o1xzzk0AxcflYeRM0A4ozGCmN2Jn9lB+OIBOUD9LikE3VNsp3GzNrs/nKYwgZbc6pFy9V5FDSFJT63A4pNAbRKhanRi5ZpHso6NTxtk4BIseUZMevCzJSQSp5N6Z8MN38fBHKShrv1TN+UtTIRNrzeWdxoA2CmPMe2bf8tUgk3frJRNqt9kSkKShnMyo0M4J53x6KTSLZr49ZHGS8FIsM4r9owYVNYwGv3EPG+CNP/I432zRGimtphK7bgiTlcPUA8Dit/4iq1T4O6ugefUCkjTjS6NuGPrmxGkpo0f60qdhaFIM08s+BrkLGJYtBEWf4GH7/jp9Zx7d1UBpZR7F5a5um5dQnpwlEUTxxQZk4GKOSvFihrRHEnDmznvxtiCR1lMYtobwPvznF1Ga86CkkciJBFM2YPyMQn7BIT1lezuLHycsaCV8pDN4dKLx/Knw0hvXbIog0lLmX1GruWpaFxROX2sFWSfJnCcHc0Eax4xdEWmHcFqGEw5cnnxNZT5/5G1HMOJLwPGQRNA9YlC9JFLeEUA7hfW9l0/gvWzNIY1MdUgaOKVIM3fwXT1zy1gpB3MWfWdtTIi2gicZt0C1twM5Bq0zxnDge6rMMR4oGBm0sh1odgcXXmEcHFnFBsiifvoB23vPoZP3tCm1pLgWimG069ByAuEl0E8cFEibOoab2aHz4WKLwCYLz6A2eWdsNzYZIRy6GkmjcNs65+6HabNh65icu0x4ygzQOEM3m+GmnzjadXu1mFqsnfvEvHmDs+Iq/iEIBIsjz8iuqaNOjQxropbktrDpS9ewQHSugzH2/qi2lMhJTCBpS6DW8x4lN510T6c9sCuM2j3B7gtIbkxut+gD9yAhEmyySQVMQZjsWhUc7izib/tdDsJhmOXn6wlym6QtZs7BpUMcHcU7oRBBI4G/Gkc+gnmljd+cIYg3jfcfR1O+INH5gaAQltW/RrdFcCERyZ0MKo0A6f1kd8UVXyWKLUIbbOkMg03R73GKkw41/cPc7SqOJZedSzOdooR/h1JGZHD6dTi+XRev7iwAunFBTxd4xcLQnRFo0bpRJg2qNuDxJaEYeOa1FMc1gZuDok1mcxNQsiseA6c8Koea98nPt/B6Gsyjt2ykcoxNmMAYXIZpGTkMewk2cEc2H/qepRq7zZLRObnFXQki7X418DwwbtHhDZDD3QibHMVzYysAiNBZOB1cjiBYCx7ksIsVo0DinxueB5QxmINsCnVwyT9Jz1Oiz+xxL+OmTjwtpDEO5Sy3uCO9tfVUdVFLvnhE0EkJTFPpyuNHjSK2J5Pvh3a4Ztw2ZtDi/9vnaSeiDUSYOlQGIEkc12CsW02tjHDVejwGL3+7G0BO/q09Hf4VM8bdd0al5cOn6POxAF8ghDXHryED/0fUsjwZ7ai+m2gk7Vaa3cRscibRMYUGkzZuOvZn18ZakAdaGYplPRl73mIkgGsiGihezU29ZZHUMP6lY/KQY/liees2CKfwUNm1T0RFNlEa+IHepRwY0kTAEJ3hjea8W8msETQhhLja1s+WpzAmRVB6M2wjK0fZmqZ9yyhRaBLHwaCGNUhTHT0dM3SJ1xO/1xUcvLW4du/dp0H5/O18P1DwLOYaRIo4agGh5K6y7L82SbnM+NOuez4kiy7CJm7Dy4lEV6aIRhV6AnsrYmkjMaeYhIz6qZij9eGdODR03rXkugChmCoU6js/hp5FF/Iun/wKaDmIWyLGQnkrHOft1oNEk/SfYgxxGQP2UD4O6u6i/mBOfRpBEVXUWcWHiT1ZRCSGuynkMFn57DrxiRCJfzWasgNihtMGlbL9IHVro/I3GyqiRKDSy6Q+Ilht0nyz6ncUWddFNmefU/wKLr0947sKTpPtzadNRHasI+xRKCI0mlNCxoy+r3fVxNH92hKBVQujLsLULs4aFavU+bb/vrTgmLm3RC9mhUA5HPumiFkUgMtzW6eU9apq7+J1Fcbs7sYgzmOLPsOnf0p0ggjqiQYcOqYK/wZ18nbvDWsWZTJknPGg2YrnLXgizZDqOZoIt35NCEybVn1mD72PbZkiD/0kNKMaDQTSm0IgDNXFp+fuB/2hBvB4xZfIjWGQuQX1vPUaPtsKmsX/oxEUnxXHxgcTegz33MdIa1Dy/forgKMNCmEu2PLOGBQIUMys1xQXD+TU2HZjjpCic1mwrXRREXjNfNFMkMrI4/6ZBVs056S5Z9Po3/2y0FoeyR5K2AGF/QmF8JBbKnCc0U1WJQNACZLlMJYR6UzDrqHDBnVVmtnIDgo26oqkrtccyjq/EMenixPEDOThx+Zj47lUSyBWL8fWIDX8ojfgw0GaZuzO8tfncl4nCoXwfh78TUqWKPNgaClQI2k4IyY6pftqkZ9Z4JOeVrsUqrEyF2awxJZ24PqdsmABRiKJw6jhxqQLHJYsNijVFIYWP0J6OOejUGDsanl3ujUWiAN3C0HiZU52NxztEcJZUQmg1hZVeBhwNMJIqOD/TtxR6IYpzNV3E6dQFhUZRowkcG/5iO0xm882XTOT8+lW4s828XkKL3wDMFEYiOyKIGhlGc61emBp05byVEy/4w5uL6ZB9vZVydre1FzOYFYVeBr+cYIU7n67CY3th1nmvzhty4ChBDGMcnfrlN1aimIloz+fR/6L/0uqQT23T23hxnEIpjTZPdpFmpxGRFqH0MrH/t6aMB+LMRzcUk2quKEz5vTLruRrnLqJAan/+mUqMGvEiFoEjxYsGCFoYUS1O/u2TRI9msZpruzr29CfSxxHx/iK0J7w/ZsOIG0X88drD/H1CIi32PsWRtJdKgj+LVFmksLDjWf45hTOfzdpCXJK1sGKxorObWVe3vYphmDhubbrDl2A+gwN5Q0es8j0dySJOhq4wNLwtITSSn77kVZTGqkNC1wyj4ZuL8CZAyF8mUjU6lsQ3R42EaS8K77WwpUre/Y4YqdFSKfO5aRb7bHSjoib6vcEIhahRgVg4NXz1pCZyfPIf+K1YDAEogtgTkTFkvOUQAiEpjTpoUT18lRa3u69iT4JF2lQiSIeog0LL5OHyCYUWbvR0169+dWI7kkbcZAa1yFR1mQoZWR3JqYNNd4KDQexzWbMYn1YTi9Ut7vWPMk6bBnUUAeJgd5UadF31zuJOFLEYt6JGUAuhMUmhZO3UKwq9hmHWkUhokoAs85r0c4Om7+UtpJBRCeREkyJF0qr6tk6+jx2eVuM8Wu6eDhTkObIYQPQ+7/N8dfeAO8+QqUWIGq3x1ti76hA0ApUKGnNmazu2GseCwrA1xo61KuZlX12XN8Z05vuJZKcmBC0M8FDHGCkiHNVUOv2acnFzZzeP/qR3yaqQEewYT3Obegrirf4qNMd2lDJ8qfzR29oLOzaGDPMlmmKruO84ftVxrBZTEBlp5kDSt1apsmmL8aLBANeK2OI7ExWIrIUcMsIrjGlWLlkUIaMPSpTGac2yT5Iu0rIP4YkzV448CzTeukKwFkLcJIXQiMK1TL6xj5DDRFVvQN0miix7AbfiwFQh4zWcYNMli/UMphNVJJByHj3dObIYQsNuJn/wCWNHPCmLmCida6Nrwuv0sfxGXlWfCwVVCHLJhQX7clFGTqt5a5bJ+5n1EMIe2plkEinsd6+1r6SRzKvRp1DHPu/wHdh0FSzWn0xtCEOBRcPnQCZADPAVHr0JHMdyGIYTVcSj5fInCEaSaOs2WDyauGhAm5nZ+2OtpaYPDQwymW0a90BkrS4ZI6r7jAppvKwwzw944hKJ1NORsykOa2EIEFkg75DRAESDMxqrIYiUKTpyw/zRt5tU1U782QGCdmDHqcxPKcy3wTtS0hBIangSRcnrQhdxa5+riUX4Qe8MRGXTXc1awHxXbp7JpgN1ZtE8nPDTqRXRCJumeim/uci7LVPmzyKCVlDYuMBKFCtMCzTZkTOFBrGjVkbK66bv3Awiwx+VMfip+li44Y4WWQRf1iBeqySKInDkH/RmLsNWEFHiO7P4YRazcvtJwQBt0t210F8+oswlJQlfPOoKQVNCaA8ofBYXSgot4egHuq9LVLgge/jr4WkreXSiHPpjnEuHkheF7NTPbLp1fZvGUUtEJhYxJC3/dYttw/MOgeP4bD2c+OzR0W+ZyLvYoS7GYhsEbWXHYfmLxyrSxFvrsbbSx+m7MuGcrh2G8g1Sof+yTXuHZsmF5QaZgUKbfleGjFEUw88lSvI0kelPIhDfpU2Hvwli3iWQY7jpOLH9LHRRwpePd/AF00evJD67oW2DQpu0raJJYdYZwbQJFrrdLzveDWx4erk3w/gNBC1SaIpFAuIjiZzPrOVDwiiBnemkSboOFu26xeiRYrhaRo6dUIhCCJcx9lkVGXE9uUzjAgs7/jmFCjKm0DeVFDqvE8d+5zYiT/ky/vQy/LSotfyDE9gzoB++ShRqXRyImPDorucfaRIzFjoV06KYlh1EjiK899CR8QSxC4BC/T5Ep/9TgZSyEc/dJYKPRPEHWojFDtVx4Djucmv/TX9tfUaKKIpbbcBAaoyrL4iRjky4Ug68Ogubmju3MOnuNGURT6KVOl6hLTXMxjWGovjAoPNX2w7277IMImgJKT9cXpZShxVmejjzuSMvSr57kkMLCM4/cxThgx/egL9DWCcMrYI0bkNGEc/Nd2xLHInL+OPHfHO7BnHFIqijBx7rjsgvJs/itNeay4I/q7zYmDDb2bFJnnzBg4lfodA/79vgFliMXhwNGn/r1ht9qIs2ByzoSg/LFxAFjsXrZGp1IYqhfNbgRywCRi0tiJ7Y/cFynfIFj0Ycd189VvHlRw/3JqCNN5lmK+ewL8vyb+9chBLkUHw+lMZSF5fq2D60Ck+rVzd6YPVzf7WA4VvPWsJx+9VIyaLFc1lJYwv5t73MdSiW8kIluDXi+/j1WOYsFthSWCjols5V/oWjZhFW82+A365ts6PLa9upjbpoiUWhTDcoXdBTvIAY7413PV/OINKUxVns1ixkzhOxtFp3QIOOSh1T70nApcz1JFr6MnJgBIdFkp5QaCe0Qf5uKmNAIa5udbGeShvqogGIjqBFUTT+AzBGP7YkydP5tShCVZalcZLHLAbx6wkhZdnUl/Mi150l9hL5yze0TVJY0fmUQv2gJRXYOvVi9xk7+tkAf7+jixZHUYVfrEzw6zYVkfLFWN9XbJVVJVW2ocd8wRh/2lYaQRHnxV70V6A0o7kUQiqwodCOHdk0WznnMP925EUxv9EjtDDqInbuVhcNpRFGblr2WA1WiK/444RGYaeMuJcKGtxfyeSE7zGLS4MIRK6KpXR0Q5EIw+U1hb6AFNqKrUXOMzoXsOqHhIrFhn/Rba2Ls9tipFWKIr+R4KxY5KY2ax1fsiJKxUWDRnf2q8UUi3Cuq37Yb47dtkAQqvpTCjdUqU0ke2tqVxo5pzL4aeDXFm/ltP3rd6yLvsCqM98XrEBZEPkCay5ZzJWsDTo30goW0/Xmia/tXWIErRBCWEYyaNOawpNnJ1j+dyYoy2KhsDZrgylL/Gsx96gsUh6nOLr+FoIJs1Z6pn+F5/5ey8bW43JC0JpFeQ7thLMoWBRphDFVLwWqaEcphFj+nEILI72ncLHpcOLyyMGrwu+P32Afp4jqOHroJFgMpOJYOoLmuphEcSmQYhM+qs57aS9W6NMMOrZ8y2KWRuzIfULmcs4XjmzEk6KQy5xSuM9/7tShneEh4RU5TVG8TwVPstTFMELdM2ubBgISMekGOC4La2YjXj1fCbq4EMVg0H4WfKZVdyxTjaCRcWcKLXJQ539xy/B7Cq/8rZQuKPQaglnbPQB90HnXvIgXsyKOBS02yIQkkoRtrqrJSiGovwAitd9PaqRNxJJTlswthbhLTWEUyz+nkOuJxc6EM1Hoq++PNQs3ty2/J5a/20tj02d+kBM55JkSKyTNp8wS1p0KrkBUbWMQ/Szy+aYcndZCGFcFhUtH/h9QeFysIHhQuKhhqqOfXp+L3B1pU8hUZmeA4LWsiSTD7forzPmuDZl1PsRY7rMl0LxvQEyZqkQqgvlrR15SiPn/wY3rQt5+m0KvwZ/KgD9zD/qYUUJRtEQkjj2I0wpES6aslW/1QCVDmUDcWTOeC/dHBeKOQiMQI4W0Nc95scD/WTeujWHdO3ItumMqk/pudACvtrS14QLhmKSxdtLSlOUuWxa9zrlqoXlBJvF00olvrPlrOXTILAzJLCMwLUw5oXmE4yGdz+4jttMaZIX4XRkHq4f+tJA7loWpKR8kWZIULiBjL5bLfri5Gmcq8DlP88SXFxTOQrFvKjm0Uu1sB+JP5PAo/wy4JdntiXDW18A1laGUc0b36CGcA6wQnDmVFp7JXmnxIIfl0aGRoeVjYZ77FsHUTZnFL0D0sYmZSzmE2kSBLUOQ/6MJylZlz6+B/LX/kKSFqXirRNBzlhQeuXC3Jsiu5bCnRqaF++zOEZz7UA8VLG5BlNOUH3wNao2dLdnaqFdYbbLCE1NeHSW875gTwlopokUCBJQrLWTIPrGAib3gEHV0+EcUcvfYqHesZhAtjJOdgPjEl9fAral94OCSwoKtB9LLN3qUWWdftsRfzols9aiLhebZcissR2q/fN1htlmeaZF63uxSBzufiOIpiJj/yAFTsTXEmwkNNe9nWJ9wrNUxDKGUGR97xgKEcBQ41cWCyFl/fkPbwqe4VIi/A/hyal0QWbK4BhGGwWwH4he+/CPgJO4NSz4GTrZHHGWsBnU8tmZLCF45BErQtvNYcB0XZl2kRiKCeEajGKZWbVkwK9FMLKJY1vPfZyDuZ6YVIgdzlLj734SG2/bgUxkURR5adkNtl9mFT5RPFethEzTgyvGGhVapExnnK1jERD9Ccn+1HGpIIsv6FzrXQi9/DSIx4cU2yvd1SRkanl0VP6LQC9/PrGcfzwH25ZFTRmwSqSp/SSQLIUeED0G8z0hljg1jSw+ZLRGJW8O+FXDlkJyBeKx8JwP/NDT8BrhjwRYVev4bxMxg5CaCVlhkpYtJIB/YcVG5t2cRI9LynWoKfWsbmZNLJ1LtPRH0hV2QZ3MA7qqfTgsWY78QTo1myFyFhufAfRHU6mLvj70sDu1A0Aoh3NqxHelip2JyFVqVLxUQL4oRR8mQmtjUIXP+foHURePM3K0LtSNR/AVJO95X0HASGn6r1uFA233pBOczaxfCazzOKazh0xR2ayZq2ItiBhGlMeWEFLXOe+Sq8O6RbtZiwcFlTzuuLnFBgBBF4V+nMP2E16b3rTE6UbgfO3WzoY7BjisKbalkZVwYCFY3ww1WlRgHUUwLnoRNh81m8Q3O+V2LASUSGfdbJelKKbOd2J/m7ADEY15TdPhzqcYz3UUsqtoxkl7+TWSYYjHrWcy/lz8BrFGs+BltC5XwQaExFvkL5PnWIt2nisWujhhCaAPEBjcXmcLGnx1zVmonDHrFwZJLruqQnuWLXtvoc3V2W3GtCpvlKfyd//4kW6z0r1ZEacpZCAXQShFFyGgRKZJG4+TbxU/AXxW2+D3JdtfTsYrGVXc8Ho6r9zh2cbbFR5BVKDTrKkftsvPlk+iQyqhM3U6tji0UzpW8ZQCn0EFhK1n89KCX1QKWsSODNtLFkRCYrjKRkhkp4sTFBnz4KRPs4nVOalFO1iweyKGQqBMR3R19t2NcXWSm3StGpxxm/nL5Sx0nDZE2JXsVi/XftpAE+xGjQDo0Ez4XyLFaJf497SGBnh18GeBzXWxV9S0ujFUUxQ0N5zg+8m7OaaIeKLlV6BM5lGTP3ogUFvkipxuoI1Gy82L25S2vrqykkQLBA4M2iU4UtrnW+IevWr8z52/7qtTxM13QPKKZxWSsi+Ufw9pWsFb+XhQLq0XmEtlWFqN82vT+2Mu10FYgKuD60Zeat9Jo7Np3O6VSOawFQzagCdY88u85taujS6R6T4KP0tRCYGWySNKIy1vONir4CMQtcydyuJQ38O5EYapEU+ib3no6nINCpK305fZZVmUhJMgIsjqO/AxG2NQ42zM6QYkyib+/n5OicI5QFMU9iyiTeTUtH4FYPVCR5SV5j+RwZ8orC17SHLqRzfoAROKPdHFDYZTGQKSBO8OXGmsRFEUCi2O5+/SZ3HmxEFMQTh8qm3jhqpC9ttz6hFRUxGeiWKEpGWLZi/vOkq3If0ihb3qvf87mk0Bc/t5DGTXazDGbmUMUR8PQnVEjc2pAzvgLLZfw3VuzFiKRBgJp9+cmgRz2tHrf65a2m1fXokhQWsh8BuIhXmunFpnNdvw9o9A3YexYmG9PaBYeTZssQhkpvBEEXUSBHO0s0/x7xZ4zTu06VJs5k8Umn76MZYbSNQ/b5WOJq4PFMGytpG0vilFQEUSqyp6oo8ZLMqQz9xSK/KLAzAk4KphAHfnnaIUoJr8WCEYKyanv5gELtIp8WHLU7huHKFrvvbUph0jkQPkWxcKd5xH9H7YIuJlfs7pk0vYslqJYamScrKz17ySnKhBBwSshZFaFVf4CUJH5/lRgzV+DsIRjEVlGgQQWzYzeZTSHEhaudpV49PsEiEV/9DLl8CLTiUzbsMKQYDXrov9b8HfOIhOmNbJp8qjkExC7X10JRAVQkw5uC9ow/3yTZRxv7OYfJQ0USnVUyspyOJfNLHxeDWlx/Csc+1U4EuOMWZqvTCJNxYvy2eDi0LgwAYo2TZa9ZlEadCB46c5fa6RFvEqF4+coh3Io8mMB3MSZb0atFsUqdlSYmg0QFYsLmx7DLWjIme06jQF0yLn5671d7m3Nz1xWVx3bD+fdlxpmLpO4nPhjFpsuthDFlUaq8vYIxIUvn/OnCqzQpK1THekXw+pfNRZaGIjszcimJ3+TQunUNMiN/r+lzLXQMoJX5S3x51VVt7sxgdr5stlcJWnU6pgg0+Rlv96KYqWRJ2im1RLEBFyFYyWfWEBmZnzt3cdfkExgHf91KlyNL/JEFs0g824aoCa5TOhsDBqJvOsfAgksNpDIxeWQGyU21cytBbLw69YTSScaiZARr3YA4iNfXonfoVJ6Jm19459jicxZLZAKSvFGmY4Xgc7RmIPwLXpslwtg0wZQNpuvk32R3OO0NHq5BXPLZYvBYrFpo5EW8IoFSO0CVasv85/zp5Wvziy3vulPA0n4Nn4dpj6IIPFHGnkP4jmIQ8p6fAfC/J6iBZu+i98iOV99XEsjHvEu1EJoOAtEQDtkZoEsWZyG26qtpqDc+zJKL5xICeKBEG792ixtWjNKhv5e/7G0zd/F6ERhmLhUFEaDXgtWFjXSRbTsRpPoa98WScKVrVjGAgBZc0ZZOz1zlD/gsjGCxJ/w9D2IWAxymgmztpBpXKegM+4lACXO0lZThm7v/LcmT6LGjwLRmVMezSyiQCoSJIhmUQ7jhKb7Xe5x2lMI+z3/DnT2cIBAbYOOs0HbKOaETQQjYY4CwVexKPQyiWLg78Cs5+62AvEX5LBWwVMKvcD7EMSzN3qMBNKERpqjuUzEKJJng7Mbux4U0YGbApnopOXq2JMzz7+kcUjg5G9ujVwmUuMqsyhlUoriE7NumLMGMQBUcZlLFptOt3r+OwNXgQh/x0DoYgKxDdO6V+0Zjvc+gJQ5lG1Oop1IKDR2TQhuE/aU4bKrHbnzaJVroXBnSzkHLFYqyF4s1S5b8xpEJXtfymHFWRRLUeBaCOp4GXdJJN8MujRPCKQtQ0anc5kcsmY3fyZv69hwbUuTmFoOS0ZlswKgEDVmaRwLpIiUgywyZ1WmRY0E3dW6+HsgfocjQiZ2yQWuvnV1VH+PN7K4+Mt+gOYEcfDnOFrUyDzcHbJcQW/+RoG5AOoYJjFW/FGm7RXg/SLLgxyupDEFl0o4NyzmfD+Q1Miw1Wzxfriks6KqpE05b2bUpBxKCr3AO7MoiNx59FiwSOQUSBhnSWQAYGjXdTJ3yAiiaPRiDjyJRtdmORxv7/BcO6WeLmLkrGOHKmn04QkIjq39OYvCoLNkWiGKmUvIf6yIynlrzmCrLgAgXoljxwWLUh1thaO5NMKCXA0tHkhdlVxtJZu+zwrguw/XjDbScpkaLzuFIIrNC7AiJhVEZAG1GS9KFmU+wacl05IoFiCuMg9BrAVPyCHtC90Y+9a6yRs9NYiVOhpCaRrHGTsukpcB5pr8+h+K4nHIeK92WzWlpQVcbalArZEC0AhcZvFbvxaiiDhapA1L2jGIpfOSHELP7OUQnaqt1FGzmA06UAgICl10467TfTJXhW0+gDb6+l/13ase+dtfAbvU5me27K1GRkCnTT9lcenXO1FMIJqlTDsCsRY8tTUUqEHESt6HLOaXJ2yD45Q6pHDtmsOgzYNFG1DGRy93df3e5RbLeZhRX8xsu+NrabwJa7rMUiPZpkHhHrMocARRRPlUOEo6S+DW+Z5JW6GYnVGIlZjBlxPgc8ViEEUTdF5HcKp8IbSBoPC2xl8Vu5uOMxhp0BBBInBzee3OVm8dcsglkTkvFpEKGum7JBCzRpYsInCDbzsRxcjWGriNUsZNcqs9ksOR7mJJHTvzV7GYKORJjM2R1BOX2Za5GH7qbjCHRF7l8bUdg6hR5FjcxYstAPWB9y4j7JJfC6UMGgk2TVb+pV/DY26rcZTMfQdiIYcJ04dyiKPQWB31m2a4albguJ1Nb6cyQNgMFi3OqX3VIMeGiBrsAud8LW5mMDZ7XCpiiZ0xRkEpjW1agsgsvsYRBZSNNtmCQqKtQPBHIH4lh0yhL4M69hcGiAWX16hXd3bk9CUwUNm0l3Qo8afu4kKLz6adYDdt8mhWyiJ1U8Dd3dc40xlFOzbWRf3vEE00Ymcx5j/F0QoEVyASQ4UvP5ZD3x23vrMu0hdVy6/CBDTNIN8ihd3iQSF1+D9qHhPp8hZFboqlTOzRPf3M2S5R50a/rvJhoaWcgFoG8a6KmHMWATVNnmSUyhhvyvkG6GgQH0WHOAoLXt/dwosRByxWoign1POgUqAa/I8h43U+SGSfm7rTOWpw/SMhnBAvUhwMzsdek36N+SSNaQGZs6VGUnkMFn1HotMWMomnWVh2yE/d4gB978tSDmOf9/cnfHXwGYuVKHY4YqLQj95x66DK4v1Fp9DUj8hzTjxMEwc/S9hZIydXxRq5lkbw9KCRvm826xMWJYVVjkEmtPm/B5Ep9OV3wZ+cTVeKOPNNqON96PReQwPfDFIX5y7mmtdjAa/HH9tEvM9ZLEvmixjlEDI7lckLybJdTUVOEMg22TphsQ4c16KYPTpm7kCUeC182Xgva/hUpgCxYvFWQc8xEEW0af12zdUWL3efayfgur7j6FKKjKbKow7LMouU+tGX2dbRqQ2B09I4CUbsbOzLAtmChlUs7qRxL4okaayXzWrIKkV8BOJdeKrj4hehFnNqE0SaWZMUggTG/Ku6hrTpRJBBbV9LIyd0lus4qOJQhkQxGLexNDo0rpFeBr07ZjY8xDMWkzSG1dF4QaRy518EkeUwRkTRrGtRPGeRQcy3G3E7xmS9h+/7yV9kTKK4Ik+SjZm0FbuPtqre514G+MSE2iKgEDuyTcMZ+dYVi4UihpzZMIFgBtGyO/8diFj4Lckbdhyuc2IxIDiO4ixmCmXCe41m4cfFEmot3QCXSQP6hV9jwxad3njTQiMRUEfQBnYyZCQWZ2POWTwXxYla401h66ZPmLMIYpBDi+XfMKfO7mywLG/x2JTJcdC1HMZ2eAHEq1m8xdj5CSErpY3ZTFX/NkF5CnHWGhmsDfXS4SMEDRDM4eOEjL/q6gcqZXK0QVIoXVugaabd+RGI2JknIGKZ9JNRxWxa3m60ZNDIIgCSAWVDdCLJstnbE9teThY4EMXS6+Gq5UjfhhBaYMVtTmvkKNlbqMfZ6g4EieI40MKy57IVObN5cCIhv80c2mSzG38EYpZDKlPFjoLFSCGzmEBcY3ArHeJCFpwFcmRO5iravrNmbHTefeFZFlCbu8OQB482RjCEj8OrvPJt+GiWMm1uKlf9XNZh4tcg+tbcV6bKm8Kx1EUDFiuDJhAr5QExM7yJeMMXBLJMg7l9yeMk6xEWgwOTO9r5GJ9Mp2f6v+ndImQ0WvA2QFVaLJNBV6KIV1EWxSMQkxdvQKTo/9qF/nKCpeUhgZNCzWLHaneARPXC29qZLa2FajajufxWKbvcq7r0Z75y6kwn6GJvsdoEYrBs2GWylRQRc/xclDq2nG/x0grenQtQXx0EiNyx2JNv94ZaFPGhy4LFBYgof2GJdHFuhSl25umEMCJ+5DzTUezEtMBb00Ben9rBLUA8pTGeQTioksnp0aZZRPqFQWP+3DRb5U0VwUkqMzvnGEQWyLff06FbjFcByGGPzgYdh3ohLpkJlED5bUCLfJ6kgHc+UFWau6wF8TPR1xOaOH7ZqadGUtToNXtMmdVRyeQEqEGm0+l14moyaHEt0dZFAeqEExBjp4Vi+BdbR+ODU08VPGYxoxlSAyij/+41LxVzvIizBbXnQPNuqdPv5VYPW6WRvuO07EZ1oo9XMtmhnmrhKYtSFANe0ZF/B8TrPzfr1fRldkYPqEkWt0PNvnyvPgLoS6SeJ3YTz2+pQJYNrCTSmaG0adnhoJVN+6ZqQi0se8FiEkVs9qEoEojWuPdWIF45bxNTmbsArE46fd/E4kYUMfVbI3Xx+753eEKzrxDOqmXpfZj6chWPFXNa6FwURZsD7zY9yWuNB7Liz+5KUJLZoKVlb1msZi2/K4oViFd6g9RNjXc0fdXi/cWfsBjbidzsNU8StnX8H3DJiayz1khLopjRvBYmoBaFU/KHNj129wKVOtpTFpMWLubOX4KYPaThfUcDdYyhoZnZmsUvjPOMm73OfYXad/K51UgtKpZWM5Q21a4DfNOjnVc/UL4HCZmPWHxk0CyKJyBieRN9PnekGz2IGjwVhOnLCYvR2UVbY7N/OQrcQhYLPG6A8OhC1ZLw+Grp1AV8wbjJppXi4sIhizoKLBzcCCwpmbm8ce8xFSF2vM4027Qds6hAnCUptv25h6oaVnj9omvHWq3QFV5taVXe8TGWRot0+r5ef23Zz1ncieLanTdxi9zLy7/hWbO8y0jvL65Z3Iz14sWwn3jur0jsowaQAKRNs1YSqqokKGJv0QEtGTf04eqO4yMWK4PO+djyShQfgYg14/uOJyHj9yx6zT/Qp8fY1YdbV/U13zg8OsxqITMsRy0MZk31R+Ek7x7q2PBwD1gstPBIFL8G8UrveA/c8p2d/GpjTocsjqPMarrZ+Vf+iK0F2XnTSc5vJVWtDLDD3e/k1CiHWhptmjKinGOD71gsDXonihlEI1fMIF4572sZbdqSTaeIkJ6t/uKw/oLt/vDov4QpjR8u5BvdHkEGqrye1EoKNEubNsGi0EvKjAt0Lj8VxQrEK70jiNqm46a/Y/F/lv5OKUf1YjnD6uFjsTvXg1HmYuEqT8DFesKmExbPRFG6cxVAdzN7+QV5ZtP8zskX6au9ZKy1LlYcfc3D8woPU3WwIojslVOnBh1K413Y6/f8P2KxJVEkFlvY9979NS5GbdO79J9J4y87eHVRFYd5fJqVRC1qPLk4rkpSELmSxrVN/w2LXUWKHfdSV4jZUMc4eQObXkqj6Lce//0PU3a4s/YsqKC45XtGZ86mjs2VEmnQ0igVMVbI8d+PWcyNlNehCJffeQYjaktJDu1v8PdfM4xRI0WQ1RX1F4FmfRFIIudC4fVhIQKRbTqUWbKYqSrvgafjloqI6RW2C2nUM5hc1Z8B+qe1lhhgh/w8XP4utfpfNcxZGg9tmkRuwSI04JdZbNZfh9K4ten/HyVp37Csn8KDWO5Pf01t1csNDqJc1YyZsAgf1bZtDoeMV+YuEjViN9WwYbGJHrzDzRfut5DG4nzgIMcp+cufz5vOwscqWJR+uN7xPMlgm6cClfdsW4ab6huQWOzkOfWKxZYuKtiLdxxnMQ96mTXf3KETXEvjr/jX2ZB+fajkdfFKO7DjLYhzcI/N/bvTyTp05xdOXZGdbVrL8JLFnoT5gSjm+1Z+oyf88t0jaVyk/62p1+1vqphZfF66TP4ci2c8v9dp56axcuokjfS5CRnzyUUWqdkrFmPKDmDmN3pQGlUQ6RX/HLD/cjJgj06EOr6b0ZslNnPEvjmf0Ey69KgzpeaW04IicxG26DfVIeeUxa1Bq5v5915jZp2a5122xueP4fph9SLeUMRwx+YL0tFMlS/iy6zBG52uUiv+pRScGnJ0c+mxtUGmPI0qVCUWsTHG7dSi6Pi+1lFjqEnm1uks2qoioX1VXx0/FFgJzjc1VzHA0ynUk83jCPKOeuH7KyCKYjRMOl7cBYulKLqOvkxFjUX661DQz+rXNPeQia6v/EpW2MpTzfTOnu267suOfXj1Bmiyqls6cwPaZDS5ZpGOXlvDrPAlG3/o1Iu07pGTwvj26JND7NO4n0V1ODqjmP8bd8IhQOSD9/GlomJmM48i487nZyPUfNtXpFW+UEmjssvqO096x8VeUrlf8iuCP0y/6rm/2wSR8sB2nmJjMHNYcxmD4kL+4sfTtOxqTca2qoJa3qSkUVZ+yuKcymxTdRJfBucqcDw5+Bcjtvbr+tddNoeKTi28PsVrJ3eRWKFlmw/SceC63/2RTe9XYV9xlOupTLgwnjp1NeFKxTZz9P9QO6kp5Rn0FFbKYLFHmsnQVfns9f65mVCuU71PFTiyUy8O/4VN50GXMuR7TXXcwbdqTBPTqNO6fkChaDJfdKkc/NiGtGkDqnDiQkGhmOVAgTmmmXu62lW3c18+0bfSSRYWHLfuJ9eVTX/BIp3qqVn/MClprKY1Ikh/moo98dxnJySbFuoFuwQJLDBd5PNBD9v/fV98dcHnbpI0PzJDyWJM840ek4HLL6V9+Mz5acPB3Z9VdJG/9EPM5W8LGQaIPSCY2j5N/Jpcp00yyrTxy5rYqm/O7jhVdUi8FtIodj9W3yqnW37f0Qv/IpfKNTia2sjR83QmMAKdUVhMrsmF4yNEDkDoR2agGZG/eyh1NFln/kqkLb55gyvn0kgFijhhwfS96c/N+hfvaY8KvyiSvLL76vSEsVpOgSt3zt5NjKYb5vrHZEggJXxqx2/SInzfzPw57YWQilUsNsLxD5x6FUAoaraitvM0secG4OTahrOZpIJeYZ67sExGBGdJ4p4uBtkY2eB1AFql57oqenshjarkMoyCSvz1220jv2D18N6NLPY7l8aC7mTT69lMQDD/OjDooj+bYXwtCKcrouBvTP9RqvksivzvTbyiKp7GRp23Qrv2/VOz/uKSKppSSuMsdnywE+MuiAz2TZMbtNTJWZfXLceUuEC/KUy/zpUbqecODN+vRI379NVhNhHvNgb9o9jx/J52qYIHc7HvjuGGyMpEmYkAEQjGf5QvZ2L0I5pmYzYjfghEnMVKff4g1tKpcurzGLQq8wjHw/NdTJNXk7LFm3A/FwSKIxG1FMDNTCxPc5GDTw49CcR8l2c3m1nc7hAk/BKdlWQfHmEjjZQe4LiLSc34twlpX1G+que45LoEiUdFtZ5A5DDRUn752Rfg5vAxHJdg6rF5lWsf/3DI36XQgqWClAHbq+t8nWoi+4FBr6Rx5hz4z6+4NtzrCbMKzMED+s3whGYVVpTg+lY4kICyA5p1KImrktpvNC2nrcqs56OHodf5M+tZi/9+8KCQf0j45MCpzF9f3dmmDXPGJ5EhAkHlvC0Hjhlomt+EeHGEj7VZC+PGNixOeZnz36fVQH85lWnwb1sSGwHlV2r6p91WzRUgE6HkQHAUDFMZKDmQ7QRoKOnLne8xrYnMXS5nY3+YaEBNre6bqOvuJY6/AkTRdJkjNqU2nPRzSwu5pnljT80nfDlooc2cFsUPNU8iS5++b5BhCBm9VbORlXHTOeatruvfWfYPMKgv9vpYf/iQMLNYATK/vaYCx99WSunaSMC0b6GFnfx68cnIJqG1YNDdUvAa8FLxYggW/4ezmUdjVISh3f4OxyWLm26Totj0pjJ1/S+IBM5mbBIwuTQRJmrxI+xwfp0xxTIGmX0eXTTMT2ocCMOA7Alheo7dAmnfnV8QvnXqRZo4xt16ynnQmtqjxbHSD2X/Qlp3+mKQojKFWbCzMjYFCgk7mZkmOhlEP/rCr/PdexGfkHymUxb5f6+tCx7uTRt1fEpkJYEn0kjPBv9gNhNGa70ASIVQ79ivK6cOomsEYs83m7RfR5mcabyXKXkNM62wl/16OlKinBY4PgvdoiiuWKylURz94SbfvJ/x02wGF5J46AcqsImcGmCt/RqM+5lfw5kGv5aAVvv+n5Rmu45ix/XYtgTcYt+qKvj1rQeHPkhF3GxmG2lEPiyDSFFjdOqWM7NfjyPu/TobN/d6fZpnl7XYvYL71xMeYI3jbLDsgMaNXbOYt55crrrM807Se8S/nTMXPLbLM5gzv2bJjJ8kk3Ag5de4mh284rIKH5edLsLQ/zIFHFUTBGGJQssGnWoTyrdIJy15lMjUZAhfELlQMg2cx461XxtNj5Dg7NcL485+LcPHxbWRu1V29H/j+CdmLVCLm8TWfeyG0stOvXH8p2ndv4s4EsaAZh7+ZS6hjrjLQLNFRpvbsaLTm5fbc+jXQeTW1x6u/nVwuVWZF2XVA9/Vv8OjPpPGs6SPHnN9MKQH+aCGHaMazbuDi2hSfbr2BIG0AKVZOMSQyR4uBlJH8muvhM46dsiJX5+MzzZaXaQj1H/5Nnjh43mCQtJY1pZXZfGFeq/rvtxtLscdEzHZW+WUhYHzT8AUZTJIpk3UeB6TjRslNvt11EXRfydc/qlk0sEEjl/LWCXFCxZlsRbzv05pZxxpWVL7NRIzCrcOfg2txmkKo0mz71QG7V5MaOKmhjT7oelk8azjyfK1Icso6f3b9P8B80jTRAplbmRzdHJlYW0KZW5kb2JqCjQyIDAgb2JqCjE2MjIyCmVuZG9iagoxNCAwIG9iago8PCAvQkJveCBbIC04IC04IDggOCBdIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlIC9MZW5ndGggMTMxIC9TdWJ0eXBlIC9Gb3JtCi9UeXBlIC9YT2JqZWN0ID4+CnN0cmVhbQp4nG2QQQ6EIAxF9z1FL/BJS0Vl69JruJlM4v23A3FATN000L48flH+kvBOpcD4JAlLTrPketOQ0rpMjBjm1bIox6BRLdbOdTioz9BwY3SLsRSm1NboeKOb6Tbekz/6sFkhRj8cDq+EexZDJlwpMQaH3wsv28P/EZ5e1MAfoo1+Y1pD/QplbmRzdHJlYW0KZW5kb2JqCjE1IDAgb2JqCjw8IC9CQm94IFsgLTggLTggOCA4IF0gL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAxMzEgL1N1YnR5cGUgL0Zvcm0KL1R5cGUgL1hPYmplY3QgPj4Kc3RyZWFtCnicbZBBDoQgDEX3PUUv8ElLRWXr0mu4mUzi/bcDcUBM3TTQvjx+Uf6S8E6lwPgkCUtOs+R605DSukyMGObVsijHoFEt1s51OKjP0HBjdIuxFKbU1uh4o5vpNt6TP/qwWSFGPxwOr4R7FkMmXCkxBoffCy/bw/8Rnl7UwB+ijX5jWkP9CmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PCAvQ291bnQgMSAvS2lkcyBbIDExIDAgUiBdIC9UeXBlIC9QYWdlcyA+PgplbmRvYmoKNDMgMCBvYmoKPDwgL0NyZWF0aW9uRGF0ZSAoRDoyMDIxMDkxNjE0MzMwNSswMicwMCcpCi9DcmVhdG9yIChNYXRwbG90bGliIHYzLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZykKL1Byb2R1Y2VyIChNYXRwbG90bGliIHBkZiBiYWNrZW5kIHYzLjQuMykgPj4KZW5kb2JqCnhyZWYKMCA0NAowMDAwMDAwMDAwIDY1NTM1IGYgCjAwMDAwMDAwMTYgMDAwMDAgbiAKMDAwMDAzMDM2MCAwMDAwMCBuIAowMDAwMDEzMDcwIDAwMDAwIG4gCjAwMDAwMTMxMTMgMDAwMDAgbiAKMDAwMDAxMzI1NSAwMDAwMCBuIAowMDAwMDEzMjc2IDAwMDAwIG4gCjAwMDAwMTMyOTcgMDAwMDAgbiAKMDAwMDAwMDA2NSAwMDAwMCBuIAowMDAwMDAwNDAxIDAwMDAwIG4gCjAwMDAwMDUxNzcgMDAwMDAgbiAKMDAwMDAwMDIwOCAwMDAwMCBuIAowMDAwMDA1MTU2IDAwMDAwIG4gCjAwMDAwMTMzNzkgMDAwMDAgbiAKMDAwMDAyOTg1MiAwMDAwMCBuIAowMDAwMDMwMTA2IDAwMDAwIG4gCjAwMDAwMDU4ODggMDAwMDAgbiAKMDAwMDAwNTY4MCAwMDAwMCBuIAowMDAwMDA1MzY0IDAwMDAwIG4gCjAwMDAwMDY5NDEgMDAwMDAgbiAKMDAwMDAwNTE5NyAwMDAwMCBuIAowMDAwMDExODEyIDAwMDAwIG4gCjAwMDAwMTE2MTIgMDAwMDAgbiAKMDAwMDAxMTIxMSAwMDAwMCBuIAowMDAwMDEyODY1IDAwMDAwIG4gCjAwMDAwMDY5NzMgMDAwMDAgbiAKMDAwMDAwNzI4MSAwMDAwMCBuIAowMDAwMDA3NTE4IDAwMDAwIG4gCjAwMDAwMDc4OTggMDAwMDAgbiAKMDAwMDAwODIyMCAwMDAwMCBuIAowMDAwMDA4NTQyIDAwMDAwIG4gCjAwMDAwMDg2NjEgMDAwMDAgbiAKMDAwMDAwODk5MiAwMDAwMCBuIAowMDAwMDA5MTY0IDAwMDAwIG4gCjAwMDAwMDkzMTkgMDAwMDAgbiAKMDAwMDAwOTYzMSAwMDAwMCBuIAowMDAwMDA5NzU0IDAwMDAwIG4gCjAwMDAwMTAxNjEgMDAwMDAgbiAKMDAwMDAxMDMwMyAwMDAwMCBuIAowMDAwMDEwMzkzIDAwMDAwIG4gCjAwMDAwMTA1OTkgMDAwMDAgbiAKMDAwMDAxMDkyMyAwMDAwMCBuIAowMDAwMDI5ODMwIDAwMDAwIG4gCjAwMDAwMzA0MjAgMDAwMDAgbiAKdHJhaWxlcgo8PCAvSW5mbyA0MyAwIFIgL1Jvb3QgMSAwIFIgL1NpemUgNDQgPj4Kc3RhcnR4cmVmCjMwNTc3CiUlRU9GCg==\n", + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2021-09-16T14:33:05.240257\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.4.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.061021, + "end_time": "2021-09-16T12:33:05.627837", + "exception": false, + "start_time": "2021-09-16T12:33:05.566816", + "status": "completed" + }, + "tags": [], + "id": "07c0fdd7" + }, + "source": [ + "The decision boundaries might not look exactly as in the figure in the preamble of this section which can be caused by running it on CPU or a different GPU architecture.\n", + "Nevertheless, the result on the accuracy metric should be the approximately the same." + ], + "id": "07c0fdd7" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.062929, + "end_time": "2021-09-16T12:33:05.754931", + "exception": false, + "start_time": "2021-09-16T12:33:05.692002", + "status": "completed" + }, + "tags": [], + "id": "f0f11f8b" + }, + "source": [ + "## Additional features we didn't get to discuss yet\n", + "\n", + "Finally, you are all set to start with your own PyTorch project!\n", + "In summary, we have looked at how we can build neural networks in PyTorch, and train and test them on data.\n", + "However, there is still much more to PyTorch we haven't discussed yet.\n", + "In the comming series of Jupyter notebooks, we will discover more and more functionalities of PyTorch, so that you also get familiar to PyTorch concepts beyond the basics.\n", + "If you are already interested in learning more of PyTorch, we recommend the official [tutorial website](https://pytorch.org/tutorials/) that contains many tutorials on various topics.\n", + "Especially logging with Tensorboard ([tutorial\n", + "here](https://pytorch.org/tutorials/intermediate/tensorboard_tutorial.html))\n", + "is a good practice that we will explore from Tutorial 5 on." + ], + "id": "f0f11f8b" + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.060537, + "end_time": "2021-09-16T12:33:05.876690", + "exception": false, + "start_time": "2021-09-16T12:33:05.816153", + "status": "completed" + }, + "tags": [], + "id": "7fce9e22" + }, + "source": [ + "## Congratulations - Time to Join the Community!\n", + "\n", + "Congratulations on completing this notebook tutorial! If you enjoyed this and would like to join the Lightning\n", + "movement, you can do so in the following ways!\n", + "\n", + "### Star [Lightning](https://github.com/PyTorchLightning/pytorch-lightning) on GitHub\n", + "The easiest way to help our community is just by starring the GitHub repos! This helps raise awareness of the cool\n", + "tools we're building.\n", + "\n", + "### Join our [Slack](https://join.slack.com/t/pytorch-lightning/shared_invite/zt-pw5v393p-qRaDgEk24~EjiZNBpSQFgQ)!\n", + "The best way to keep up to date on the latest advancements is to join our community! Make sure to introduce yourself\n", + "and share your interests in `#general` channel\n", + "\n", + "\n", + "### Contributions !\n", + "The best way to contribute to our community is to become a code contributor! At any time you can go to\n", + "[Lightning](https://github.com/PyTorchLightning/pytorch-lightning) or [Bolt](https://github.com/PyTorchLightning/lightning-bolts)\n", + "GitHub Issues page and filter for \"good first issue\".\n", + "\n", + "* [Lightning good first issue](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)\n", + "* [Bolt good first issue](https://github.com/PyTorchLightning/lightning-bolts/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)\n", + "* You can also contribute your own notebooks with useful examples !\n", + "\n", + "### Great thanks from the entire Pytorch Lightning Team for your interest !\n", + "\n", + "![Pytorch Lightning](data:image/png;base64,H4sIAAAAAAACA9ycyZKjyNqm93UV0bmlSeaprPJYCyQ0IAkECIQ2xxicScyj4MZ631fWKCKzKior8++qJvQvTphFhAbnlfujz7/BAf/ll8fPb/9jKQu6paxewiZN/vXLb49/L4mdBV8+gezTi2c3NuzmSV7Bae6BL5/stsm/vpxEQdjATQjS6fXXJ1/f8Ozq9u31x+NP//rl5eW3ENje48H0MAWN/eKGdlWD5suntvFh9q1NEmW3lwok03FZDRcV8EHjhp9ewunRl09h0xT1rwgSRE3YOp/f/tn1JFJ/dvP0n0jY3dTRqv6q0dagcvOsAVnzT4XeBGA3yVvvc018tlN7zDO7/8c9evQBjlI7AD/pFfK92qQ0vZ0Bt/m7gF7cKq/rvIqCKPvbYv8vVNPPN6V38pOlZHk2pHlbf3pJgRfZ0ytJ8uklmo4NqqgZvnyqQ5vCcNiXx8NWJdSTH906wtnTZd0KCI4F29JGk8vGjWQNkitdkNex0KZMhvSieSwLYWgWWMmZLbRsSkjZ3zVA46q+XSBpdzx9+fLpbWR1MySgDgH4u5iQt4eIX9kp6PPqVsOMC1zXxTHM8zAS81mbdHyGdh1g2w4OUOezW0/DRL6a9wwWMUGcj97O3TW3nU9owJeyrawf2up2zA/4YnfQeNs7G2YjhgpBu/uDn9BSs6Lp5RpSBSjz2Ei/6xy7rTdR1soXY8MevH4+CweEdhflVQ2zPuN5JEF5OOXitMehHOMzJAcY1+ZcluD+hOK7P3PIBJHHMAsTv0c34QRWGdJsq7TeC8nWxb1LpXgLhowcvVzjhwCctqeVme2tzVG7onVFnE5kcr0dUdwduCSPwkAfh2N3uX8Ama8OgMVxhnGAi1IO7dI4h+IkBjCc8YGH4w5n/47lMVlqt4qK5qcgvMlDVJO3ePz7EYoj8DOLlRlhzRy7QD/J5yWn007vHO8BNx4tXTQx/RZltLppz/q16xiNNCoO0xeNEfKWO/hifR7posmKw11VmoVOss0rimYoHj6+KJLItZsoz5B4mv5v3f30Ulfu38YCsi6q8iyd/ARMUIABDE18jutP//oNeZP7Oldmolg6SVlQq/rg7YyD194JbGfv617Y14BIr9XF1bt1z09tTbDDufWSQBrF84f7itMXldcBVb2G2UndQNxVdXyeWt4QjT59LAo3bLMb/M6boJ7D0bbNPIGHrZ16NPUGIk63NX4vlyil7RPnDLVXM2zoneS3F1xJJZe8mGlP7NpBI52twSuF44pLtBCXS5fL5aqk4w5V8bO7btxN8AweHci8KaugOZwkUA//C4sPmCc4F8ThGu3jkJST7rjyd+VtWbirTOnkLC+I43q3Gpv2ZLNHbx+YeOKdnJRaczWISrpQgtBIvcWBuJADvz6wq71u3Cvtg2H84Vg9x2NxgiV/RGI+C/68dKrL1ZG4U16dKx+V8pVzXexCPRshvjykGOqUV0Gie2rTD4xly+qJFMnlJdzfFobe2bvwGN7v4BKkwW6voDrP6PLf8BmvCeGUOrYJgCPvy6fPX79+2+vszAXeY7Rvjf65CX3TgFGKnOaU7T1hQlGNKWkVk4aLUQtJ7U5ZqhhIfuOt71K/kTJ5bfXUMrnHm/B2Ms6qJZ4J/kZtD/L6ILWLaKcnAiFfrANxH9fuEI/tSjz9DQfzU27ZlCM2AM4zeGqaJ8ksgN+LwYD2qCm7sZ9AskYVZXJDFxkTzCW6VBQDVTjCdiplYSWIKOvbzk9DSJCRi5ArxMpTOG7ICHHfHRtyf9gyu9Tk/XVkcztRuRZVfmYX1QwLdKd6Jo2qKq/mIPxDBXYIknB9/xnsnBCMLc0PA8Nsh9Aq8hNKM0TAXvOy4nmUIBzI3g3cyEQ4CV0Y7YDcepKEIPm01lRzGEvkdrrR7A3KVG5TnV31wp70v+HJfs7uW0VYz4P3uwxMA4xxCMd5Aj0G21xbTKd2ZqkeRa6vUvx4uS5Ky+jG+5VddnlFnakE2bcr3lkL/IVitGygUTWGyvP1uuhW/FpMjvWIMdxyleEbsQ+MxRx62VQCNtE8dG8aMPApzOOewk3BuTOaHNPwHBMUYVSCWdDu6IXH4tCIDHo3JJ9cT2gMScjLdXMv0HroFwcWIWkoEtCbJ0Zqd8CIFd8ts5LOB1Pto1kzNmuqyGkfB9RwXUTeVDIHlV2EM0n+RBUmfNqnyB8kJ/PZdkWD616Do40sqU2lmo1s2GqE93y+sP3kuD6r9L09BWcEtXr/3jn2WKICHoZS30YC69ALFiOCyzqFWD9Gj8Iqrx1rRlxx82KYR7EYYAdwDkOTz4jDduUnB9E1RmNh2QqeMSkT3BJFCM3aqIVloyp2hwsO7ZR7sPPZInDy2NwRqxpIl+1e2Jb+PUE62q54r71y2j4tDqvrDFv0QGIPU+qR5LYXZQEMEvCodeYg/IkkTNsOAygCfQJVOkZyTYRujgkNoDkq3ZZmSWNctFwxtgFtxHgIES2294RWDVUyztpoc/XBSUpqIru7OqIRGyjfNozYRMEKsUEeQ8GMuOJVdjD9yYtZHL+JwMAmfMCS7BPIGZrmQdYY4dFBg4rIsehwiZjeUNwXKy93KBFXD9K9ldzFRsNXhWjtFKBfcuPeoMdLgBKNH+PezcbTVbPRoC4uLM1ezrBH4EUNHOb5Da6BW4HmIwzyZ5owReEk5/ncE7iuqvIqJpNFjVtnn183Xe+X3DXOqTGHLGxBYmgGlOW6kyDT0A0FQ6Oj7uelJ8va9agNWNDspa253G4j2SDQpKLH806YEasfDGYzxHDH5jiMfoZfjMrDdr2+rnN2cTrEbr5NJeWg11fDUBM6bATNYvhCTYu1skYWImSWkILIMeI1oXmStzfOFY4HZWXXjIyPC5XOywxt5vBK8ziCi8i9TVH0I2zwB3qTR8Rt1sWf4RHLU7lOOAOTk7B0q4uoiqRzb04dxVR+6KTKQuba/eLSWYudZp2IqGNXTrDWK7zsJ69oEqlGWsgaNR2CyMotr6yzLXmeEZd9O0qcfBbCrxKwzaE2S3PPWHaqB1Tix4TYVC3oobzaYciKdIf2HN0bA201lNC0TWGpY7gAhQniNhiUmlj0m+IErVJqYx98DzsoI3IgofvKFp2TSwUzrNAHdtNWAHbtJMnbD3GGP5Gcqj7cQ230GVRDdq+We9/rPAlHbpoUsztroOOB9xJ96O97Mil7g5YN2SM0QVGD845EKkLaiKer0lo9umYWphnuLGSbR4217xamCKg5thglzeMkVVa081C+04FZxsU9zH5GjI74273ak5VPSpkEZC61jXsX3kRO93pFOi9RrNTw7YiE2g2J9zjXI41zrHhkLWJUYYbLprGvnjey9AKxw2t143eMvpxhlcEUQitQ5B9hjt9rwSyLcQRmu0/gyIuYf3XDpj0JK0zPi3Spaq686xRX2hyYmPe3w96uR0kO3dWOF2nfE+zbDU2j0ZExahAO0eUGkcZ2qR/GYJ1ytIzshBl2GGX142xzDZctqGZVLX9WglGS8iifpp5xoiLiyMEV66PT9LRFjXjtbijkUm2QlBV2ZUst1MU1lqsTHflMq+3D+5E9nPl86y2RqtZD7M7wPdL640nId4V19cbVjp3FsIua1+azVnDeycAowH2GdZ+RFXKccYw2K2tvIt41cc3t9WofFxdraY7a3T22Q7UgTrVfF8iFCLc3NhfEIYgSos28usZodc2XkJaqEpax1C6wqTTmr9iMmRy3aQE3+RxyXyVgn/Eo0mOfUqM0nt0fEFmrW8ijO5y6WvTZPdnIer1faI220uK69Fb9HQIDy66P1vp2wskc2oliTiYWxPT7g+oztlDt77cFpZUtnx9n2Fxit5kbvi6ZfoQL/IEcTNEeQ9nuMzJDVaXON0hacQdEqMu9mwYqEEKNuS7jlA3urpwtLdO8HTyg11VhIvdlstucVsHZb1cCvdld/EE5uiaU8/qmPGStO/HW59CMsikbCe1qHsTfVWCSwgBFcs84TVsYtb/wTmEa3ZBlMV7rUsdzRQrprbxsOfVmub3e+inEblFDZ0ZcNZo4XlfQ+TiW2MI8bPTqpJ1ZbMkOtKx7uj+ihTJj/qagqSJ3KmsT4D4O+ghz/JkmbFOUg/ko9oyV7fCKElgWRCJB9+vad/NhEEy97K17fIFKoO8bwWCEHbSVoX68jUKKqblKmKPpjr6Z9vtjWUGblrid5ODG2+uMzBpzhk1meRP5Xw+YTKtuYD9321kR5ieSMAAYzZCe/wSq66yJ177cHy718VGI3NNRJ1DaE2hiL6GsXFKbympHHlRbE5cRHhnZcuPEy20qnnnyOrbsrZMM7H7m+PTMCPZ+D45gxqpYUYE0alO4arMMVLNoficFYzbj0AT2jJjdX5IOOR1Iub5Yw7nVfDG+E5dSSzt/XzZxx9/pXkzWx3QzevTyllbY5qZfa6wsBEsX4mjosTGlZNUmZTIwF5ILuZIyY4VsGnoNMhfA3y5om4fxT1qwizEc7fvPWPlObVSWByKLURdzS/6O8jc33Yr1vnS1K3TYtrfTfViJU5w39fFarKHTIZBzU1lsriNJHGh/iBfnNRZvaBeh90oUSYowzOKYTwUcgIspef4Iv/kjvSnhsz0UEM84qxUGuSbt9WTPbomtWSH6Xj4KZ+hI3jyh8aqNUFWtRx6lMFCTfaX36OWshCVvL73rORK1cXlSeyQTT9itxw2gp2jAQcmMWPR1/B+BkKVRG8PxZ+SSquwUVd+JKh72ayVfQtZKWPDn1Cx753yXiMO5d1RLbp2Oijjs6sZrG4yrUlyyyvVK7/B7appXh0MiSLTMgSC8yoeCGZGmaJN6mn5tEz6unXk7XfcRxvhzWZgkAQ04+xkR5yiMuqrp3V2IWhe9VyDARgznq6QhfI2DzCaz1mfXWy2XxcWJETSqDDek9YpW48kh+IclsHdxT9zO/qrOtM1JO+WXcUbEqYDtpWAqT9yPgPpXNZggcdcjyKdU2uPoqaGeD1qr3jc3K9G3JunvLwQyWLSRGLcAAhU+SEN21pxoX+pqHEFNgnNnMfJdUTOcm3lsyjOkjDvKOS1Iww4Os1j6X5PCeVeavNeBUZ9wPY98xhkF3yIsixVp3TQ3mGllVFuL1NlkN/LIN1bskuesWVGLSORW/Ha7Ellvx4B0rB256WQ26RSBiqsjI1mcgB5jeSHeDHsxI95MGUveTDVKEhVOblfe7FPVPxSEGY/1PPYpF6lumWjLofgSW9A126bCEg8dkczRUDKxpCqOvrrDN0zb0753MFSE3XlNhnEOsUaWy61OJsVFPuv59aKnjjUFnoU15aT2LIusizyrow7AbeaB6lEDZnY3j+kPJWGccFiW9Z+xOq5NYYTWcHZXQ4g11p0VpAW0XXkCWjYHl/fp2gov+sqNMVtUhL0i63ImS+UoRsdhg+dHY+lBLXWKWFBa26O2bXF2L86ISG8p9RQz8raYlVz+SQgmOdp1GPCMDF0zPAmVaE7JTqQohF6R0BQqq1BXJJZ5KT3vfryt1lJ48unmBsKyk27iKqD2J+0aCIbU4I3vckJm7pJUEnpH16NiFc+wyxpUEajhxnbmpUPvdSZ+FENOyeQzViUXpX48rdBaOHJIW+Do2Pmee2x2tGgZbblWdDoufINVO0pTbe1SMZt7tRbQy+riXlG7SwS/tXZHlI8vEnPVOdQy0N0sfh14dPBxjsptE/sRKD4gfP8XsrBPojZHEM+I46GvgzE6rsIIq7YlQfKurEku5ldouyrPoeQYUrhDRIfsDt3ZI8OgVXTWPuqutO4cNK36tO54pjAMkO6OrLpV3Tw0ZsShaQxvFvXtku1ZTL8Xg1nKJz3ggieQbEDkVxsoofJtVmcx12bbcap/Lv2aH8+XeIFVtXA1kvNuu2sM3GnHKzk22kjYKh2up/juNAdlHO8ANKpDhl6b7v0rNKPiqRvbvQEPtqfEcP4K5l/VYIcCLOc/5VwY2maM5Wvk8sAzVy5VefR+2+/jo3hJDqaZefGIjP3V2J33dmny3R7BRstU9ukqk0k5xynpmq0J3+Uvd2TJqAsu2Y3dckbUeR19/fVc6uOi0CiAuwj0s4n+QBP2cMD5Dv2Mlcxdcdyt2eJ+Pm9wYkzjWMlXF6jzdisnqrQM3V8lJ7+emtsJNAFtiUTAa4YULQX84jQub2PMSbph10OJImvG2cQr3HfbOb70jcFUWQdTblN/FNM/6cE4zREkhz1jlSOTbOLsLXCZMQC5y3AysHkcHc7+rtOYPsk89sQKUGQu2QhvLvvAlkC6NG97AO7g5FU1ZGccKwoJ5cvRgO6gaMygZoadNmEFwOe3N+ZwfK8Dc65NOwz1DHtky0aXEyPi013RJsWOEBCIWTgEoTEHhWoWq62/ue6ZHZlCRBQ0xnqxIitmCAas4tUYcd01u22xQpC2Urfw15YxakUyh18065rI6XDYx22HIohn3J9gSqxx6YnEP58QQzXRMQkkJXQP53KDtnrGI9IIlLPiXUMhRTbF3b9cNpK+7mJKQGVwZ6DR2R7wlTCe906jCzrZBsQwY/Y2uV3PCiqvAjDF2j5GUc9Y8TloqsZqVpCJ3UnbSrFSskRTk+OpbDTGkq9TRrgkLz6qgQtPbpRmBDVqgeVCPAHhVtxbtoAEV8bPzUBWi0GxrfEkrOYQ66fwXude1KazuP0hAxMYTlK4/4zZ6XeuLFt2IdyV21I9mSNunsaq3NOVGl5YdtiYIsv4DnfJBlwsyaSKUPloG+KeuSw1PdKHlpT1AmLNfPAPRhGsz7ZtzKDXZh2IZt2K9aYAM8BnUEA8o9pr/GXetX5cFsuDcknLc93a9vEoy4h3iVhbFfST1gq7Vidxi79DZnvrAJlANo3yIdNzVnZgz+NZrBbKHrEgKI4Pi+WciPC64cEUFpu2huvWSeddZ/tXtSkL9FHAOs/IqF2pRaALE+iQaWzI+iI1aECsGcbxT74b9G2lys4B8hr2MiRKyoGVuYUQ25RCw3Zoo7barmu3FK9XkXQJL/6un6pDbwbLHjhBAvd2NStI/KECM6jtOP4PrvD+gNtyN1vqKrAUdL8BEB6DRq6ZPe0q1EYkXCyClFUodXRTHblDahNOes9psUgGylGCbBky6WINePsuCy1vHQ66xzNx50V/J4X+J0Ael9bV0VTpPhYSMJcFFId+n6bNLiXANQVcOmp8jq7qgTxuQc9qUm0BRhukxmR3bR+NWxJQcbPQ+kL3T0hm1ZsrD6G6rB01x4gZs11qneUuoWUMdtap+eDbk73I92vYwwBN08z3AH75tpVLZj+2enmktMVU1356+bpByJdPfeQ14RcPdJEL4Ncnn74ZURM1CfhXMUyM3fBtL5ksygIkyYP8c5EFL3bzkk5BGFQv/+d/vyiD/mi3/73ZXw58tFpHzaZ1fkPetN/tNfPWQQ+89XwC8q6PegheXlV68Pj77aNe+moCOH26n1cv4fQOPD2ZHqePW45fFtuXxylge2r4+UVz7QS8DHlbvTzuZUzq//mS5c1LMwk7eZSAqkjsBnx+gV8+brif3u818raDxWtvfvTl5wXI3t59RwC6p8m3rS7etfj8+vIrwC+f3nh+v++Mn7SRB0fug+IPt8p42M9ro0ebx+B+KPj6xUz10IT1MSl859epz/+OvHdfDUaiGMmyjyqJoCmcfXfg2zf6GOXkCZs2m6bp9OTdsdOzh4vESIYhGJpDqT/2Z3kn0PRRMzH/9XXjnV+nmfFO4duofvcFw3+9Pw/GsDROMziKMBzmcSSLPm78ADCGAQdmKZuBOZZhGQfFMAb1Ht35UU+mTwLvOvG/3j7rZ61du3oPrG7T1K6Gfyd2FYB/v/b2Z0e+fiPvDv1A0/zJJ/5HTb8/G9MfZpwHv37F/t9uR3/txK920vxHoP7BAB/z5N8PA3s3wK/u5ccHPPziu7a5E79udvXjtv8Nk+NPH9hWyQ8s5p0//f9W/o+bda/7jU05xJ+D0lu+8nf3bnrLQqbf/wsAAP//AGQCm/08bWV0YSBuYW1lPSJyZXF1ZXN0LWlkIiBjb250ZW50PSIwODAzOjc2QzY6NUMxRDhDOjkxRkJFNDo2MTQzMzk1MSIgZGF0YS1wamF4LXRyYW5zaWVudD0idHJ1ZSIvPjxtZXRhIG5hbWU9Imh0bWwtc2FmZS1ub25jZSIgY29udGVudD0iZmUyYjQ0ODQ3ODBmNTE0MGViZDgwNzY0MjVhMTAyM2NlZGEzMjNjZjEzMTQwNWE1MzYxMDkzZjk5NTcwMThjOSIgZGF0YS1wamF4LXRyYW5zaWVudD0idHJ1ZSIvPjxtZXRhIG5hbWU9InZpc2l0b3ItcGF5bG9hZCIgY29udGVudD0iZXlKeVpXWmxjbkpsY2lJNklpSXNJbkpsY1hWbGMzUmZhV1FpT2lJd09EQXpPamMyUXpZNk5VTXhSRGhET2preFJrSkZORG8yTVRRek16azFNU0lzSW5acGMybDBiM0pmYVdRaU9pSTNOVGszTXpZeU1qUTRORFUzTXpNM01UWTVJaXdpY21WbmFXOXVYMlZrWjJVaU9pSnBZV1FpTENKeVpXZHBiMjVmY21WdVpHVnlJam9pYVdGa0luMD0iIGRhdGEtcGpheC10cmFuc2llbnQ9InRydWUiLz48bWV0YSBuYW1lPSJ2aXNpdG9yLWhtYWMiIGNvbnRlbnQ9IjEyMzIzOGJmNDFkNTYyODVlNTg5NTRkNzkzNmNlZTE1Y2Y0OThlZGVmNjRhOTFjMGEwM2UxNjZiYjcyNWJmMTciIGRhdGEtcGpheC10cmFuc2llbnQ9InRydWUiLz7sPWl32ziS3/MruOrt3t3XocT7SGLPcyfO0XGOiZ1Od8/M44IkJDGmCDYP28q8/u9bBZAiRcmmZMk5Zmc6Y5IggCoU6iYA3bsnwf8ezWhBpITM6MFgyi5oFpAslPPS/0iDQi7IZCAFLCloUhwMMpqyPCpYNn+g2o6lWbamDKSQFEROP5IruchIkkdQ9fDevXvLXU+iYlr68jmd+4wDmLKsCMoiX9v9/ZyVWUDlgIV0LYCDQZGV8Gp0iIC6wHIaA/I0lOMoOR9IFyQuqejeEx1fh3SXIBPGJjGVASkqA2micRSQImJJC+lAPS+fyC+15y9+eWrpZjxPg/zDW/bEOI9iZsofj7zLn57Oz4zy1W+Dwy5R+rt/eWZOcmeqXF4QMjl5efTLhz8c36evL5Pff3+pZuqvf53nv+pXZXby/jbd//5p+sv8+Omlb1/qVJHLN2dxMXN+zoNz7al5WvwSTZ8o9OpSG+dHt+n+2a+5+ZK9f3/++jEjR78n9uXb13KhqG/nl6n7Ss+PPyaFp3u/f3gbQPf32r2zoGDxvIiCXJ6yvGhTnMU4vSwbCqYiaToM2AyZ4ZoOoIYcha0uRMMbWtALqCeXWdxqNC2KNH8wGl0Df1SxOL0qaJaQeORn7DKnmcf7ugHWbaDUr2sJaHdNALjoOWYrMzL6IS4eloCWjJV/mBQPeQnKRlPix8wfgYRebiR7XfgsLaJZ9InGcxlbj6OYthD45w9/lKx4CAyTA2bi4YEkroa43K8eMxgjAxVRV/rbP+o3xTyl4VEZAjoBXfOeJCyZIw4v3tYvEef6dZox1G4vwmXwqmXrtm0pqq0sI3JBsoj48TpIY0qKMqNPYzJZ85ZepSAYMxh487IiQF5Aw3wZgXdlkkTJZBk4qYb5IlwDgGOGU7zS/w04R51xa4qhO5YO2ktbBg3aerkmziLMinj48750K1iO4Zq27to9sIoMSIukq6Dd1KXtapptaEZPlxOQyGLqTaY08FjCDRGQ2xOzhDRebh+TOc26TALQLNsxHNM1l2uDfIxBBR7FtdR1ZwSGEhXzNf2tJwhNwjfjdySZ0LqBqihKQ/Rru1s/l6vdmfvtztqsu00H6+y3O6Sd0vDQmIEXEP6yIj3//LOB+VVJqKk4mq6Zqv4ZJNTUdRAmxTF7YG0hoaaqmbrquOpmEpqWcexl9I+S5oVHAk4+D7T2LC02ElFTcx1N05QO095aRK+hyG1l6prJ/BfkWkM3bU03+jhpL1yrKo5t2eYe7YqpOopua2Yf+hXXRnle0tuxq2ZqhtaVjtuz63pS3Jpd18/iLuz6jy4XrvAZsEQYLTUT17/9XVxZ9vcanX9WdzNSBNO/19WrG3oFE7KoWt2go9qt+J/gt3phOZvNPVIUWeSXBe22Q7+z2y4o84LNrm/Do88bgLXf//mPZRJ3WbKNY0W45QbchV9q8qbxxp/RhGYE4mKpdp0lmCLpJxKcX4IrlEuPgWNhqvwoBj7oigcweZmukcQ8PH/ZFYgPZ4FFLpNn5W9Pwmeu8/i3dz5JutxzEWUsQUFcaQ6iE5ZBw/ELZqlpvMItXUKpFuh/wwbp7ZNdjIW8unkjCysd2qqhQ2igmX1OZp4SmJ/e/hzVNi1L0RW3pz+WTUgSfeIi5KVxTcebunZUXVFUsxvGrHQd5V7MJhMaelF/r65i6w5oRK0P4QllfZ1pEGOZhqXpZp9DUDkBNHxcZhmw7Apb+qx4GsUQci8M3ANpTOJ8EeyRIGBlsnmwx8P1FQZrIrm19nKVXVwVplg1jL4BTudhxrwgjoLzYUjyKY9KhgUlwRQYs2As9tmVFxSkS9NtMXJUzVBh1EqfC5mX/iwqhm2+yzH1kMa0oF4eTRKvTHfHxjAdV9OdPnOdMG9GQeyDIZjD4Bx4FVMCUUg9NvaaVMPu+NiObZqm5fTgI2YKiNOOIGGugvM4yoshCUMPkym74+MopqlrvfxTzVaTO/WiWcqyAqYMHR26B0QsFb2gzQgzpXE6jCnJEm/GMvCGfJgur4Udx2qh2XfCyzF1RTX6lFyBMQQX6mHIvIQVHqh8UHhe18rsgouruhAlG322QdCIgl8495CFeEoaZHvIk2tjEPiGUntAyrRUzXY2m7g1SAlG2idKmqq4hmm4fSakR8gi8B1AFc3ozKdZvgesVMUCUbs1oYSk7ZdQ4HEDh+9IqDINETHgdMzC7o6Vhj6Q3ZsY6MEqjPIZxEq7o6Mbjq5pjrW9QeNR2gW/XZi23fExNR1N2uYGtsUy+1LXmglmTDX1PrvaQkLIkcelqiLJvnBxIbTVrD4VvW6CKp6tWHhnVEzLAGfb0vpY9yKil8MgJnmeMYjqxJdEr43ZHlABiwoecB/bCilq4YKeF9gtXr47FjbMDkQzfXPTIUgTYO4E29AUCCY20yMN8ErP7nc2HEzvGeq2uNwJZziuYelAm9vRZVGwOyKuqms6ODS3Q2QcZeBz7RUdC3Sr1ucLd9GZgFeF+eOA7sHagOFTddXYVoNs5++Km5UvWjeg5eiqBaFCr9EpU6BGiOlJHg6vzSRsSxNXsQz4Z24oPKjAdo8cXcUG66ZbvZGjAIrTsRi0H8Ux+iIpmeysxQAPWzdNpzdCq2J74fR4gghL2eGdcHCBK119M9cV0+zg+Hg5WeTUd4ANNgyCHqU3y1TDnqUkW+a/PeBgqpar6OpmvNCVgT1kVVyQUdXV1d6sWOUTpzRZgM8vowKzPDvjYII11RWnVzXVEgEeZzMP+4kNXPAkXBvCqA1VAR96MxG4OuRqd3GwdUV1Hac3iVM5nFM2o6gKvD1pJnAlHF2xtM24cd/QYeCmrSrKhlZSmGronmZpFuXUK7KIrHyC2xILTQEn29Qdzd3QdVj16LyCkl1dBkADDJOpaxtmGqIkLYsVWnhjlu0BEwukU1F6066Nio6SknogINM9kcICt9KGkPDWMzLO6K4KAtBwLEu3LHMzU1WZyZocQ/xMs4fYA9AAX9Jw3A2zYl00KupUemtnZFSQFt22tvKfGmTojES7C6yqqhiPbcgeXRxS8HMvWbby/Wx7NFBmDUCmBw3uXoOQBnvy4wAymE8LM5ObGY19swBIBDCl27f0roJeuzApmSOQ/ago1UYa6Op2FNhj6KspmCYyTMvezGxkFNxJ6D2kIXckMS0iOBP97KAEP3vOysxLKA13dTIRN9swwKxtmFLja6Hn4FhFM5LNPf6BeT+iqukGRqG9+bReIhWAV2Pt9oAXfs22nZ0nby/2TtNdx7A1tXcxax82C1u8F/unmaBibIgVetDqIrQzYFzBCf65vtns1Ko1IHFQxgQXoKtVhnh3TEzLcl2rN1dxPSbmnj76iFW8gIm+mbe8Dhdlf7jYuJfG0DfjWOGbhmSeexnqlWZp376zSxwz3bStDTUO6l7cQuRVq+L3QBhXc2x3w5BmzSRVi7P2NlGOqiq2siED36BaSJrGc69e5eHThI6jYh/46Yrj6r3OxLX02lK+bsFQjqE5urvhl/LW58OC5Lt+9BDLxV1Lczbjp975y4syRCdsu/m7DdEsW9VtbbMvRh2ieT64a2GQlTP/zvDDPKTa7z8uZ+EKtrUPchvUcKciaLDNVCuJzxGv8u6m0jVUw1A2TF7XSePbZWlug51rQ1WzdyKrT+ahR+KYc9mdEcxUwOOFQLlXYSxQ6vD/3WEGHG8plr2pn0lCL2TBdth09k3ciA04ecDofdjUy9NpgivrN4zbb0Mcx3UUt3dtxjKfC9fGJ0myqbfZgtdeqn8TZrppgPO34cfEdWv6F+hu8xHnFoiatqLgKubbI5rToky9S5adj2N2eWes55rgFGkbpr2X9vbcBsEbMAE32lIUXd3Mu8jBS809nvvdHbKh6aqiK5tl8kKKORTQ5gKFpNxLeGVZmusqSv9umyrnfT0OCyAZxa9F3X26i3Vpf/JdwJL06D9kWTpjUprxhZYS5uWkcUzyKWjh+1IxpVKzVFf6+VTiiRmpYJJPpZiBNxBKUYL1xBb8KSjLQ6kgE3g/ZhnlPTx580rK0BvLckmWOdg8yKK0kIKM5TnLokmU4PZn3PvLynwghXRMs4MBvwwAQEEnWVTMDwb5lAB7yzYxsl989uPo5On79ER7Nc0/Pvvt0/vzX1j44VzV/8jJ89+OkqcfP72JX7OXz85+zt+w55QVx6Pgr1f+rz+RJ5f5h1+jy9J2Ez26SH4/vWLp5cHBQMKNIoAKeInVpvjRR3JBBLoDKc+CZqO32NBd7/LOQSL4auuRuB21dlPTkFDNN83hx3xw+GgkejsUG7DF2QXd4xzyAu9WNr7z7fKHK6cd8G3hMQMqtloMxFkO3brInPyIhT4gq4cqVJZHroLE9gkQP707ev34uff23Zuz48dnL9689t69Pzn2Phz/9PzNm5f3Xx29e3l89vbk6PGx9/b49ZMXr595L16fnh2dnBxh7dNBazM80lcGNRNdHAyuxBb2atN5C6DiQDTi+4GvanhjqYEb6qaqmGEYjlXXdyHgUwNNH49NzSCq4YahaxPbIgoEpo6htg5FWAMvyNM1MG1KLDClYagSh8AlMFxqjxWTgm84JiHofV8lZmhYjjGmTqhRywdrhSs9wrGtm34fzHwNTG1smJrt24Zt2sR1QzM0LUMlJHQU8K1MxQwoOFqBQQwH7btt+MQc65YPow+d0HVuhvlxHUjf11RLd2zbGAcaDcaur/lAQYhiAssIA6pr9pjothM4Ci51Jw5VxyoxTAJuFVBHDFNwz8rBE7JYmryW50Zv52csC6Yn0WRaYG5klM4LLJDjukSC2lJH/jZriWI6WDlxoXWYQ4gKGykhVxuLWjiaju5YrqnecBZEp3lXGLsIbtRTe2tAG53WES7b9ZJcshtwWiXatv2npQ9qswWiPm9iq6Hm+G3kvNUL3xW09Vhpgd6JlzFW7It87S5vQ8t7tVLF423AJMYHgwCtHpiaeCBNMzpesS6bcbc4+GNGcog2RxCr5CNxXM7Iw13FUTCKZmDY8xGwJRumiMxGZ/9UB6HI2Ml1x52QNBq2cMWvJxekoPUhKiPedlXy6q5plrHstn2Lxtd33jLB4DpB17uBa7rzRHfLCDSTChNxLkcBatW1k7riMqQRBE6hjAwIXscwv+CHN8UMfKDvFP4/occbECTmh9UUVBJw+LrFgwEodH6sz5hciHLhz/DpH/GJ3wyhqn1e33Cm6aCwOWAYz49Xs+tYvB840qNzyhD4lTM8Zgpo1JrJ71SqgXmCyu26vJacB9ikVZlLDwgC6ppRxfztOUyiMcXziwTSo7oAnDg+bnRd31SuK+h8GZxyzClCHJx3ka0dp3rhkJC5mPuXckySSYnS2fHdAJsRd6f5rc/CeU1rsfVTZmUh0eRCbrZDcfcdNFaeAvWiCyqeUTeA61rMY0AFP/nLlxlJH0iYXjyXseBh5VNW2imMLmpQXPcJNYjoQpcw0YgVSBf2AoGOVDWGhqSi1Xcg9Fkhs7FcjWjBJumVrEnpXDYEe8v+RI6SMbgFCfohtCrFJWXy5RT4ScLDjGSAP2ZBmSPw/DxK5YItej48hQIMSKqCRyOywCdPSSI0HS4+kDHTxJKWYaqRytgEiJYLfcijmgxBtR5ln2TS26qetKDKOLqCAOgyCoupPIbwuPbNOeCK4j4JznELdhIKdn0gfWe7vjMeP+TtHkjK9w8HG2BZQ5eBLDNpHc6IJI8vADpIgLgKUlQUWUQEi5Lq4ZGY0xrWczHDLA6larJbPAeUCYG3ozjnk0CiBBo+ESXSKsOMDZxwbSBlTJADc0RCl7QYbdGTfBVLoRxP5HFMryT8wwcMsHguVUpb/F2Tu9VP2Gr3scyLaDyXfbDblCarnS04l/Nu1cMsk40bjPFAwmMRQGp91BLPK4muJnBCZB6jHwz++4QTTAKC/Y8kyHlfmjBk1FoL3Oeq+wHaZC6FM9REh615eQR6D+iPaupgoGsV5GkUgp6pmQNZ5id2BQGRpEiqBf+gTLj04OkMwWXlXCba9zMZWCBESqquMuIki9GvyGZN/ZQUU2kcxbGclTjDmElgIXhc4cHglSMpj/WhCRf4r7pxAn5rSNpQcyVraOqSOTRsyR6a7tAYKnA15aEqLjq0gAdXHiqqPHQ0flWHhitrcDfUbbiauiwKLKjmGlAFrhoWOuIZmgEtoECDAhV6teBqir6g9dDS8VZSh4qDKKrQGNoMbQ1vsdyx4R/gq0NdCzDE1rwzeGEiPlCkDm0o0GSogiBdcaMPXRNHABV1rAhDBGw0joXC6w91CzvQ8BHKNU6qoQUwADRAQUywRxUR06G+BpjgXxgFTjq0hwFLmoQvVCQG9MaJiwOvr0PDgJfwH/DI0EVgEgID7IemhfVU7JHXV00+V3xIMGp9aJs4FlPCscCUDTUTZm9o8zmEqRBYIFE4DV1BTICL49AAJMwuzLIJc3nkwCtdEn9hnAqgA+wgG0MDCWc6Mv/vE6ox5Cuuxi4mLTlF9d6WkSXZB7UB7jSYEVCMwNdJgA5S+0EGwzShRa2yIZpMQTUdLivDdvcj6L8F8DpVc41W6djFkfgI+Rd4wCXsB6fw+GOZ/kBm6UMsi1lwIPTtj0Lf/gjqY/EWdcbB99rT7/XHi3MAv9ePRcniHEBRgjYfLmg9eXsRDlSd85hm0EFSaoYVJeAGcb8hOJcaoo7NVfOcMLnExB42kHxQYmhDhGXnD1A3A5com0vcAoIFAfNf+QCqBKpWxz/5TDZX8eHaip+yUCvVOr+KqUoPHcwqu1l9ICnBKQQHLKi2Qja77e6LS0rmaCirRlVn9RE9fGP5YlVi1SW6tJUFXOppKTKvWiRg/+83mKzid/ri2Wvv/duljkTuExBIJh4EJUv1P0skuITN0pEqfEB//jlYmQd5OiMBRCCEarZGNd8nNtXUUDOoGRg2pb5r2cFYoVbojDXFpoFvmdQhvqlS23RMkxiWqQRhd8K7UiNJKB4/JH6ePizTrkh11YBfFgXYrbZtPgMRioFFIYiYVKdp8tf0CvwiYMVFYkGEKqKHTYxkywGq1IlfJPzI2pa0zApZJBnvdXwtbtXXWPLa0GvGFlYdK29v1QtQRxS9xfx2Rh2Vu20ewf/hH6pxVcUbbYr2hCyKVRXtgvlc7VbmzT/NFGm1D3vjPmzs4xVvpGqtBgpv0O1GUeS6m1XjwvOVj0aCAw47ur9rBtomQDh2r2hSSs2t3HaWO6FCwVJZkTKcabj6DADO4EbMvbBHdQNgpIUP3XGIex1bUQJqiDXB2arlaqn2dq80CStVLgsEm1gtp3i6F6rzVNbb3vPXIX5r5G1radPAczL2Km1XFf34w4KCGwmaiS4o/iFLcqJYknICjqKqgHcbg0umyfinLTW8Ev45UXV+q/F6Uree3KoIHWLdWF5TUxE1ZVET4UKXHDVw4G2nXVHhlTaTshX/CthlEYihiICzoOMf4FRFmvmyiX/wYTkOewZWj8TL0VMZLzmGnOXxGAuZB+ac8VddtThqGrUcoEb2uMhg6mPxuCR4y4JTPUGlhaiDu8P/QJku0XBCeXtUD+tiZwTRAOvK/XUeqLQyLpT/KkJfUV21NEm1VOFK0JjMF88QcDcuc5Pl6PSflzNcNr+m//pNq4gbS5hWBR1BvaaTgv4kH+4S8dv+6FrYkvRhOpeeRcXz0v/L2vdcDaCEp1cDaV5dl62sITkD6WoWP8hTAq7yIMVhZxcg1yidBwPOL/XouCwHU3oBl5BdQqB8XkyaCSR+zuISfMd12vyaIdS6gBvY+2psDbX7KL331UaS1rZaDpCWXgjCr33ZNgcZS8Ug0JAtTEftsot5EtLoywaofhiMsYZb24NdUCCmY2zJLyg7xjXDb4Kk1uflKj83h4AhnmK2CTR4Dpa1jIuMLJjkBJhJlqttG8shydSUfipnKec3WeY/k7BNquZpjcphfVfl9irMmr7z+cxnMRCPkULm9l3iTlXCshmJ227WIiJKM7ShP2Qkyx5WybpW8rJDnUaZdVQYhmUwLXrrc//aDhrFtqR0gLcWhJ8xXxz5vg3ZG5dgifDjNYQ/fMUB3D0JH43i6Doh25QYNReOqpVZd0iWIwHhG6ML/sQHV5V3SZrHCyDfGHUA5/Pqo85d0eZtBeIbowwMscTFXHdImdMKxDdGGZQoGRfu0cvRHcuUJMB8YwTiS2Xzu6TNCw7hWyMLXx9J7tpMvWiB+VwUWu/SPBqV8eE1r653lCBwVFoRWcFSKeU+qX69R77BNPCPkbgKZFPKL1H67nzX0xqvQxEYLQq+LHdvQlKx+Y8vPGJZtIUR/UykfVzhJ9X4HXZLviCJhXCsK1+kMzvFIsxfSYWsgdCavK0SHevyEYvZxr3jixnu5giWpo3nCq5PDWwxg2cI8xD/rgm5PtvIm91hn3v8xw3kw+b+Wlr8O0v2L5ElO75KQcHQf2fImhd1hmwfKTLwJjTMxXyGZNmNXs5u/gwVTPIVZN8qdgUNJW6qDO+X9F9u8jynRrOYbRVUGxV+P2MJQ0cbJky7zi09wfP4JZKEfHWj+AmVR6Op8fm5AtCKgjuMLrax3QKVQ3H90rHaRv6s+M3Pu43PtvFfW/gcth6+/sigwH2EfDfFV0DGsxqZw/ruqydgvaYoJv5waVHrV0BOruyQiIBcpfrw6YR8UY2/FVnxlFix3mo4KaPwDr+lbEHXN4CUJLCSOFYg9Ktl/4+s6mOWJKDxJDypU2LFlGb5bY2qsotRXbPADw+bmN3lJ7izKZXeAZBXx7idAX9M+ZsRrhaZxE99fRXSdSxQORTXb5CYszK5068xW/klFTb484blDCV1qeCboS4Ny2oVcmtL/ddA4SqCOq7xWySHFyXfDI1xj1l+LX15GP5liXyKCIpNY5hsXCpNRemXN7rrym+TJ/4c6VLcjESLNCbBZ8+XvmqBPmw9/Dtj+q+dMX0rTlr8djOm30jK1KhSpne8tLA6OLMxFf6Xym1WjDU4fIs/p/IZVhV+9oxhRevvqmUKsvgdmShvHRrwhX1N/Fkbif+cDfc0m8dvxgdqPiC2sznVrwV9JWTmuEin+MtFPPRuHr+89/N1LCHZPGro82p3Wv+wTdDbxA9fReBwV2selvt9NEpIs1llZefxdQcWtHew8HFWL/g9lKL9Wt65srbnWZTIwlfiG2H0xUaY7u7le/faHVRbfnNKsmDaMsP1gS3Vi2t8xJzGYxCAjBZ141koCnknNR5Q2Hig8KBLecBSGta9C0hLRQD/I/JIwQb3uodBrCBTb5bC0wr/a/C/eIogv380QiKC3kbmuJqlh/wNP6iHIc/gj2wcPsK/rc1jrWHL+Ko+mUIULe8rOsUtkUIwOPqy2Kz2brH/d+llFC6dLtW8qUeNBw+BadxgG2+NDO8DvCicwjU9sWySr3a31LhM1jVcDJZ/+dkOJ/xdRlB7wRQiWVocDN6fPZXBz53RYsqAAhPasPMjTsea+EhtfmYIEFzix3fK+Uxa4tDFqTK4TxImK5hmbEZjfu4Jb9AcObK2Xc1Uaxj61keDcMDVNkXkt/bO5S1GJrqp8AMTQOOwJQVNwTJ/1sfetAt5zQhQxd9rx6OF2hiJ7dqsOKfzg/z+qPVGHEb0x0pl/KF1WfwsKp44BUpuCeV2fR70Tlkc4vGcp4Ifut0tGG5N5So3stLm+hbtqqhzApJGBYmjT3hg23jcfi3EGIymz3x2tdSQbwAlecrSMhVmfV2N7tbU7nsOvzquWvSyUqXiAdQ01bxCyAdGOl+pWWuY9USsW+cl2OCcf52VMVwDScVfR02nf8SjZ7Q4Fa9p+HqxzfYJVk+q1cCtbvOUxjH/Le91w1sa2RJdG0H4PwAAAP//ALcASP88aW5wdXQgdHlwZT0iaGlkZGVuIiBkYXRhLWNzcmY9InRydWUiIGNsYXNzPSJqcy1kYXRhLWp1bXAtdG8tc3VnZ2VzdGlvbnMtcGF0aC1jc3JmIiB2YWx1ZT0ia1BhY05iNUg0c0tSYXBZRSt6VEpXaFVQdjkzemlXUnBRK1hDdDVUWUc4RXhGODVsczJXL0lGMDBMUk9STmF2cXhJUWRCckRnUGNUbUR4VFJmblJ2Snc9PSIgLz7sfWt33LaS4Hf9CmzfczMz54YU3w9Z0hzHiePMyE7OtePZnS9akESrGZHNviS7JTlnvu4/2j+0v2SrAJAEHy11y3JsZ9rHYpN4FoBCoapQKBwR+e80Xa7WNanvVuxstkiThC1nJM5oVZ3Nfqu0Kq2ZVjFaxgsN02jzlGXJjCxpDukxZEbOj4jy77TaXJHbPFtCAYu6Xp0cH9/c3Og3tl6UV8eWYRjHkGJGbtKkXpzNLGtGFiy9WtTwbswILVOqCTig/HLNWmjyUjMhKU1Y2UB0ze60CmIXs/PTFa0XZJ5m2dlsWSwhW1WXxTUA+ZfQD5+HL2akWNE4re/OZrozI8nZ7LWtu7q7MK3Y1H1iEJuYug1Pe2PasQEfvsYDNPhbaJBM4+k0G4PhaW80TMhDeQgm/QCgHCMsPYhaGLBa09QD4l0ExHR1c6Hp4YVpYMjCVDJjJ50fHSlde5qkm6YvvituyaqAwUmLpUajqsjWNSPFhpXzrLiR/UcSDXuC/LbOV1pdaNX66opVmKUiMLQTwVpcLGuaLlk56w/q0ek6a+puSp0uoWb5KqOANP2ijo5Os7QrYZ6xW4IPKAMyzmEYa1rWIghQLgdQ2LJmJVlpBpm7gG+b9Iry5mI01j4RNAZoRsoiAyQoVvwLQTmlpKZRukzY7dlMM1v8WhbaGgLLDIAmKoh0XRcTkE21ng/5JCBtlAJ2sYJBWmk4A0o2P5sBetCa8lo0MSGVhogBUZGgqaSI6xR6W623CeJQV4syXV5DP8IMskjNbuumCWIo27HmU7eZjKbXzlF8lXU2BctfrWSrYlDLGAyRSlZG6rTGEfk74whclHdy0mc0Ylk/fJOyG0D1s5kB09P0CAICOF5BfwBUutmMbppfqRRAK9cYyjZsWSSJmHQWsXT3OfzhL5RmmA7+LgLddyn8wX8eKl43JqRSgzXxvtB64aaJsx9ICGbRrIUWUJOYmMGADKaDNEQtxNQN34JAAyFxQoQFnxgHwVDSRgs/5EANXM18H0Jxsabbrgc0R/dCBzI7QAJ1ywjeIxiyKhPKXQQfXrsEgLbcjQ0PCn/wn0OiO7qVmbrjYvWBr0QhdTMuAt1DSuT382i6tdGGRUGgi38LiBlFwP8R/Xo0Xq3K4jcW1w+jVpNwgF2/iOA+arWBT4hXJh/c5/zHlANt8MVDINELAzvXMj3dDxxeF08JxB7xsZ8NY2Gw3PcYMoxzeF+/woAPUKuI6g2mHBuBur1wRIthsByyV8NimjF+L2viENjKXDDkXNj4vSliIEzGRhuEYmL8+5BrAdbSn20ijzuYa7IkCP7wOpiu256sG0KfK6GQ9+kwUrAcDyOkTDfAx7c8tI+OTdhTYiN2hE8dICshEU+kUtD1YRgAmvYjoP95ehgbPbAg2HeoRzyeBWiFhwTDy2zdcAg+VLrHowiP1zCOPyY7+/QYlqxzwcicQhOa/qUbWOtKsSZt7VRl/ZTpm36lWY0Lptqf7xjNge0rY4xo2MtAYS+DWQPHxCqqVIXM7RYoeFTHFQw5Lr64ZmxeE+Ambkq6InEFPFG5XsbAE/U+NGjNFatnE72kQBcVJbAkgADImiRA/ftdFRdZUWrRFbBdZZ3S8o6sbiGRCObAtBF5hrk9MmbfIppcNUy+wgus6FIRBKYSixoSNqfrrKPA6pCk0CWLtCJlt6h3jOVPo8im7mOsfH9IrrIiotk2QGiWkWJOfkzrV+tIAeM5hIvAe6rfLpYkWrpEllGLsiK+Ft280WiWXi21HDJkwF/9v//zf9VCR4O9vfRPNv6JxrF5o82LeF2N0QG/2k76N/ggdTE1II9vPr7Q86PT4yyFvjg9Xmf4c5+cAXOqZBXg2lYx4wEhQ7K9Y14eJIz7xBrOpEvg1cYr/VwxgCShiOBvCiKzsoRLCdB1RALe9YNot9LsNOmIkUzdUP4sreqouG2xAoWiXFBKRZxou2JSNGl6Tkk1LcGNEn4OAS4G2SgZrqVftyjHm3TZI7MHae4gzR2kuYM0d5DmDtLcQZo7SHMHae4gzf0ppLk/Wl4obpa4K/jnkxp4wy4PssNBdjjIDgfZ4SA7HGSHg+xwkB2+YtmhKK/oMv1AFZsWVXpQow/yw0F++IPkB4ErfyrJQTTpIDMcZIaDzHCQGQ4yw0FmOMgMB5nhq5QZDvsNB3nhfnmhsaMi7b82mfjgA4nJ50WZn7fRal/28rUh6nRqT9yULAM2fMOQ/tgkjzQHH9kV9Gm/Rb1jNCA9CGb9OCuu0uW/lqxel8vLujjDc1LVX+3nf7Vewv+rtF6sIz0ucvj45e5dASh6gURomS6vIGh1V2OQlilhUFsEPzmtYMjgJSniCn6qYl3GDF4uQVyCFQbe0pwCOYIXAKLQV8urWe+kj2zqK37O6jVbrqGa5fUAcVR5p5+dSyGLu6QstDhL4+uz2e/f/GNd1M9woakvUTQR3yfiB+jgAiLSmIs1Os8jYr4VPyt6lxU0kZlkYdC1PP1lurxcQXN6ReKRNXlOjOTQgF5xHbW4TJtCl+ss+7YDZwzk259+fHP56y+9gooyhTEEKJZXl+sy66Xnw3lyfNwN5PFwGI9Hg3iMQ3gsBvAYh+9YDN6xHLpjMXDHzbD1oFlXrOw36L/+azYaDG2RU1jeYjuYO5Yf48OkocFYaEfePGLUg6dtJI4fxm4UUj/yWRT4iRUkSRKGvm+HXhx4EyN+RZvh/ueLApa7hBTr+l+IwKJvCY+DwLcwr0m6/JavdCcVzvJ02ZsjpEnTm5K0SzKeserEwiLXK5hZ88u4pmdY1t/Wq29ovnqGYYA3ZwIz/pZxMP8GYLaxiElnOAXtF9iffLX+q/2DCEHEUULkfKsWxQ3PL8ZKFs4lViX4Er/PdpnJ/a7dcTYOiGhPGSFXAEnp+UdH1OW6gFTfIqs7zRxWf5jN/91m8yPGfEmzO2hRpfMEDw83sq9XMGT9QcFZDzNVzU1jxIpeMg4RmlNXIj2B5ZzQOAZE7tfM1/te1maGnxx/k4ku5tP5m6v6GQ9pJzgP4cOHs/uZpCUnEsKGjpxInMxacvdnQSjDs0LDsWnMgsBivhfEpm/5lmvQKLTjeG6ZlmfPE4PFhhEbsTN3IsYoDYzAZIYbJH2EOh+gl+zGPv3uCLzCtimMnuTVRJc3rG+7EHAWDa3nuUpYK+bcdB1wseWDcSCRY+XcKoqZIiv/x8vCEnh3rH6jt0i+M2i+sMivtDmeMO+Zw/NamyMIinAhUjYRE0pXnoCIZHMYBkJ+/13EvUHREEbl/KifgxPn5oz9abSu62LZKw3Grqi4RKZ8zuSBfpG+L8x8n1Z5WlVCespZVQHe9JW4EwKFqu14WOOg6EN4r2IO6MB8BbLDsh4IKUNVyW3Tb/fqKmzdtwg++gohw0OdJPH00Mls3bJQCWmpqodW83AR4lvAU5Fhqk5BAYWFUk8xSid0skLRcYFVkoCDhLqQQE1m8CSdguOop+EQgyS+mmFABAXUkMMDiKGge090we8G4Tp5RpR1xN07xNk6Ydq8pFewftYKti6LOp3LtRq4CZbNtWFqOX4RrUBWRtXIBO1Si6mOI1bTY14YtnZY3nkz57CBRwqHQ1errAElh2lGBBXhtUMleVprC1SXxLRMKo0taZSxpEuSpBXMbMTAe5MB2q+ZRpeJtiqnEyqMJqcquGuDxlf8TTrJEE4tqnjBcsqdWrwt5vUNLdlbTpFfFEmH3A0u8zZJesJXGk5oOpqiUJ/ugE4DzFGHGi3ROupRvo4R6rJrgly2oCxu+fong0HsJ6sapFeY54zTIq2KS8aWKMm6szE1bHlulTrJvS7IYiOh4o88AXEYfkEcdmdbJOlOB5anS42TC2Bkxe88rblcPZALThdmSxPVLTauJxtvsd1EWlQyeq3dANNL5rZQri1B+qdZe5Trs9M6vrUlmHOp/pWnyLhmcydCeNia+kO3pjrS3VNkoahRyJOFFVLSqi5Zjfp5xMpVWayaNM2g0iYrsIdkjo5KcHGWaSRv1lCoxhhUtZ0YJoFiQACGJNWYm2xTNdvFoySz82GIZMqUA4tqe/PbVpuotJZMH4g8Vgupy2J5pfQKstyqjx9rohPbLmup0tnsL9so6ZYWjlnq2fkoqGs0B3PU7AtkoQh/aspk7assCXTmOoLlrGk2cK6m6stnQuvYHXtFGo0kWhPiTzUS9qUCNm/EfvR0dIdkYUWTBNpwQqzVLTGeCdp7ivpRWUmLcnVRZHW6WoHg0r1qFYnqJf5pVd5nGf9XsSb5uqpJxLjkBflQNw+ka0GXwJ6oLACpWI1iTyUxelnMiywrbsbyxmfQJvQBXUew5KXcvIPrFsgNBWz7eA3Dxc8/Xv705s8iDyYRM12XJj4DqS9yYyNymeuYZuSwxLfjwArtxDUdm9m2MTeiJPYTh/rm3HdN16KePdum9d5Ru/3lrNYwF7LemozrbwC1UFiE5U5yGOBy5Luxbpi+pps2PIzQlwuM3a0yMUYGhK8wlm6aNo+Gj+dNYVjylDuxKS4A0gIXAMUSW3ABBnIB7sYCqcSPDaLbjgcVGpbuBVgTLLqhnyGkho2Mg+tRADjAP7nuGYYNX5mOjTIMD8swILv4c+Qfz+Xjn1hdIQRjMJePORew5Iee+MYiTUzv4V+bXhNF9UMdDPV45fjm96vhwGkInWgA8i6uQ3smDyZvpAatfN/vF95TH17bxKXQQYLPgEfXU4YLJcNfCL3hwMjYYdZ1ExpWuCYvQ/QUDJzuBMDk2K8sDKPDeADR8hBE28sUaJ8rTIZNfKz8/SSbQcgbVcA6apcoaSinEPg+ke+IOf+5AeqicUUZIVvo/45EH9UshKq7sl8gpedACsn6QYJu+oFneb5l/Dei6gFN5mwe+14QGtSI7MhIWOyzmFrwF8dzCHQTz7NtM4oMN45DN7RjO5pbfsJsw4meiKp/EXSd40rDw3HmNSoAcXIUwsydhLCAcIFCkbY8H+iCGcCUDwLUGwWmS0BegQizp0GCRKhosvxQmMJ4QGZCkK1MNLQxw5G2KQC5LjTReSRQFMfHTL6HJYTBOCnIamGm+5aGRTkXAASqxmzfGUKARCrIED4QyEzzwtdty9c9L1AMokzeyA+5gcKd4154kBIo6FCcdD0HSuSNgXY4KApaDjzMoF8rGrV5sDRADgvVZkaAqXwv1FCk8/qdieZHGcSio02IhDy2xrMMi9R4mVilhlXyVCGEOmMoIRD70UIJdIMrzTT1bSSA7WjVkd+3gEhHkutvldodRa6KOKWZJMLoXFX53kNuOUZ8vaIfYAI0unaVbpsurPUEiUHFSXWJVH6LGQ2sc/a1VENSqV2cWld2WVU+blGZF+X1F7+oCEtkBpJmVgmIDyvM9o1Jk0UeDT3DtcMkShLqWswzk9i0DMYC37JMK06c0E0YgyRGDKIEdW1qeZ4ZRwadP9kK84WsMlxZgUjDkp3WFGRZLXewK4EvQwtYQZJNy6KoJiOWYCuFBg8YWz3wg+dqjGHw/AHX76GIYFoBz2oS8cTMPO97mYb2s+OLxlVyGha+vdpJRaPTM7OFJcRrS/qQ2xgMq4+a956G21pfzWkavHrFitfg7P4Wyo7/XsKIHKk7kHQPSrxk9Q3kP85ZHiE5nqL0yn5oj0wHbiCptECKe2wdYXW/bjdIFfa/Z/XW1yopWwMVoGeVbpiWM76bMrHn0Or4extI+O/odEk3O+ndth3DeHimNCViTdznOExQulF2MrAqwlO1bWlNXDDT0JD21ybyDURu35dArdvD0KmFAWOY3BF0x6ZxpdvgBEyW7lBea66Dir3mWBGOVFwkaNcb7aPCFPVBJr77cTZLja4UHlWxjMU1LMRoOFSJnaJLQfI5rl0mxc0Sl0GBepdis01+wArMaMXkV02v5FtU0mW8aMJXNL7GNUOWB6hU3OFmX0V2b8GiqK/Z3dnsisSzoUlZh0zfkjftEShhVPYtwe020rVXIinHJ2kGMI2p8Ro4IxwcXOV3wdIeHvADZIvbSxUNRWBjur0dXdvTZ+3YDE+mNRGz3qaw+u8LWM8Q0XozrT031uqoq7zRUe+y4DliS98ab+nzDXar2a8fbfz39ut1x8elzR7vv6N842oOF2I8HXef1IVDKcK0+V4+l61GdRnd2YW2PPUwiaxL6yq7Z/FRRAuJrGczvn98js++KbmaQ54LeVPUhG6AG8Xt613G8AUuSGjC0hQt8AsXFbGiPAkh49vs1Z6k7FjkGlM0Uy1uO00TqQQN4quQfM/TjFXAo7OdyFEfio4qpXtSpZ8ENPvRpS+JBn3RpEfYceAZWGCbHkOChM4mxL1mfhau4R1t0n0j7zh5nce0AsigAQlEMdDc5+ID1c2GOHAXUKA5xNOlGZINP10AKmwgZF9SIbBsdi5+HyQXlu/sRSQg/SelEyu0OinZPwD2em9ygZknqIU1Ueh2osELIcDKxNe7kQe11o46rPakDr+g6V8D44FIfBoicZVC5ynI8KS8iq+bvo9aS9++ADLi27rn+8qeDu4p6YHrbICpQSWuekLWsXjui6YM+FYTGGga43x4zUXhvlWOkGYHzAYejrXk4dieJG7aQi2wwaK9oYQvher3LgjbVl8/wKVrVD7gAV4EYaGZ752F2RnmEBMK9baqDXr2SIYo40MOuYyBRmMk1wugZONN6+G270kxezMP7TSUzwfpZ+juRT5D95NSz87mcW/aqWQdU1B7VPB2+qkk3Yl8jivuiOjVnkT0e6XuAwn9RCJejoK8Yl77pETU5ERi2g9BMOmGwJ30QiCt9tRz/i7KkRe45w80xnHehwMprXEsAJRtyn2BdB8gLApG7hkQZgTlhQEsZOucwZDOGYxX1gZa5tgU96rEjpUv9aZOgN+GDbTf8JAeLmzdDMKhtwaLBA0QwyikoMRcaAGHD408nH4PKraMUxaduhvrocdP6fO4wCFtDZtRfaZwHGFar0xnulGmprQqRBt5NNYM+kS+cQZgYTw+Nrg7Fw7MU/l4mD1rTXU09qT0CnmYnSsfX6tQLc0B9yX1MtuYzDu9AreT+MYMcf/KOtJO9yTtz2WdB7L+Scj6KqN3T07Id5JvH5CT0TYhJDD9LZWXNjwIchzkY02zb80NyYEAO6iJ8xxuWBWq8QYy28GFUuy+VERi4uxcvnyt1EM6Otpfypb5xvTD7Rd5j4wt05Elu7lUQ0jv4xHQdAQm2lcAb2DqYD8Qjh0Ihxyrp6UdBxdYX4ULrB0F7GaKnjdvD5LM/aTqTytUVyxel2l9ty+dbPKN6aTXL3KSTjZJ+EY2Ak5oxkogT6sC6NUdqYtrtrysYrrEygjuvHVfjwCvI5zVnoTzbQPpgTX7NKaoC7xf/YlVlY7n66ZtDwzkUSIDOTDjhMvUvWAo+qEO0Ane+/wudNfzNN3G3VAzsPACdMMGydODdz0McNvECQNkwAIU6QPTBuJj8OSBi5ssKCD6DXGygRWMUW508IZ1w0H7VMMJ0Ube9UVGjWd8YWKVIPOaaGGP27po6kr89wjYqDmWicV5ns+bhK/Bh1xHMRu4wLAvIpsuwJBpbdsHsT4sFnbTdNsFwst1sR63tDVc1NoGckFCy1quZ0Utp+1Ap9iQytatEFlVXRzlgPocaDPapQIxD11M4ELfQT5DZsEziWHwwkTFhUtCURcqMAKIFW+y3UNQ8eCIL9oiGv06xD5rzlGaIKLjj9TdWmg3xdeo/lrSGm7ZU+vCvstAQydm583b5DIwOp3OD5nvQ9GOkWTSrLWipXHMVjjL2G193BT7t0WdZ1vOo3+KVSQFsgSAPmZPq2LjJcTvl7ed1b4C0rhoDXmWdZlGa6ijIglbMSApy/hOpJEBNCrqy/UKCmSw1mDlZMWKVcYIav7WSyT0+8K98w65bNOBBd9vV4sP31ML72afrWxIAbK8eCTKAebZtj2RgGvpHH1k9IKcsGR6c2S20bTetQelNt4yTIOfeYIHEGKQ7v1ROmJccJcbQd/HR2d+c+ETXygVHR+PJPijVGJt07Yd4L6PfjXYOTtv3j6D5C/MO3lFaM0pj/lu0iqN0gzm5onAwmd7T59Gp3ffLGlsKknr9ZBGVZGta6AR6PUQHsKisuQWlaTEXtKM2bnsmsZMfgcyKlLyGlEz1XwDZKwmI6eLgtBW6zwXPswyxc/NPYc0lFFTHFGMqUpeaoZ0m4atGYz3Z5//1yyChWFRlOkHZK+nzqOGDxq18Dn/cLIcdXa7W8iMZ9bg9HxVasUyA57gdVFOGLe1jm3k2LaukHYcVYk2/CS3QAt83aWTk7JYoR2syNv70qobcf9xr1XNvOwFNjwDzzY2xxUoM87HuYYOXr4CKZbRDxqJdvCKrFutW798U+A9XDdMdCPhtsCcfo56+Lh/SvaeEeuZGz71mA2G6g+zZtzbCnO6f4Wd2cf28JSJ1mfr6MdagO1rqDbdoT37k4/t17Hhxmfr1Y+yC3mcHct0Bysbvx/bvf0N08/WtY/aj91/v3i6O+Uu2Md25WDv6PNN/k+xNfWIzbUtxEFGf2xvDzTQn6O3/0AF9yN09tPd3yiuPrb7B9qbz4bsn1k5tK8WbAsHIvtyx0GRByRBslCEg/POg6YI7QKgcxWPs3ivgCIrxsABl/P0FqmF6nGyTnPGnW13BxZvs23HDgdOG7fumMg6B44dUUWKzmCFpkDxC8l/8P+p9AraeLalnbKyubljxdAJIroTq0COrON1PdgW2msW8fPkUeC4jul4TuIZUUxtZgSJE7tmMg+oHxlzz4182wvYLifOZ+e/NAA2TgmgHf9D0whWRSTqEoD0hIf0kPlkY1knIXXtKGFsbsydJJ7PmWkHfhJEpmMnkROa3twxWWI7CKnvOyY1LI/5AYuSKI7RXFnTJi5TUZ1OCleTyhV0rX+2FbrCHDimzLHvu8/mBruI1TfocLNJ07vZrsX/o3tvoJAYJXUtA4WKUKAM1StcwZFH8JhxNBQSnKQZfIK0HrekjkVx/aDeQTN184PEoJt+XHON0U2KTgJbkbEoCQqSX5IrGDw0IODbTXdqPpWR/nuv58PTNHC365XXud1Em3tz6qh+z5B/ULhU4+qB7XHHnBYRT6zCI/7CacuHUjYamvQPjgKE7VEAaQn6cEtf2/rYd8E2S/97fa40hHfi4iNlnZX+k4VPi3tu/2lXypiWeHVS/1aZVuk0Ne/e8snxupkb01FaXiSdQk76hx+nkm55J0FUkvE5MzsfzJljnDCDNvadgitFcFfg30nVaN8huNhiKq6uMoY+KaapQDcf5bH+F9zVuNCq9ebpnPI1vrtp7OAw/FM4DO/7Ch86xE+XqzVeqJWvb0UvClkOeo9GijMF4bPzREn9lwqWTvYTBpCptEkvsWAJeeoOiXvZplbMuMjWOb9ltEFlDVEZeeKq9WCKzqAF4pwIB6YtHzc94WCsa2Uqdb3QbKEiqcDt2BZ6XfAew0sv4J/wwLAUF5xyzWeKTvp5DfDDsmQi0xgkUT3eMsX7A2bERDY+d4AScQZvrnF/Ek23TCRvt6agNFzV51K+SHbLjn62+c0H01E4zTJWo3/h+XwbsJIAvOTd0adGE1m4M5FFkQFy7pxH0CfsfjVSQYF7LuZSySbHJ0GI4B37ZrYdG5oh6l1TsZ2gcgGuT0jbmiRFbESvZoPiO9ns/tT9qIrO3/E1YFjeZAe1uWDBYNmMTOOM6CCJ6w8P3j3ze+JyQ/yaKfcm96crAiP6rCj7PKNoveDFxpymGM1qMLlR3J1I3BDCAdaJrNxfPNA4FZK/8KgfRMwgF5K6ZpT7mSDmrYwY5OHTj3d6PwcPf5mWVX0Bca+5F6Fe1j7I/1gzEARBNl8VKS6Lu8hnUN+gS3ofMYVx1TjTvjFOTM82/dC2bE+3HC90rMHlcGJXSdJHJJAwQv/x8kPy4/u7QbeLax0lim1Lhb6/hQs5IIaIbr++yn6N7H/7jf5wu/pPa2FEebb+z7vwhv3Pv2/i/E1x8R9K+M3ZYERX0Fb0F45XveTFumKIgz1i0lcOtDe49OiDOkA6L+8lyE8sede/1kUpZws3KK9WEaVL5P/9d8Lf+FUvL4p1lqDzbIK7bu0k611dKKtQ7vgY1r5DK5bFa/Rs/W77xTQTcJ+/KeoF6t/Q5euiuGmVJAoofSBRRn/38/c/n8i7ZeitXM1JiirWGAsGcRLEPobapHrBZJu5RyVyUyz/qSZVDDQr0wm5Wdz9K5fER329vaFYzotOiaLuAo8byivteI8W2hNi28bqdnfz1h0RAgcZ+5Nfu7ECAbzvMwvTYEfJZBMKMM6PS3Cj4lar0g/c33sDFoQ9Ey74T8iGlv+sacrVEqsyRfnmX5613LZtdWw6vk+x6chNoxPG5U4WLXSZ5lpZ1A2OncZpGaPCEAoNINUd/4F57mO3l8U1NETSlBcIaROqFSsapzUkxyOAbWhzKy9KDzjcGpvP4YXDp1UxqoyuNJF2Ro45AK1ZhIknlNAntUXEkwvVvvYQJF2dMgA1fTFdAeLh7Xm7gjJhGkHI5EQfX2+42xxHlVRvgreO7D7CTySQq3WZ/bBEhXzydzaXt1TtpMabYo65irqv5C4B24uRs09hxIC7sshKIdWsXojhIa3LySmSetTOk898chpBJ2rTMXbwrYlku2maQE4M0INwT5w05CXaPupm/IEP4saMjp+6DXuHcg1TkUPRWBikXw8dBXuTAvKWm6tUzQVn/Mx7Lqxu7cn4YAq+EvDsBWbnQwfhZYthPd0Mr6cZSp4dlqbv5cXRiA73386h3ifCNauSOVFuJRFXCW9b2CDBvCjqSW2OiID+2cdtZMt4n78Xe2GZsvij9Z8otW8cpc728+2XiWxl+7m4rVhETbP7y4RQ1Mp+LJM/sc/1EK+Ptf55efe9pJanYfT/PLz9F865C6Xs7lz743n2J4B9mq3fCe4DJ3LgRA6cyGfkRJ5a/D1Ivwfp98uTfgfpnpb3FhYHLd8tlu7deO5JMGGyqntfzdacwuApFkW9W3yl8dHR0cBz+8ISdhmw7Gk4gi2C4YWmSVyu80hhwTuaIm8+VW9UVS49JfmtZjVXrKLZsXjNE6UYcerFbC1SxDe3G8px2sIDQtVTMT3K2/hxL6EnibxHJ0twH7mfCBsFIIsTn+0hSmgvLFTtva0U71obXPl4r8XUHocHOEBTtz+2N7e0r+Knd4qFrWhJa7zJ83gq/vM38LguGVPv+GhajO8TjfzKWyfZv6aR4utP28yGy22aKz//9O2VXH3TbPH1mFaLC2gbZjFd0kxQ2fNGXlBvfxXEfGG17/ssNVB4Ils0kLW7TufdRVf1umTSdgDN7Pg9uOpVr3wYJhQKmH+7GV6tUOofC37NEsjDbbNot+Y0FoTtwpMXJTIi4sSDjN3pKN0fcZhTsUicOJw3tFrC04ZENuVz2Sh9Bcc2sR8fffiyu794h7OUSr4s7fSMgtQkwlx2OaHxoKM6hAX8yDA3ohUTVyOL6bD7pWGdvfWuF4YJQaxXyMuf3nx/+fKnix8uv/v13buf3+x9PdhXfOtX6DLDcwIrTCzDcGiCVwf70XzuM5P6CfPNKPANFlvMcMPYCVw7iv14HiaJ7fssMJrJNe1FAgkqJ2HfkuaethN+J1tzrGQ2pH/qQiVVL48i31Pi6FDnwHXg5x2dnfIYMMop7krPURSZvDx9Qnt0/q67WLx/CGLiAERzIqVn67mVwI+mnLRQGhXx2zpf1QUXB5vMWpJSwJ9JS9PdTl/z2Yy3CqLtf0PY7+++e+b+5BGbewcNs20btCcYtoutCNH5f0AKLA27Hh7YYf8BuU6TTvEjiOhk1tE4w2RbRQUtExil1d0+KNM3EMbMQmbd0GwNQOykn90XR+J1WUGf882RdnKOurUFBju138CHO/fT9lBzwKXtpo+g6k969uYTDcWnmLP9nnxwbu2FAuLc2OicmIhqzoaNjKiAfyTbtkvbA0J5pNmd3fTjHWCpp572WNnbYRT7fFwt21vfBg2SxxUmmCyhbupbc6qq32uWMVw76IbyC8P5j4bMxLBHMm1posiDz5o/I3i2umnOeJ9Yzur2mVRR8/fZ+UhdOFl/86JxGo3kWnOxTmt2/s0yqlbPBsXcZ2iMHcKvANzaHeQx8JhbQBmeg+lWmgTt8kuy8IgYQI2VJYq5L+gSdyVLBrgBLC5RsYTQWhjm4YHF4XxpcX3kIm2A9ALxO9wf7EbQJGdAF2NdKgRm6uTgSsSRDMjL6s67iXL51xgNV3cgGK+EdNyNQItKjeKyLG4mxqahTWJXRR0c3qd5sQTGzVN0q9zRkaIZtQaaUfNpJnLC+Bb5XtN4uzM7K9QD8u/fHamYPHGOEXrSxAcqdIfq5Gk9sGx9lWtX0L9NruFpRuB5GN4OzX/zZDZxkvK7evljWaxXXe/to1Up6c1e9I6bvNMbDQSp2WM0DPKqa9JA3fmIg//fS5c5knVrrvxWSMjpcEarV2m3vtumL89e3ggPXXnBvXNlrKrQXqfG4yUDgxgh58i4ConmEhbp6lsQygYp+1vxCauu62Kld4zHILnKvPy8ggHmJATlG7y5+8e0frWOyPeilCmYpiU5vLuNoHUIkQCMRY/Pv3WesE0ao3yjQriDN2u8MWmLE2l/Px/S0hv0yEc1v/oDj2O+mvB0LS8o8YXvbLPznW1J39nWAg+d+bGmo29Uw4HqXIwLfPTsavFTor7rq56gHel02+v7/dNdD32OWo4NhfmQXQ8CDDBDG56+6YnyNF6eqHXCZbfRtHTSZbf54XWoG2aAt5B4ehhY1MUjc/whL5b2RJ/b6B92KhYd097rAVBRhfJvtGT/p9n/RvNz/o478rewyDHUM9/mq3Mecyw4fb53WObnp/hscKnxDIozlTQ2bDv5tEHuYAdFuELhhM9TPKxXVrgK//rupRbMSM7qRZHwQ+P17Pz/AwAA//8AowBc/zxpbnB1dCB0eXBlPSJoaWRkZW4iIGRhdGEtY3NyZj0idHJ1ZSIgbmFtZT0iYXV0aGVudGljaXR5X3Rva2VuIiB2YWx1ZT0iK0xOalNuLytHdm1Ha05nSG5uNkVCMzdmaWpzNS92a1B3RmdXSjFRdmZzemZoZFdmNjNGc2ZTa00rTEZKaFcyWEwvZ3l5YlpzaFB4NW9QR1c2YjNBc3c9PSIgLz7sG8uS3Lbxrq+AeXAuAocA39Luumy5YjklV1SOo6qcFBAEh/TyFZKzu6OUq/Ib+b18SboBcobkzEor27G95axGQ7DRABqNRj+Anidk9neR7IahqYksRd9fWslQ00YOhQTQrExTUW9VR4amKYeibVU6K9L61iLDvlWXVr9LqmKwnpDFn+gKQUuRqPLS+luzI9WuH0iiSF9sa+ipqKEzUolrRZqOtF3TNr0iMsche4ukYhA0LXqRlIreFkN+ter+or/ZmjHyIk1VfWkN3U5ZJFfFNh8uLRZY5KZQt180d5eWQxzCAqJhquuLBtCZzSxyW6RDbpD1iNiCyqZqm1rVw9TnyKaJRRN7hk70uWUIu2jFkJOsKEva7UpgirpRdZOm0O+l9U1g+4TZoS9s7sMHqHEYPCn8z7m9BsPnjZtT9w02eVd50NgBAAe4AAh8NCZMyfZfct3tDEoR6uu2L3xih5FHfP3tkEAj4YgvmGNz4Aj0wQwOPPVw33i2FweAGsz71d16MbeZX9qBrv4c0XUbHNfxbc+BVz/3bRZLO8ZGduBzageR7tpjSFkYYHsK7WdEj51Tm3lxScf+50wBRnmxzYFbFLs/qaJYp1tiz++sq4sNrsfVk4sNiMlSdC42RvixMmu66soA0+IG8caCWVMoTmuf0qxUdwRkTdFya553JUEgbbpUdZSbl23X3FLHmsa8SNUgirI/9AOCnja3NRnhtFO9Gg5vDYhnKfYkpUVdFrWiSdnIa+s4g4t+V1Wi25/Zuta4HUTfNu2unYR3vg9hj/UF7CgioElT99acNb+BHXWtEpHQvOmKd009iHKxuXAjRSQWTG8mI5UOdcnx3XEc4r77Bl8+jFYxlzgfxLpXlODVrMQoLBq2K9frTCtV78jijfagOfthj1pCc+sZYaHf3lkrOS2Ltc4DoDgZoBhUda+KJt+jhFUgVnRXl6rvaVuKAcV+ra71n16xCaNH8moYo39aCXkWP+9Udmnlw9D2zzabVPXXQ9PaW1DXu8SGRb9/kK2gsizk9aX1rQKpLIam2z8lTatqgtqejH1Zpywg5M8HrC8N1imbNmK96ZfcPGHuPYy1xiluXu+/azqZv8KtUBf1dtPuBwTQ8gDpxO2mEv2guk3ayH7TN7tOqs3bfhAg3ZuiEmDZNmWzbey23p7M7EsYs2xEuqJ7MREzjZkM7sqDotmMOmTcMkaPTepsRHoyKjWcGppc2I7qbjhsR9jZNGnSPWmpQ0DzJPS2EyBJnV4zbe0pNiBkW/QDoGrFh8ruoO2OGhMRqYQ9D81b6q5UTSuO3ofpBoeC3VZUW9J38mEsRxo/guefwRJdjlqxBF10XAvYzUDRjJdnrMGRlU+enFHs79fny3fwrLprixSg0r7fVe3QUK3sJ6y0EEDapPwmja/3Td4M12oPpC8V+5+gF/SnsBs9m0k3mVVZ9DtbbYIrPjM62sZpMyabclfVRNRFRTMBxg78tQwYrUdAmt83/sTGTyglf7D+Tii90uWLDQqF6JQAEu+q9krXbJoWjdHVaI4v8HuiEdQXDkYnDum6g5gagq3Rml1aWJKqBbHLRQfLcGn99bs/0sgilRryBpi9VcPclhZ1uxumobBrmHY9dM1o18UOPdQOrMI0aWpanFBVqDKdvGGzo0CJSpU3JUj2kjuf5qosi/b5e/hHcOCskbt+RuvosWshkCX4yuNq4qwXbvgDzC44DZoN8Pmq0fJ8n1O0EJxxR0xq5slRuzw5UTimfSWKermP5vWjTporjaxpUF/gMkBT0ArgY2kbSbNdWYIege0FU+qLG5gNLBRMG3FhikWdNXpS8+60ZQHRgGZg1KDRQsbRU+sUui/KAMotwkwZ1dEBWjfH1+8hjimy/aTaJpQJnKjhVoF9AikMSJuAW1hhKQtgUmXTaf1JewVUp7ipR+UHRmysHwFHjEkNHH2LEpWvdiCAsHo5J00n9EQZ1w+gzP8ouitNcoJVMy+2LKbB9X6AL1P/qWza/XPCHc7IV8Xwcpc8JV/X0l7a23ubg9VduRCgxGf+w0bVG/O2gYVUtG3AZ9iPIGAlOCm0yYBZ3U0h1Sj6+R5M+ORd/PPTf+ya4TlGY8Nb3Cfm/Zl5iFqUezAUva0RDPCpebRij+Z4xB/7kWJQW3BVFr38UQvtorHRSAusbYMbXNO8QNUaYIGJIvLsFBG84m1RgxjX27e7btlk4t+MdT+/+VxQswOmvy0m9tSwO3/44XQBaA6u46UV+hHnMpVxlAZRlLHIZYEUSZqphIWZ5/MgE5KrIM7SiGVByOIs4UkmlfKjLJEgKd8hO9Ab+kUkq+2KGyH3FPmgwGcfHoNojUQ/RLjOoT5i8QpE6HMn4UHoe6GT+Z7L3UgCyHH8KIzcIAyUHyaODLNU+dLPHClllClPummS8ci6em0Y8nEC9tsXCbAiu64YHiQTZ3EfsVDEXsADlTI3EIFkTDlCKBmmcZR5wlPSSaM4CWTsuW4oMj+QSRx7MU98cHa9kCWZtVIgs2lOrLKu/jKWfopmur29HRUTsmDX6yEeg8Ix5D5Itk4xH7FkeahhROy7InKdNHQzL1Fh6quUMxFyKdPAT5gfg5VzQ8djMgjSVMVeBOLmcy4yEBvNj7NCc1AsxxMSsyZPieE6xA/tU6LZ+iXMci2mKztnXSHSciRzbGCKYhGGvGwq1QKXIKQoBnSujU9nLd3zw11BOvmfINn6oJJU2qf07t87R5fyA6eN3PuI00ZE/vjTRoiQr6kh7UGn+BFxXri2H+HhIBkLkdRFj3CbxySwfZf4theS0PZj27MdePrUZubhQgt4iamN59YR1088+qYcSrYbwtN3qQEEgBZ7gIKn2wiMzDs0g6kDgAOA+Xi+Tm3f9AWt7cDFImG2EyGJDBpDGzvkWER4FMIH6HUBNwAKsbXuDCp8pAdAzA4BwCmg4JCxKbh27OMMANHVp/kxUsM1FY7Gt90AO+D4CnCuWWUHMAYMDaMgJdgjQ8JcwOdACX7DLPSNgQsUxYQTrGDIDOhNMxcnPj1tz4NK+AciYcc4GMHBgHrbDxCPYY8an/l6rfSUYNauHfo4F5/gXGDJ8CbB9+xQryEshaECmaJ5GBtmwrg4Dw5DwurCKvuwlp9HUOUS843nxUAOiAP1bA8Z50dU/zs9Qj4c6v2qMd3HmKh+17ZNN8z0ymeD2PaXaTNAmRq18BhMlj5YkMNDbNY51EdstIIgDrgQTuYEPI4hAONcsdRnUmZSuH6aRMzPvFSqzPGd0HVCHrHAi+IwcyPQzyAVLwxDxjj/p3g8MxZBJCKBD49Bdl4bUh8iO+dQH7HspGmg4ozFnoycWGVJmHiR5yR+FvOIhzJkIsgymUVZ6Pt+nLmJh36QC063ZICtdHyFDFkLzY8O3x+DvIi2eIisrNEesZwwHjMQC+GgcnG4DFjiJ3EgXS8LMnSXvczzIhAdEJnYS1yehlJ6mQpiziPlc+vq89df/3gZGY8BH5ucDJ0o6gcqlrO4j1hiMjAxaZqxzPFk6jLHcQLXF8JzAsESV6YKQifuxFEqGIuZDKXnqBC0ClgzplwmrKvvRo78DPYowUu3RyAwSOdDhOUE7xELShQmTpCg8Ykdnvks86QbZannuhBlMwZSw7IwSJXwOITjwnViyRzupNyTAQ95bF19AUT8uIhbJM1uGENuXX7P0ZCpv/ocH+ei7vnl8ml60Tmnvk1ocLgKnl2bL2+UAHMoRLefX2WvcwAOt/8QzIrvxR1VXQftK9X3OuYfOz6tArpEn5tvU2MRE7VPdP3qmUOiBAY8LIq3Oddpa94sjQyCWogpnVcMwm4ITV0IsPm8GtpAaBrlEGSCV7yqwag3enXs912F8TPG4aETSgg2Y4pROIaxHg/HMsSkUUCcEuJdDFgx+o+WKXWMeabOfwmBKIvEshbjb8/FE4PQfxXYno8Bs+OF776JoTsB0avO7qMYuZoXB+NxIA4zD/1lMqHOGiTOjU5GPEINEMc4nwg13UqbC2jzcjws0uKi76rxxnwmVmnRV0XfL6/AvzRAYuTrocdEv4Bo3T1IrFw8YcGveTImLAis8auIBHbslS7KCn7N8ioNCn69irEUaSyyxqJHtIggYknP4DkGT3+9wiFJpEkinh1G6yRR5548yWMqAL5h5q4U9X/+9e+BtKrT+RdDLoYx54JAaciLngxFpezlTf9MvcHyg0EoFe1BoeB9/EyhjGrlVnT1WExEXauDitFk/F/P/L71zNzwnRcmavLKaVEfbBPK7jHbXCcMiroZckxpFwmmnZv8RpscPELr6luFHhuabjT9AOwUyOge3BoyjmfPk8U+gjLjOJyQBmAk72em7OBlQBGT/UpwOrXh17fdZxLNfno+Gylz6DgTu3KYeyZtV+j0tfzubddDQIY5nqvMNpPIM5mPuVF4oa3HROb/Mq/NkDd1Nb5VTXpIQj751YROQ5v9cqLCpMnZ+yH5SCQ9jAr871BNAdbQtBCHrKzmvdM+zfmap4P/6hrxd2Ye1wZyaeiAKQ3Ed7RvC7RhpNrTAJ2fpaRO1TjAcZu+P9sOcyfNNj7z64TXTYvCjiPlWJCiS+mYGHcqh4fsc/DDoMf9M4JXDs9RE2H+4fhqoSoCLaRQck4y68YRDxHC6p3SpAEeVbRU2XBaWYoO2uBGNYqizwVoOwNe5ca7gdPePbfOxDNndFuNee8DRnFt0hgetHs67rD1FN6prpljjimHH9y343ItB1luXwAc9suLCe8Loz1AReA6rZpXYD9bPcQxe78BsYHlmRQATiVTKk0EhqgwRqHST8bKsRmITqfGZNjb35IP3eIRwWraUIIZjaqTP0iL6B3KFv6EKd+40iG2B56M6wYGBJ/cW7oZePmHyNRdd4H/cwro7yoEu2/AFcndN15OXXBkuB2FHnhuQbhuh3exPF45VI7WdP5NbPtIVRwE5hdWBxz8yiPbXzVjHnqEHOiDlksvTF+6eug8MX/+oywkgWgS5j8hYzywOQuRgNVPpbCc0+gUTg/jrjrCm9vwXlX460uWzJW8XkkWgoxgzXNud1KC/pmSFh4qcrAiYQQWYmlI9CUxWJkQWRWufphHR7vG0QTFNo/WNm+0QLCAaPdAOjgNzprG+yK0pfJY6MaZlRjzsjFVHmvyocK0j/8CAAD//wEAAP//wZyU6jf5AQA=){height=\"60px\" width=\"240px\"}" + ], + "id": "7fce9e22" + } + ] +} \ No newline at end of file