use_case_examples/mnist/mnist_in_fhe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "48790992",
   "metadata": {},
   "source": [
    "# MNIST Classification with Brevitas Quantization-Aware Training \n",
    "\n",
    "In this notebook, we show how to use [Brevitas](https://github.com/Xilinx/brevitas) to perform quantization-aware training on the famous [MNIST](http://yann.lecun.com/exdb/mnist/) dataset. We then show how the Virtual Library can be used to finetune parameters and find the best setting, before finally checking computations in FHE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "01d93a9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using rdseed seeder.\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import onnx\n",
    "import torch\n",
    "\n",
    "# Concrete-Numpy and Concrete-ML\n",
    "from concrete.numpy.compilation import Configuration\n",
    "\n",
    "# The QAT model\n",
    "from model import MNISTQATModel  # pylint: disable=no-name-in-module\n",
    "from torch import nn, optim\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "from torchvision import datasets, transforms\n",
    "from tqdm import tqdm\n",
    "\n",
    "from concrete.ml.torch.compile import compile_torch_model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9878260c",
   "metadata": {},
   "source": [
    "Here are the functions to train and test the model. Everything is very classical here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "95137753",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, device, train_loader, optimizer, epoch, criterion):\n",
    "    \"\"\"Train the model.\"\"\"\n",
    "\n",
    "    model.train()\n",
    "\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data).squeeze()\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if batch_idx % 500 == 0:\n",
    "            print(\n",
    "                f\"Train Epoch: {epoch} [{batch_idx}/{len(train_loader.dataset) // len(data)}\"\n",
    "                f\" ({100.0 * batch_idx / len(train_loader):.0f}%)]\"\n",
    "                f\"\\tLoss: {loss.item():.6f}\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f432e903",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, device, test_loader, criterion):\n",
    "    \"\"\"Test the model.\"\"\"\n",
    "\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for data, target in tqdm(test_loader):\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data).squeeze()\n",
    "            test_loss += criterion(output, target).item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print(\n",
    "        f\"Test set: Average loss: {test_loss:.4f}, \"\n",
    "        \"Accuracy: \"\n",
    "        f\"{correct}/{len(test_loader.dataset)} ({100.0 * correct / len(test_loader.dataset):.0f}%)\"\n",
    "    )\n",
    "\n",
    "    return test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67d9b236",
   "metadata": {},
   "outputs": [],
   "source": [
    "def manage_dataset(train_kwargs, test_kwargs):\n",
    "    \"\"\"Get training and test parts of MNIST dataset.\"\"\"\n",
    "\n",
    "    # Pre-transform\n",
    "    class ReshapeTransform:\n",
    "        def __init__(self, new_size):\n",
    "            self.new_size = new_size\n",
    "\n",
    "        def __call__(self, img):\n",
    "            return torch.reshape(img, self.new_size)\n",
    "\n",
    "    transform = transforms.Compose(\n",
    "        [\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.1307,), (0.3081,)),\n",
    "            ReshapeTransform((28 * 28,)),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    # Manage datasets\n",
    "    dataset1 = datasets.MNIST(\"./data\", train=True, download=True, transform=transform)\n",
    "    dataset2 = datasets.MNIST(\"./data\", train=False, transform=transform)\n",
    "    train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs)\n",
    "    test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)\n",
    "\n",
    "    return train_loader, test_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59bd35b2",
   "metadata": {},
   "source": [
    "The following function is used to test the model: it can be done either with the Virtual Library (i.e., without encryption) or in FHE. \n",
    "\n",
    "Checking things in VL is a very good habit to have within Concrete-ML since it allows to see the effect of the quantization on the accuracy, as well as to check the maximal bitwidth of intermediate values, without having to wait as long as when things are computed in FHE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "24132f48",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compile_and_test(\n",
    "    model,\n",
    "    use_virtual_lib,\n",
    "    np_inputs,\n",
    "    quantization_bits,\n",
    "    test_data,\n",
    "    test_data_length,\n",
    "    test_target,\n",
    "    show_mlir,\n",
    "    current_index,\n",
    "):\n",
    "    # Compile the QAT model and test\n",
    "    configuration = Configuration(\n",
    "        enable_unsafe_features=True,  # This is for our tests only, never use that in prod\n",
    "        use_insecure_key_cache=True,  # This is for our tests only, never use that in prod\n",
    "        insecure_key_cache_location=\"/tmp/keycache\",\n",
    "        p_error=None,  # To avoid any confusion: we are always using kwarg p_error\n",
    "        global_p_error=None,  # To avoid any confusion: we are always using kwarg global_p_error\n",
    "    )\n",
    "\n",
    "    if use_virtual_lib:\n",
    "        print(f\"\\n{current_index}. Compiling with the Virtual Library\")\n",
    "    else:\n",
    "        print(f\"\\n{current_index}. Compiling in FHE\")\n",
    "\n",
    "    q_module = compile_torch_model(\n",
    "        model,\n",
    "        np_inputs,\n",
    "        import_qat=True,\n",
    "        configuration=configuration,\n",
    "        # Note that in CML 0.4, fixing net_inputs and net_outputs to 5 will no more be needed,\n",
    "        # since it will be the default\n",
    "        n_bits={\n",
    "            \"net_inputs\": 5,\n",
    "            \"op_inputs\": quantization_bits,\n",
    "            \"op_weights\": quantization_bits,\n",
    "            \"net_outputs\": 5,\n",
    "        },\n",
    "        use_virtual_lib=use_virtual_lib,\n",
    "        show_mlir=show_mlir,\n",
    "    )\n",
    "\n",
    "    # Check max bit width\n",
    "    max_bit_width = q_module.forward_fhe.graph.maximum_integer_bit_width()\n",
    "\n",
    "    if max_bit_width > 8:\n",
    "        raise Exception(\n",
    "            f\"Too large bit-width ({max_bit_width}): training this network resulted in an \"\n",
    "            \"accumulator size that is too large. Possible solutions are:\"\n",
    "            \"    - this network should, on average, have 8bit accumulators. In your case an unlucky\"\n",
    "            f\"initialization resulted in {max_bit_width} accumulators. You can try to train the \"\n",
    "            \"network again\"\n",
    "            \"    - reduce the sparsity to reduce the number of active neuron connexions\"\n",
    "            \"    - if the weight and activation bitwidth is more than 2, you can try to reduce one \"\n",
    "            \"or both to a lower value\"\n",
    "        )\n",
    "\n",
    "    # Check the accuracy\n",
    "    if use_virtual_lib:\n",
    "        print(\n",
    "            f\"\\n{current_index + 1}. Checking accuracy with the Virtual Library \"\n",
    "            f\"(length {test_data_length})\"\n",
    "        )\n",
    "    else:\n",
    "        print(f\"\\n{current_index + 1}. Checking accuracy in FHE (length {test_data_length})\")\n",
    "\n",
    "    # Key generation\n",
    "    if not use_virtual_lib:\n",
    "        q_module.forward_fhe.keygen()\n",
    "\n",
    "    correct_fhe = 0\n",
    "    idx = 0\n",
    "\n",
    "    # Reduce the test data, since very slow in FHE\n",
    "    reduced_test_data = test_data[0:test_data_length, :]\n",
    "\n",
    "    for idx, im in enumerate(tqdm(reduced_test_data)):\n",
    "        target_np = test_target[idx]\n",
    "        q_data = q_module.quantize_input(im)\n",
    "        q_data = np.expand_dims(q_data, 0).astype(np.int64)\n",
    "\n",
    "        prediction = q_module.forward_fhe.encrypt_run_decrypt(q_data)\n",
    "        prediction = q_module.dequantize_output(prediction)\n",
    "\n",
    "        if np.argmax(prediction) == target_np:\n",
    "            correct_fhe += 1\n",
    "\n",
    "    # Final accuracy\n",
    "    return correct_fhe, reduced_test_data.shape[0], max_bit_width"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06a39bbb",
   "metadata": {},
   "source": [
    "## Settings\n",
    "\n",
    "Here, the user can change some settings. The most important ones are:\n",
    "- epochs: how many epochs during the training\n",
    "- sparsity: to define the number of active neurons in layers; make this value smaller and there will be less active neurons\n",
    "- quantization_bits: the number of bits during quantization. The larger the more accurate, but also the faster we are above the limits of Concrete-ML in term of maximal bitwidth\n",
    "- do_training: whether we do the training. If not, we use the previously saved ONNX model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fb62ea33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Options: the most important ones\n",
    "epochs = 20\n",
    "sparsity = 4\n",
    "quantization_bits = 2\n",
    "do_training = True\n",
    "\n",
    "# Options: can be changed\n",
    "lr = 0.02\n",
    "gamma = 0.33\n",
    "test_data_length_reduced = 2  # This is notably the length of the computation in FHE\n",
    "test_data_length_full = 10000\n",
    "\n",
    "# Options: no real reason to change\n",
    "show_mlir = False\n",
    "batch_size = 32\n",
    "test_batch_size = 32\n",
    "use_cuda_if_available = True\n",
    "seed = None\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "267c0c5e",
   "metadata": {},
   "source": [
    "Seeding if we want to, to try to make everything as reproducible as possible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ddb9ee3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Using seed 3108559580\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Seeding\n",
    "if seed is None:\n",
    "    seed = np.random.randint(0, 2**32 - 1)\n",
    "\n",
    "print(f\"\\nUsing seed {seed}\\n\")\n",
    "torch.manual_seed(seed);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efcd103e",
   "metadata": {},
   "source": [
    "Settings few things for the training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e14bdea4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training and test arguments\n",
    "train_kwargs = {\"batch_size\": batch_size}\n",
    "test_kwargs = {\"batch_size\": test_batch_size}\n",
    "\n",
    "# Cuda management\n",
    "use_cuda = torch.cuda.is_available() and use_cuda_if_available\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "if use_cuda:\n",
    "    cuda_kwargs = {\"num_workers\": 1, \"pin_memory\": True, \"shuffle\": True}\n",
    "    train_kwargs.update(cuda_kwargs)\n",
    "    test_kwargs.update(cuda_kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02c0a2e7",
   "metadata": {},
   "source": [
    "Managing the MNIST dataset, and splitting it into a training and test part."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b6c83939",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manage dataset\n",
    "train_loader, test_loader = manage_dataset(train_kwargs, test_kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5caa0263",
   "metadata": {},
   "source": [
    "### Definining the model\n",
    "\n",
    "The model is defined in `model.py`. You may want to have a look to this file to see how things work in Brevitas. You may also want to apply few changes here, to see what are the effects on the QAT training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e02f7dc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model definition\n",
    "model = MNISTQATModel(quantization_bits, quantization_bits)\n",
    "model = model.to(device)\n",
    "model.prune(sparsity, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84c9f5c7",
   "metadata": {},
   "source": [
    "### Running the training\n",
    "\n",
    "Below, we run the quantization-aware training, which can be quite slow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "afd06825",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing MNIST task with 2-bits in quantization and a sparsity of 4\n",
      "\n",
      "1. Training\n",
      "Train Epoch: 1 [0/1875 (0%)]\tLoss: 5.305962\n",
      "Train Epoch: 1 [500/1875 (27%)]\tLoss: 0.918988\n",
      "Train Epoch: 1 [1000/1875 (53%)]\tLoss: 0.727811\n",
      "Train Epoch: 1 [1500/1875 (80%)]\tLoss: 0.441159\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 79.66it/s]\n",
      "/Users/benoitchevalliermames/Documents/Zama/Git/concrete-ml-internal/.venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1365: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0124, Accuracy: 8822/10000 (88%)\n",
      "Train Epoch: 2 [0/1875 (0%)]\tLoss: 0.422780\n",
      "Train Epoch: 2 [500/1875 (27%)]\tLoss: 0.711815\n",
      "Train Epoch: 2 [1000/1875 (53%)]\tLoss: 0.512439\n",
      "Train Epoch: 2 [1500/1875 (80%)]\tLoss: 0.069856\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 101.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0105, Accuracy: 8959/10000 (90%)\n",
      "Train Epoch: 3 [0/1875 (0%)]\tLoss: 0.159041\n",
      "Train Epoch: 3 [500/1875 (27%)]\tLoss: 0.477454\n",
      "Train Epoch: 3 [1000/1875 (53%)]\tLoss: 0.492690\n",
      "Train Epoch: 3 [1500/1875 (80%)]\tLoss: 0.119687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0094, Accuracy: 9085/10000 (91%)\n",
      "Train Epoch: 4 [0/1875 (0%)]\tLoss: 0.139899\n",
      "Train Epoch: 4 [500/1875 (27%)]\tLoss: 0.416240\n",
      "Train Epoch: 4 [1000/1875 (53%)]\tLoss: 0.588317\n",
      "Train Epoch: 4 [1500/1875 (80%)]\tLoss: 0.074020\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0087, Accuracy: 9163/10000 (92%)\n",
      "Train Epoch: 5 [0/1875 (0%)]\tLoss: 0.138318\n",
      "Train Epoch: 5 [500/1875 (27%)]\tLoss: 0.303529\n",
      "Train Epoch: 5 [1000/1875 (53%)]\tLoss: 0.401453\n",
      "Train Epoch: 5 [1500/1875 (80%)]\tLoss: 0.173679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 99.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0086, Accuracy: 9140/10000 (91%)\n",
      "Train Epoch: 6 [0/1875 (0%)]\tLoss: 0.118950\n",
      "Train Epoch: 6 [500/1875 (27%)]\tLoss: 0.401570\n",
      "Train Epoch: 6 [1000/1875 (53%)]\tLoss: 0.463107\n",
      "Train Epoch: 6 [1500/1875 (80%)]\tLoss: 0.195206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0086, Accuracy: 9161/10000 (92%)\n",
      "Train Epoch: 7 [0/1875 (0%)]\tLoss: 0.057101\n",
      "Train Epoch: 7 [500/1875 (27%)]\tLoss: 0.327124\n",
      "Train Epoch: 7 [1000/1875 (53%)]\tLoss: 0.423646\n",
      "Train Epoch: 7 [1500/1875 (80%)]\tLoss: 0.176952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0090, Accuracy: 9115/10000 (91%)\n",
      "Train Epoch: 8 [0/1875 (0%)]\tLoss: 0.210740\n",
      "Train Epoch: 8 [500/1875 (27%)]\tLoss: 0.476865\n",
      "Train Epoch: 8 [1000/1875 (53%)]\tLoss: 0.536574\n",
      "Train Epoch: 8 [1500/1875 (80%)]\tLoss: 0.104642\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 101.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0087, Accuracy: 9124/10000 (91%)\n",
      "Train Epoch: 9 [0/1875 (0%)]\tLoss: 0.055264\n",
      "Train Epoch: 9 [500/1875 (27%)]\tLoss: 0.409403\n",
      "Train Epoch: 9 [1000/1875 (53%)]\tLoss: 0.504187\n",
      "Train Epoch: 9 [1500/1875 (80%)]\tLoss: 0.163340\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 90.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0086, Accuracy: 9162/10000 (92%)\n",
      "Train Epoch: 10 [0/1875 (0%)]\tLoss: 0.115935\n",
      "Train Epoch: 10 [500/1875 (27%)]\tLoss: 0.522204\n",
      "Train Epoch: 10 [1000/1875 (53%)]\tLoss: 0.513674\n",
      "Train Epoch: 10 [1500/1875 (80%)]\tLoss: 0.141357\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 99.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0085, Accuracy: 9151/10000 (92%)\n",
      "Train Epoch: 11 [0/1875 (0%)]\tLoss: 0.184956\n",
      "Train Epoch: 11 [500/1875 (27%)]\tLoss: 0.367457\n",
      "Train Epoch: 11 [1000/1875 (53%)]\tLoss: 0.456636\n",
      "Train Epoch: 11 [1500/1875 (80%)]\tLoss: 0.134292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 99.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0086, Accuracy: 9146/10000 (91%)\n",
      "Train Epoch: 12 [0/1875 (0%)]\tLoss: 0.100917\n",
      "Train Epoch: 12 [500/1875 (27%)]\tLoss: 0.479832\n",
      "Train Epoch: 12 [1000/1875 (53%)]\tLoss: 0.484519\n",
      "Train Epoch: 12 [1500/1875 (80%)]\tLoss: 0.143926\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:04<00:00, 71.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0088, Accuracy: 9126/10000 (91%)\n",
      "Train Epoch: 13 [0/1875 (0%)]\tLoss: 0.137374\n",
      "Train Epoch: 13 [500/1875 (27%)]\tLoss: 0.397918\n",
      "Train Epoch: 13 [1000/1875 (53%)]\tLoss: 0.570611\n",
      "Train Epoch: 13 [1500/1875 (80%)]\tLoss: 0.086632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9265/10000 (93%)\n",
      "Train Epoch: 14 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 14 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 14 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 14 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 86.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "Train Epoch: 15 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 15 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 15 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 15 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 101.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "Train Epoch: 16 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 16 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 16 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 16 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "Train Epoch: 17 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 17 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 17 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 17 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 92.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "Train Epoch: 18 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 18 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 18 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 18 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 101.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "Train Epoch: 19 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 19 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 19 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 19 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 95.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "Train Epoch: 20 [0/1875 (0%)]\tLoss: 0.153252\n",
      "Train Epoch: 20 [500/1875 (27%)]\tLoss: 0.405863\n",
      "Train Epoch: 20 [1000/1875 (53%)]\tLoss: 0.474483\n",
      "Train Epoch: 20 [1500/1875 (80%)]\tLoss: 0.131933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:03<00:00, 100.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: Average loss: 0.0075, Accuracy: 9263/10000 (93%)\n",
      "\n",
      "2. Exporting to ONNX\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/benoitchevalliermames/Documents/Zama/Git/concrete-ml-internal/.venv/lib/python3.9/site-packages/brevitas/quant_tensor/__init__.py:94: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
      "  x = torch.tensor(x, dtype=torch_dtype)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAHgCAYAAABeuZKxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZtklEQVR4nO3deXhMZ/8G8Htmkkz2SWSPbAgJEQlBhNhToYqUFq1WqPfXVlGqqy54+7ZviurKS3VBi1a1qKIhUrQqpLIoShJqCdmRXbaZ8/sjMu1MgiyTnJnJ/bmuuciZZ77nO2Hk9pznnCMRBEEAEREREalJxW6AiIiISN8wIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBGRQZkxYwZ8fHwMpi4RGSYGJKJ2ZsOGDZBIJDhx4oTYrdA/CIKAr776CkOGDIGdnR0sLS0RGBiIt956C+Xl5Xd9bf/+/SGRSLBmzRr1tkOHDkEikTTqQUT1mYjdABGRPvj000+hUqlE2bdSqcSjjz6Kb7/9FoMHD8bSpUthaWmJX3/9FUuWLMG3336LAwcOwNnZud5rMzIy8Pvvv8PHxwebN2/G7NmzAQDdu3fHV199pTF20aJFsLa2xmuvvdYm74vIkDEgEVG7VlZWBisrK5iamorWw/Lly/Htt9/ihRdewIoVK9Tbn3zySUyePBlRUVGYOXMm9uzZU++1mzZtgrOzM1auXImHHnoIly5dgo+PD1xcXPDYY49pjH3nnXfg6OhYbzsR1cdDbETUoJSUFIwZMwa2trawtrbGyJEjcezYMY0x1dXV+Pe//42uXbvC3NwcDg4OCA8PR1xcnHpMTk4OZs6cCQ8PD8jlcri5uWHChAm4dOnSPXvYuXMnevbsCXNzc/Ts2RM7duyoN6buUNKhQ4c0tl+6dAkSiQQbNmxQb5sxYwasra1x4cIF3H///bCxscG0adPUz/1zDVLd6999912sW7cOXbp0gVwuR79+/fD777/X62Pbtm3o0aOHRq+NWdd069YtrFixAt26dUNMTEy958eNG4fo6Gjs3bsXiYmJ9Z7fsmULHnroITzwwANQKBTYsmXLXfdHRI3DGSQiqufMmTMYPHgwbG1t8dJLL8HU1BSffPIJhg0bhsOHDyM0NBQAsHTpUsTExOBf//oX+vfvj+LiYpw4cQLJycm47777AACTJk3CmTNnMG/ePPj4+CAvLw9xcXG4cuXKXcPD/v37MWnSJPTo0QMxMTG4fv26Omi1RE1NDSIjIxEeHo53330XlpaWdx2/ZcsWlJSU4KmnnoJEIsHy5csxceJE/PXXX+pZpz179mDKlCkIDAxETEwMbt68iVmzZqFjx4737OfIkSO4efMm5s+fDxOThv9Jnj59OtavX48ff/wR/fv3V28/fvw4zp8/j/Xr18PMzAwTJ07E5s2b8eqrrzbhO0JEDWFAIqJ6Xn/9dVRXV+PIkSPo3LkzgNof0n5+fnjppZdw+PBhALXB4P7778e6desarFNYWIijR49ixYoVeOGFF9TbFy1adM8eXn75Zbi4uODIkSNQKBQAgKFDh2LUqFHw9vZu9nurrKzEww8/3OBsTUOuXLmCjIwM2NvbAwD8/PwwYcIE7Nu3Dw888ACA2vfTsWNH/Pbbb7C2tgYAjBw5EsOGDbtnr3/++ScAICgo6I5j6p6rG1tn06ZN8PT0xKBBgwAAU6dOxRdffIHU1FQEBwc36v0RUcN4iI2INCiVSuzfvx9RUVHqcAQAbm5uePTRR3HkyBEUFxcDAOzs7HDmzBlkZGQ0WMvCwgJmZmY4dOgQbt682egesrOzkZqaiujoaHU4AoD77rsPPXr0aOY7+1vdQubGmDJlijocAcDgwYMBAH/99RcAICsrC6dOncL06dPV4QioDXOBgYH3rF9SUgIAsLGxueOYuufqxgK1M2Fbt27FlClT1GeijRgxAs7Ozti8eXNj3x4R3QEDEhFpyM/PR3l5Ofz8/Oo91717d6hUKmRmZgIA3nzzTRQWFqJbt24IDAzEiy++iD/++EM9Xi6XY9myZfjpp5/g4uKCIUOGYPny5cjJyblrD5cvXwYAdO3atd5zDfXVFCYmJk06TOfl5aXxdV1Yqgt8db36+vrWe21D27Q1FH601T33z7PY9u/fj/z8fPTv3x/nz5/H+fPncfHiRQwfPhxff/21aGfkERkLBiQiarYhQ4bgwoUL+OKLL9CzZ0989tln6NOnDz777DP1mAULFiA9PR0xMTEwNzfHG2+8ge7duyMlJUUnPdzpOj5KpbLB7XK5HFJp4//pk8lkDW4XBKHRNe6mbkbsn8FSW91z/5zRq5slmjx5Mrp27ap+bN26FdeuXVMfBiWi5mFAIiINTk5OsLS0RFpaWr3nzp07B6lUCk9PT/W2Dh06YObMmfj666+RmZmJXr16YenSpRqv69KlC55//nns378fp0+fRlVVFVauXHnHHurW7TR06E67r7oZncLCQo3tdTM7ra2u1/Pnz9d7rqFt2gYNGgQ7Ozts2bLljqHuyy+/BAA8/PDDAGovTfDDDz9gypQp2LZtW72Hm5sbD7MRtRADEhFpkMlkGDVqFH744QeNU/Fzc3OxZcsWhIeHw9bWFgBw/fp1jddaW1vD19cXlZWVAIDy8nJUVFRojOnSpQtsbGzUYxri5uaG4OBgbNy4EUVFRertcXFx9RYqe3t7QyaT4ZdfftHY/r///a/xb7oF3N3d0bNnT3z55ZcoLS1Vbz98+DBOnTp1z9dbWlripZdeQlpaWoMXcNyzZw82bNiAcePGqdc07dixA2VlZZgzZw4eeuiheo8HHngA33///V2/x0R0dzyLjaid+uKLLxAbG1tv+/z58/HWW28hLi4O4eHheOaZZ2BiYoJPPvkElZWVWL58uXpsjx49MGzYMISEhKBDhw44ceIEvvvuO8ydOxcAkJ6ejpEjR2Ly5Mno0aMHTExMsGPHDuTm5mLq1Kl37S8mJgZjx45FeHg4nnjiCdy4cQMff/wxAgICNIKIQqHAww8/jI8//hgSiQRdunTB7t27kZeXp6Pv1L3997//xYQJEzBo0CDMnDkTN2/exKpVq9CzZ0+NXu/kpZdeQmpqKpYtW4aEhARMmjQJFhYWOHLkCDZt2oSAgACN6zlt3rwZDg4OGDhwYIP1xo8fj08//RR79uzBxIkTdfU2idoXgYjalfXr1wsA7vjIzMwUBEEQkpOThcjISMHa2lqwtLQUhg8fLhw9elSj1ltvvSX0799fsLOzEywsLAR/f3/h7bffFqqqqgRBEISCggJhzpw5gr+/v2BlZSUoFAohNDRU+PbbbxvV6/fffy90795dkMvlQo8ePYTt27cL0dHRgre3t8a4/Px8YdKkSYKlpaVgb28vPPXUU8Lp06cFAML69evV46KjowUrK6sG96Vd9+LFiwIAYcWKFfXGAhCWLFmise2bb74R/P39BblcLvTs2VPYtWuXMGnSJMHf379R71WlUgkbNmwQBg0aJNjY2Kj/PCIiIoTKykr1uNzcXMHExER4/PHH71irvLxcsLS0FB588EGN7QEBAcLQoUMb1Q9ReycRBB2tNCQiIg3BwcFwcnLSuLJ4Y1VXV2PcuHGIj4/Hjz/+iNGjR7dCh0R0J1yDRETUQtXV1aipqdHYdujQIZw8eRLDhg1rVk1TU1N8//33CA4OxsMPP4zk5GQddEpEjcUZJCKiFrp06RIiIiLw2GOPwd3dHefOncPatWuhUChw+vRpODg4iN0iETURF2kTEbWQvb09QkJC8NlnnyE/Px9WVlYYO3Ys3nnnHYYjIgPFGSQiIiIiLVyDRERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYTsRswVCqVCllZWbCxsYFEIhG7HSIiImoEQRBQUlICd3d3SKV3nidiQGqmrKwseHp6it0GERERNUNmZiY8PDzu+DwDUjPZ2NgAqP0G29raitwNERERNUZxcTE8PT3VP8fvhAGpmeoOq9na2jIgERERGZh7LY/hIm0iIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFA0jNKlYC/8ktRUFopditERETtFgOSnpm7JRkjVh7GjyezxG6FiIio3WJA0jNdna0BAH9mFYvcCRERUfvFgKRnerjbAgD+zGZAIiIiEgsDkp4JcFcAADJyS1FVoxK5GyIiovaJAUnPeNhbwEZugiqlChfyS8Vuh4iIqF1iQNIzEokE3W8fZjvDdUhERESiYEDSQz3cbq9DYkAiIiISBQOSHvp7oXaRyJ0QERG1TwxIeuifM0iCIIjcDRERUfvDgKSHurnYwFQmQXFFDa4V3hK7HSIionaHAUkPmZlI4etsA4DrkIiIiMTAgKSn1IfZeMFIIiKiNseApKd68FR/IiIi0TAg6Sme6k9ERCQeBiQ9VReQrhXeQlF5tcjdEBERtS8MSHpKYWkKD3sLAFyHRERE1NYYkPQYF2oTERGJgwFJj6mvqM11SERERG2KAUmP1c0gncniLUeIiIjaEgOSHqubQTqfV4rKGqXI3RAREbUfDEh6rKOdBRQWpqhRCcjILRW7HSIionaDAUmPSSQSLtQmIiISgV4EpNWrV8PHxwfm5uYIDQ1FYmLiXcdv27YN/v7+MDc3R2BgIPbu3avx/Pbt2zFq1Cg4ODhAIpEgNTVV4/kbN25g3rx58PPzg4WFBby8vPDss8+iqEj/1vpwoTYREVHbEz0gbd26FQsXLsSSJUuQnJyMoKAgREZGIi8vr8HxR48exSOPPIJZs2YhJSUFUVFRiIqKwunTp9VjysrKEB4ejmXLljVYIysrC1lZWXj33Xdx+vRpbNiwAbGxsZg1a1arvMeW4AwSERFR25MIgiCI2UBoaCj69euHVatWAQBUKhU8PT0xb948vPLKK/XGT5kyBWVlZdi9e7d624ABAxAcHIy1a9dqjL106RI6deqElJQUBAcH37WPbdu24bHHHkNZWRlMTEzu2XdxcTEUCgWKiopga2vbiHfaPGezizHmw19hIzfBySWjIJVKWm1fRERExq6xP79FnUGqqqpCUlISIiIi1NukUikiIiKQkJDQ4GsSEhI0xgNAZGTkHcc3Vt036k7hqLKyEsXFxRqPttDFyRpmMilKKmtw9eatNtknERFReydqQCooKIBSqYSLi4vGdhcXF+Tk5DT4mpycnCaNb2wf//nPf/Dkk0/ecUxMTAwUCoX64enp2ez9NYWZiRRdXawBAH9m698aKSIiImMk+hoksRUXF2Ps2LHo0aMHli5desdxixYtQlFRkfqRmZnZZj2q1yFxoTYREVGbuPdim1bk6OgImUyG3Nxcje25ublwdXVt8DWurq5NGn83JSUlGD16NGxsbLBjxw6YmprecaxcLodcLm/yPnQhwN0W25K4UJuIiKitiDqDZGZmhpCQEMTHx6u3qVQqxMfHIywsrMHXhIWFaYwHgLi4uDuOv5Pi4mKMGjUKZmZm2LVrF8zNzZv+BtpID3cFAM4gERERtRVRZ5AAYOHChYiOjkbfvn3Rv39/fPDBBygrK8PMmTMBANOnT0fHjh0RExMDAJg/fz6GDh2KlStXYuzYsfjmm29w4sQJrFu3Tl3zxo0buHLlCrKysgAAaWlpAGpnn1xdXdXhqLy8HJs2bdJYdO3k5ASZTNaW34J78nezAQBkFVXgZlkV7K3MRO6IiIjIuIkekKZMmYL8/HwsXrwYOTk5CA4ORmxsrHoh9pUrVyCV/j3RNXDgQGzZsgWvv/46Xn31VXTt2hU7d+5Ez5491WN27dqlDlgAMHXqVADAkiVLsHTpUiQnJ+P48eMAAF9fX41+Ll68CB8fn9Z6u81ia24Krw6WuHKjHH9mF2OQr6PYLRERERk10a+DZKja6jpIdZ7+KgmxZ3Lw2v3d8X9DOrf6/oiIiIyRQVwHiRpPfcsRLtQmIiJqdQxIBiKA92QjIiJqMwxIBqJuBul8fikqqpUid0NERGTcGJAMhKutOewtTaFUCcjILRW7HSIiIqPGgGQgJBKJehbpTBZvOUJERNSaGJAMiPqWI1yoTURE1KoYkAxIDy7UJiIiahMMSAYk4PYtR85mF0Ol4uWriIiIWgsDkgHp7GgFMxMpyqqUuHKjXOx2iIiIjBYDkgExkUnh71p7XzauQyIiImo9DEgGRr1Qm+uQiIiIWg0DkoHhqf5EREStjwHJwPBUfyIiotbHgGRg/N1sIZEAucWVKCitFLsdIiIio8SAZGCs5SbwcbACUHu6PxEREekeA5IB4kJtIiKi1sWAZIDUV9TmDBIREVGrYEAyQHUzSGc4g0RERNQqGJAMUN0M0l/5pbhVpRS5GyIiIuPDgGSAnG3kcLQ2g0oA0nJLxG6HiIjI6DAgGSCJRILuXKhNRETUahiQDNTfC7V5RW0iIiJdY0AyUDzVn4iIqPUwIBmogNszSOdySqBUCSJ3Q0REZFwYkAxUJ0drmJtKUV6lxKXrZWK3Q0REZFQYkAyUTCqBnysPsxEREbUGBiQDFsArahMREbUKBiQDxoXaRERErYMByYDxnmxEREStgwHJgPm72kAiAfJLKpFXUiF2O0REREaDAcmAWZqZoJOjFQAeZiMiItIlBiQDp16HxMNsREREOsOAZOAC3BUAOINERESkSwxIBo4LtYmIiHSPAcnA1R1iu1hQhvKqGpG7ISIiMg4MSAbOyUYOJxs5BKH2vmxERETUcgxIRqBuFukM1yERERHpBAOSEVCvQ2JAIiIi0gkGJCPAe7IRERHpFgOSEag7xHYuuxg1SpXI3RARERk+BiQj4O1gBUszGSprVLh0vUzsdoiIiAweA5IRkEkl8He1AcCF2kRERLrAgGQkeMFIIiIi3WFAMhI93HjLESIiIl1hQDISAf841V8QBJG7ISIiMmwMSEbCz9UGUglwvawKeSWVYrdDRERk0BiQjIS5qQxdnKwB8DAbERFRSzEgGREu1CYiItINBiQjUnfBSM4gERERtQwDkhGpm0E6k1UkcidERESGjQHJiNTNIF26Xo7SyhqRuyEiIjJcDEhGxMFaDldbcwC192UjIiKi5mFAMjJcqE1ERNRyDEhGhgu1iYiIWo4BychwBomIiKjlGJCMTN0M0rmcEtQoVSJ3Q0REZJgYkIyMVwdLWMtNUFWjwoX8MrHbISIiMkgMSEZGKpWgu5sNAODPbF4PiYiIqDkYkIwQF2oTERG1jOgBafXq1fDx8YG5uTlCQ0ORmJh41/Hbtm2Dv78/zM3NERgYiL1792o8v337dowaNQoODg6QSCRITU2tV2PdunUYNmwYbG1tIZFIUFhYqMN3JD4u1CYiImoZUQPS1q1bsXDhQixZsgTJyckICgpCZGQk8vLyGhx/9OhRPPLII5g1axZSUlIQFRWFqKgonD59Wj2mrKwM4eHhWLZs2R33W15ejtGjR+PVV1/V+XvSBz3cFABqZ5AEQRC5GyIiIsMjEUT8CRoaGop+/fph1apVAACVSgVPT0/MmzcPr7zySr3xU6ZMQVlZGXbv3q3eNmDAAAQHB2Pt2rUaYy9duoROnTohJSUFwcHBDe7/0KFDGD58OG7evAk7O7u79lpZWYnKykr118XFxfD09ERRURFsbW0b+Y7bRkW1EgFL9kGpEpCwaATcFBZit0RERKQXiouLoVAo7vnzW7QZpKqqKiQlJSEiIuLvZqRSREREICEhocHXJCQkaIwHgMjIyDuO16WYmBgoFAr1w9PTs9X32VzmpjL4OlkDAM5c42E2IiKiphItIBUUFECpVMLFxUVju4uLC3Jychp8TU5OTpPG69KiRYtQVFSkfmRmZrb6PlsigOuQiIiImk30RdqGQi6Xw9bWVuOhz9QLtXkmGxERUZOJFpAcHR0hk8mQm5ursT03Nxeurq4NvsbV1bVJ49sz9an+nEEiIiJqMtECkpmZGUJCQhAfH6/eplKpEB8fj7CwsAZfExYWpjEeAOLi4u44vj3rfjsgXblRjuKKapG7ISIiMiwmYu584cKFiI6ORt++fdG/f3988MEHKCsrw8yZMwEA06dPR8eOHRETEwMAmD9/PoYOHYqVK1di7Nix+Oabb3DixAmsW7dOXfPGjRu4cuUKsrKyAABpaWkAamef6maacnJykJOTg/PnzwMATp06BRsbG3h5eaFDhw5t9v5bk72VGdwV5sgqqsC57BL072Qc74uIiKgtiLoGacqUKXj33XexePFiBAcHIzU1FbGxseqF2FeuXEF2drZ6/MCBA7FlyxasW7cOQUFB+O6777Bz50707NlTPWbXrl3o3bs3xo4dCwCYOnUqevfurXEZgLVr16J37974v//7PwDAkCFD0Lt3b+zatast3nabqVuHdCaLtxwhIiJqClGvg2TIGnsdBTG9F5eOj+Iz8HCIB1Y8HCR2O0RERKLT++sgUevjQm0iIqLmYUAyYnXXQsrILUVVjUrkboiIiAwHA5IR87C3gI3cBFVKFS7kl4rdDhERkcFgQDJiEokE3XnBSCIioiZjQDJyXIdERETUdAxIRo6n+hMRETUdA5KRC/jHITZe0YGIiKhxGJCMXFdnG5jKJCiuqMG1wltit0NERGQQGJCMnJmJFL7ONgC4UJuIiKixGJDaAS7UJiIiahoGpHagB0/1JyIiahIGpHagbgbpDAMSERFRozAgtQN1M0jXCm+hqLxa5G6IiIj0HwNSO6CwMIWHvQUArkMiIiJqDAakdoILtYmIiBqPAamd4EJtIiKixmNAaic4g0RERNR4DEjtRN0M0vm8ElTVqETuhoiISL8xILUTHe0soLAwRbVSQHpuidjtEBER6TUGpHZCIpHwMBsREVEjMSC1I1yoTURE1DgMSO0IZ5CIiIgahwGpHambQTqbVQxBEETuhoiISH8xILUjvs7WMJNJUVJZg8wbt8Ruh4iISG8xILUjpjKpehYp4a8CkbshIiLSXwxI7cwwPycAwMFz+SJ3QkREpL8YkNqZ4X7OAIAj5wt4wUgiIqI7YEBqZwI7KuBobYbSyhqcuHxD7HaIiIj0EgNSOyOVSjC0W+0s0qE0HmYjIiJqCANSOzTcv24dUp7InRAREeknBqR2aLCvE2RSCTLySpF5o1zsdoiIiPQOA1I7pLA0RYiXPQDgUDoPsxEREWljQGqnht0+zHaIh9mIiIjqYUBqp+pO9//tQgEqqpUid0NERKRfGJDaKX9XG7jamqOiWoVjf10Xux0iIiK9woDUTkkkEvXZbDzdn4iISBMDUjs27PZhtp/P5UEQBJG7ISIi0h8MSO3YIF9HmMokuHKjHBcLysRuh4iISG8wILVj1nIT9O/UAQBwkIfZiIiI1BiQ2rm6s9kOpfF0fyIiojoMSO3ccP/agHT8rxsoq6wRuRsiIiL9wIDUznV2tIJXB0tUKVU4eoGn+xMREQEMSO2eRCLBcL/bN6/lYTYiIiIADEgEYNjtw2yHeLo/ERERAAYkAhDW2QFyEymyiiqQnlsqdjtERESiY0AimJvKMLCLA4Dai0YSERG1dwxIBODvs9m4DomIiIgBiW4b1q02ICVdvomiW9Uid0NERCQuBiQCAHg5WKKLkxWUKgFHMgrEboeIiEhUDEikVndVbR5mIyKi9o4BidTq1iEdSsuHSsXT/YmIqP1iQCK1vj72sDKToaC0EmeyisVuh4iISDQMSKQmN5FhkK8jAB5mIyKi9o0BiTSM4On+REREDEikadjthdqpmYW4UVYlcjdERETiaFZAyszMxNWrV9VfJyYmYsGCBVi3bp3OGiNxuCrM0d3NFoIA/JKeL3Y7REREomhWQHr00Udx8OBBAEBOTg7uu+8+JCYm4rXXXsObb76p0wap7Q33cwLA244QEVH71ayAdPr0afTv3x8A8O2336Jnz544evQoNm/ejA0bNuiyPxJB3en+h9PzoeTp/kRE1A41KyBVV1dDLpcDAA4cOIDx48cDAPz9/ZGdnd3keqtXr4aPjw/Mzc0RGhqKxMTEu47ftm0b/P39YW5ujsDAQOzdu1fj+e3bt2PUqFFwcHCARCJBampqvRoVFRWYM2cOHBwcYG1tjUmTJiE3N7fJvRuj3p52sDU3QdGtaqRm3hS7HSIiojbXrIAUEBCAtWvX4tdff0VcXBxGjx4NAMjKyoKDg0OTam3duhULFy7EkiVLkJycjKCgIERGRiIvr+HDO0ePHsUjjzyCWbNmISUlBVFRUYiKisLp06fVY8rKyhAeHo5ly5bdcb/PPfccfvzxR2zbtg2HDx9GVlYWJk6c2KTejZWJTIoh3WoPsx08x3VIRETUDgnNcPDgQcHOzk6QSqXCzJkz1dsXLVokPPjgg02q1b9/f2HOnDnqr5VKpeDu7i7ExMQ0OH7y5MnC2LFjNbaFhoYKTz31VL2xFy9eFAAIKSkpGtsLCwsFU1NTYdu2beptZ8+eFQAICQkJjeq7qKhIACAUFRU1aryh+e5EpuD98m7h/g9/EbsVIiIinWnsz2+T5oSqYcOGoaCgAMXFxbC3t1dvf/LJJ2FpadnoOlVVVUhKSsKiRYvU26RSKSIiIpCQkNDgaxISErBw4UKNbZGRkdi5c2ej95uUlITq6mpERESot/n7+8PLywsJCQkYMGBAvddUVlaisrJS/XVxsXFfaXro7YXaZ7KKkVtcARdbc5E7IiIiajvNOsR269YtVFZWqsPR5cuX8cEHHyAtLQ3Ozs6NrlNQUAClUgkXFxeN7S4uLsjJyWnwNTk5OU0af6caZmZmsLOza3SdmJgYKBQK9cPT07PR+zNEjtZyBHkoAACH03iYjYiI2pdmBaQJEybgyy+/BAAUFhYiNDQUK1euRFRUFNasWaPTBvXFokWLUFRUpH5kZmaK3VKrq7toJK+qTURE7U2zAlJycjIGDx4MAPjuu+/g4uKCy5cv48svv8RHH33U6DqOjo6QyWT1zh7Lzc2Fq6trg69xdXVt0vg71aiqqkJhYWGj68jlctja2mo8jF3d6f6/ZhSgWqkSuRsiIqK206yAVF5eDhsbGwDA/v37MXHiREilUgwYMACXL19udB0zMzOEhIQgPj5evU2lUiE+Ph5hYWENviYsLExjPADExcXdcXxDQkJCYGpqqlEnLS0NV65caVIdY9erowIOVmYorazBiUs83Z+IiNqPZgUkX19f7Ny5E5mZmdi3bx9GjRoFAMjLy2vyzMrChQvx6aefYuPGjTh79ixmz56NsrIyzJw5EwAwffp0jUXc8+fPR2xsLFauXIlz585h6dKlOHHiBObOnasec+PGDaSmpuLPP/8EUBt+UlNT1euLFAoFZs2ahYULF+LgwYNISkrCzJkzERYW1uAC7fZKKpWoF2sf4mE2IiJqT5pzity2bdsEU1NTQSqVChEREert//3vf4XRo0c3ud7HH38seHl5CWZmZkL//v2FY8eOqZ8bOnSoEB0drTH+22+/Fbp16yaYmZkJAQEBwp49ezSeX79+vQCg3mPJkiXqMbdu3RKeeeYZwd7eXrC0tBQefPBBITs7u9E9G/tp/nV2pV4TvF/eLdz33iGxWyEiImqxxv78lgiC0Kx7SeTk5CA7OxtBQUGQSmsnohITE2Frawt/f3/dpDc9VlxcDIVCgaKiIqNej1RUXo3e/9kPlQAceXk4POwbfxkHIiIifdPYn9/NOsQG1C507t27N7KysnD16lUAQP/+/dtFOGpPFJamCPGuvZzDQZ7uT0RE7USzApJKpcKbb74JhUIBb29veHt7w87ODv/5z3+gUvFsJ2NTd7r/oXNch0RERO1Ds66k/dprr+Hzzz/HO++8g0GDBgEAjhw5gqVLl6KiogJvv/22TpskcQ33c8aKfWn47UIBKqqVMDeVid0SERFRq2pWQNq4cSM+++wzjB8/Xr2tV69e6NixI5555hkGJCPT3c0GLrZy5BZX4vjFGxh6+0a2RERExqpZh9hu3LjR4Fojf39/3Lhxo8VNkX6RSCQYXndVbR5mIyKidqBZASkoKAirVq2qt33VqlXo1atXi5si/aNeh8TrIRERUTvQrENsy5cvx9ixY3HgwAH1lacTEhKQmZmJvXv36rRB0g+DfB1gKpPg0vVyXCwoQydHK7FbIiIiajXNmkEaOnQo0tPT8eCDD6KwsBCFhYWYOHEizpw5g6+++krXPZIesDE3RT+fDgB4mI2IiIxfsy8U2ZCTJ0+iT58+UCqVuiqpt9rLhSL/6dNf/sLbe89icFdHfDUrVOx2iIiImqzVLxRJ7c9w/9p1SMf/uoHyqhqRuyEiImo9DEjUaF2crODZwQJVShV+O39d7HaIiIhaDQMSNZrG6f48m42IiIxYk85imzhx4l2fLywsbEkvZACG+znjy4TLOHQuD4IgQCKRiN0SERGRzjUpICkUins+P3369BY1RPptQGcHyE2kyCqqQHpuKfxcbcRuiYiISOeaFJDWr1/fWn2QgbAwkyGsiwMOpeXjYFoeAxIRERklrkGiJuNtR4iIyNgxIFGT1QWkE5dvoriiWuRuiIiIdI8BiZrMy8ESnZ2soFQJOJJRIHY7REREOseARM3Cw2xERGTMGJCoWeoC0qH0fKhUOrtbDRERkV5gQKJm6dfJHpZmMuSXVOLP7GKx2yEiItIpBiRqFrmJDIN8HQHwMBsRERkfBiRqthG3b177M287QkRERoYBiZptmJ8TACA1sxA3yqpE7oaIiEh3GJCo2dwUFvB3tYEgAL+k54vdDhERkc4wIFGLDL99mO0gD7MREZERYUCiFqk73f9wej6UPN2fiIiMBAMStUgfLzvYmJugsLwaqZmFYrdDRESkEwxI1CImMimGdKtdrH2Ih9mIiMhIMCBRi6lvO8KARERERoIBiVps6O0ZpNPXipFXXCFyN0RERC3HgEQt5mQjRy8PBYDae7MREREZOgYk0olhdTev5WE2IiIyAgxIpBPDb19V+9f0AlQrVSJ3Q0RE1DIMSKQTvTzs0MHKDCWVNThx6abY7RAREbUIAxLphEwqwTCe7k9EREaCAYl0Ztjt244cOJsLQeBVtYmIyHAxIJHODO3qBHNTKS7kl+Hnc5xFIiIiw8WARDqjsDRF9EAfAMDK/elQ8d5sRERkoBiQSKeeGtIFVmYy/JldjH1ncsRuh4iIqFkYkEinOliZYVZ4JwDAe3HpUHIWiYiIDBADEuncrMGdYWtugoy8Uuz+I0vsdoiIiJqMAYl0TmFhiieHdAYAfHAgAzW8cCQRERkYBiRqFTMGdYK9pSkuFpRhe8o1sdshIiJqEgYkahXWchPMHtYFAPDhgQxU1XAWiYiIDAcDErWaxwf4wMlGjmuFt/DtiUyx2yEiImo0BiRqNRZmMsy5PYu06ufzqKhWitwRERFR4zAgUat6JNQL7gpz5BRXYMvxK2K3Q0RE1CgMSNSq5CYyzB3RFQDwv0PnUV5VI3JHRERE98aARK3u4b4e8OpgiYLSKnyZcFnsdoiIiO6JAYlanalMimdH1s4irT18ASUV1SJ3REREdHcMSNQmooLd0dnJCoXl1Vj/2yWx2yEiIrorBiRqEyYyKRZEdAMAfPrLXygsrxK5IyIiojtjQKI280CgG/xcbFBSWYNPf/1L7HaIiIjuiAGJ2oxUKsHCUbWzSOt/u4TrpZUid0RERNQwBiRqU6N6uCCwowLlVUqsPXxB7HaIiIgaxIBEbUoi+XsW6cuEy8grrhC5IyIiovoYkKjNDevmhD5edqisUWH1wfNit0NERFSPXgSk1atXw8fHB+bm5ggNDUViYuJdx2/btg3+/v4wNzdHYGAg9u7dq/G8IAhYvHgx3NzcYGFhgYiICGRkZGiMSU5Oxn333Qc7Ozs4ODjgySefRGlpqc7fG9UnkUjwwig/AMDXiZm4VnhL5I6IiIg0iR6Qtm7dioULF2LJkiVITk5GUFAQIiMjkZeX1+D4o0eP4pFHHsGsWbOQkpKCqKgoREVF4fTp0+oxy5cvx0cffYS1a9fi+PHjsLKyQmRkJCoqag/nZGVlISIiAr6+vjh+/DhiY2Nx5swZzJgxoy3eMgEY6OuIAZ07oEqpwqqfM+79AiIiojYkEQRBELOB0NBQ9OvXD6tWrQIAqFQqeHp6Yt68eXjllVfqjZ8yZQrKysqwe/du9bYBAwYgODgYa9euhSAIcHd3x/PPP48XXngBAFBUVAQXFxds2LABU6dOxbp16/DGG28gOzsbUmltRjx16hR69eqFjIwM+Pr63rPv4uJiKBQKFBUVwdbWVhffinbn90s38PDaBJhIJYh/fii8HazEbsmglVbWwNJUBqlUInYrRER6q7E/v0WdQaqqqkJSUhIiIiLU26RSKSIiIpCQkNDgaxISEjTGA0BkZKR6/MWLF5GTk6MxRqFQIDQ0VD2msrISZmZm6nAEABYWFgCAI0eONLjfyspKFBcXazyoZfr5dMDQbk6oUQn4MJ6zSC1xMrMQYf+Nx4iVh/BnFv9uEhG1lKgBqaCgAEqlEi4uLhrbXVxckJOT0+BrcnJy7jq+7te7jRkxYgRycnKwYsUKVFVV4ebNm+rZquzs7Ab3GxMTA4VCoX54eno28d1SQxbeV3tG286UazifxzVgzXGzrArPbE5GSWUNLl0vx4P/+w3fJV0Vuy0iIoMm+hokMQQEBGDjxo1YuXIlLC0t4erqik6dOsHFxUVjVumfFi1ahKKiIvUjMzOzjbs2TkGedrivhwtUAvDBgXSx2zE4SpWA+VtTca3wFnwcLDG0mxMqa1R4YdtJvLrjFCprlGK3SERkkEQNSI6OjpDJZMjNzdXYnpubC1dX1wZf4+rqetfxdb/eq+ajjz6KnJwcXLt2DdevX8fSpUuRn5+Pzp07N7hfuVwOW1tbjQfpRt0s0u4/snE2m4eHmuLjnzPwS3o+zE2lWPNYCNbP6IfnIrpBIgG2HL+Ch9cm4OrNcrHbJCIyOKIGJDMzM4SEhCA+Pl69TaVSIT4+HmFhYQ2+JiwsTGM8AMTFxanHd+rUCa6urhpjiouLcfz48QZruri4wNraGlu3boW5uTnuu+8+Xbw1aoLubrYY28sNAPB+HGeRGutQWp567dbbUYHo7mYLqVSC+RFdsX5GP9hZmuKPq0V44OMjOJyeL3K3RESGRfRDbAsXLsSnn36KjRs34uzZs5g9ezbKysowc+ZMAMD06dOxaNEi9fj58+cjNjYWK1euxLlz57B06VKcOHECc+fOBVB7jZ0FCxbgrbfewq5du3Dq1ClMnz4d7u7uiIqKUtdZtWoVkpOTkZ6ejtWrV2Pu3LmIiYmBnZ1dW759uu25iK6QSoD9f+bi1NUisdvRe1dvlmPB1lQIAvBoqBcmhXhoPD/Mzxm754Wjl4cCheXVmLE+ER8eyIBKJepJq0REBkP0gDRlyhS8++67WLx4MYKDg5GamorY2Fj1IusrV65oLJweOHAgtmzZgnXr1iEoKAjfffcddu7ciZ49e6rHvPTSS5g3bx6efPJJ9OvXD6WlpYiNjYW5ubl6TGJiIu677z4EBgZi3bp1+OSTT/Dss8+23RsnDb7ONogK7ggAWBmXJnI3+q2yRok5m5NRWF6NwI4KLH6gR4PjPOwtse3pMDwa6gVBAN4/kI4nNv6Om2VVbdwxEZHhEf06SIaK10HSvUsFZRj53mEoVQK+nx2GEO8OYrekl17feQqbjl2BwsIUu+eFw7OD5T1f813SVby24xQqa1ToaGeBtY+FINBD0QbdEhHpF4O4DhLRP/k4WuHh24eKVu7nWqSG7Ei5ik3HrkAiAT6YGtyocAQAD4V4YMczg+DtYIlrhbcwac1RfJ14Bfz/ERFRwxiQSK/MG9kVZjIpjl64jqMXCsRuR6+cyynGou2nAADzhvtiuJ9zk17fw90Wu+aGI6K7C6qUKizafgovffcHKqp5KQAiIm0MSKRXOtpZYGr/2otwvrc/nTMct5VUVGP2pmRUVKswuKsj5kd0a1YdhYUp1j0egpdG+0EqAbYlXcXE/x3F5etlOu6YiMiwMSCR3pkz3BdyEylOXL7J09MBCIKAF7f9gYsFZXBXmOPDqb0ha8H91qRSCZ4Z5ouvZoXCwcoMf2YX44GPj+DAn7n3fjERUTvBgER6x8XWHI8P8AYAvBfHWaTPj1xE7JkcmMokWD2tDzpYmemk7iBfR+x+Nhx9vOxQUlGDf315Aiv2nYOSlwIgImJAIv309LAusDST4Y+rRYhrxzMbiRdvIOancwCANx7ogd5e9jqt76awwDdPhmHGQB8AwOqDFzD9i+O4Xlqp0/0QERkaBiTSS47WcvUP7ffi0tvlBQ7zSiowZ0sylCoBE4Ld1bNqumZmIsXS8QH4cGowLExl+O38dTzw8REkX7nZKvsjIjIEDEikt54c0hk2chOcyynB3tPZ936BEalRqjBvSwrySyrRzcUaMRMDIZE0f91RY0wI7ogf5g5CZ0crZBdVYMonCfgy4VK7P8QppqzCW5izJRndXv8JsadzxG6HqF1hQCK9ZWdphn8Nrr158Ptx6e1qbcyK/Wk4fvEGrMxkWPNYCCzNTNpkv91cbPDD3EG4P9AV1UoBi384g+e2pqK8qqZN9k+1KmuUWH3wPEauPIw9f2SjqkaFzccvi90WUbvCgER67YlwH9hZmuJCfhl+SL0mdjttYt+ZHHxy+C8AwPKHgtDFybpN929jborVj/bB62O7QyaVYGdqFqJW/4a/8kvbtI/26udzuYh8/xes2JeGW9VKdHervdJv4sUbvGYVURtiQCK9ZmNuiqeGdAEAfBifgWqlSuSOWtelgjK88O1JAMCs8E4Y28tNlD4kEgn+Nbgzvv6/AXCykSM9txTjV/2GDw9kIPNGuSg9GbtLBWWYteF3PLHhBC5dL4ezjRwfTAnGnnnhcLGVo7JGhROXuC6MqK0wIJHeix7oDUdrM1y+Xo7vk66K3U6ruVWlxNObklBSWYO+3vZ4ZYy/2C2hf6cO2DMvHP19OqC0sgbvH0jH4OUHMfmTBGz9/QqKK6rFbtHglVfV4N19aRj1/i+IP5cHE6kETw3pjJ9fGIao3h0hlUowyNcRAPDreV4XjKitMCCR3rM0M8HsYb4AgI9/Po/KGuM7zCAIAl7feRrnckrgaG2GVY/2galMPz6ezrbm2PJ/oXhvchAG+TpAIqk93PPy96fQ760DmPd1Cg6m5aHGyGf3dE0QBOz5IxsRKw9j1cHzqFLWXiU9dsEQLLq/O6zlf687G9y1NiAdyeDtd4jaikTgKSrN0ti7AZNuVFQrMXTFQeQWV+KR/p6YN6Ir3O0sxG5LZ75OvIJF209BKgE2/SsUA7s4it3SHWUV3sLO1Gv4PukqLuT/fYsSJxs5ooLdMbGPh3rdDDUsPbcES344g4S/rgOovcXOGw/0QGSAS4NnK+aVVKD/2/EAgKTXI+BgLW/TfomMSWN/fjMgNRMDUturCxEAIJUAI/ydMS3UG0O6ObXo1hti++NqIR5ak4AqpQovjfbDM7dny/SdIAg4da0I25Ov4YfUa7hZ/vfhtu5utpjUpyPGB7vD2cZcxC71S3FFNT48kIENRy9BqRIgN5Hi6aFd8PTQLrAwk931taM/+AXnckrw0SO9MT7IvY06JjI+DEitjAGp7QmCgNjTOfgy4bL6f95A7f++Hw31wsN9PQzuh3FheRXGfnQE1wpvIaK7C9Y9HgKpAYa9qhoVDqfnY3vyVcSfzUPV7cNtUgkwpJsTJvbxwKgeLjA3vXsIMFYqlYDtKdfwzk/nUHD7KuWjerjgjQd6wLODZaNqvLX7T3x25CIm9/XA8oeCWrNdIqPGgNTKGJDEdT6vFF8nXsF3SVdRdKt25sJEKkFkgCumhXohrItDq19YsaVUKgGzNv6Og2n58OpgiR/nhUNhYSp2Wy1WWF6F3X9k4/vkq0i5UqjebiM3wf2BbpjYpyP6+XQwyCDYHKeuFmHxrtPq70VnRyssGR+Aod2cmlTnUFoeZqz/He4Kc/z2ygi9//tNpK8YkFoZA5J+qKhWYs8f2dh8/DKS//HDuLOjFR7p74WHQjxgr6Obu+rax/EZWBmXDrmJFNufGYgAd4XYLencX/ml2JFyDduTr+Fa4S31ds8OFniwtwcm9u4IH0crETtsPTfKqrBiXxq++f0KBAGwNJPh2ZFd8cSgTjAzafoC/FtVSgT9ez+qlCrEPz+0za+PRWQsGJBaGQOS/jmbXYwtx69gR8o1lFbWXvnZzESKsYFumBbqhRBve735X/evGfmY/kUiBAFY/lAvTO7rKXZLrUqlEpB46Qa2J1/F3lM56j8fAAjxtsfEPh3xQKA7FJaGP4OmVAnYcvwy3t2frp7dnBDsjkVjusNV0bJDwI9+egxHL1zHv8cHIPr2vQqJqGkYkFoZA5L+Kquswa6TWdh07DLOZBWrt/u52GDaAC9E9e4IW3PxfhBnFd7CAx8fwY2yKkzt54l3JvUSrRcx3KpSYv+fOdiefA2/ZuSj7g4yZjIp7uvhgof6emCwryNM9OQyB03x+6UbWPLDGfyZXfv3zt/VBm9O6In+nTropP7/Dp3H8tg0RHR3xmfR/XRSk6i9YUBqZQxI+k8QBPxxtQibj1/GrpNZqKiuXThsYSrD+CB3TBvghV4edm3aU1WNCpM/SUBqZiEC3G3x/eyB7XbhMgDkFVfgh9QsfJ98FedyStTbnW3keLBPRzwc4gFfZxsRO2ycvOIKxPx0DjtSam+HY2tughci/fBofy+dBr1TV4swbtURWMtNkLL4Pr25VhaRIWFAamUMSIal6FY1dqZcw+bjl5Ge+/c9xQI7KjAt1Avjg93b5IawS344jY0Jl2FrboLd8wbDy6FxZzC1B2eyirDtxNV6lwwI9rTDQyEeGNdLvw7BZd4ox8G0PBw8l4ejF66jskYFiQSY2s8TL4zya5VrFalUAkLeisPN8mpsezoM/Xx0MzNF1J4wILUyBiTDJAgCTly+ic3HLmPvqRz16eg2chM82KcjJvbxgLXcBCpBgFJV+6j7fe2vUG9XCgJU2r/XGlu3rUYlIKvwFtYcugAA+Dy6L0Z2dxHzW6G3qmpU+PlcHr5LuoqDaXlQ3j4GZ2YixageLngoxAODu7b9ta+qlSr8fukGDqXl4+dzeTifp3nz3mBPO7w5IaDVZyXnbEnGnj+y8ezIrlh4X7dW3ReRMWJAamUMSIbvRlkVvkvKxJbjV3DpetvdgHXucF+8EOnXZvszZPkllfgh9Rq2nbiKtNy/D8G52MoxsY8HJvXxgK9z653NlVdSgUNp+Th4Lg9HMgpQ8o/F5TKpBCHe9hjh74wR/s7o6mzdJicBfJN4Ba9sP4U+XnbY/sygVt8fkbFhQGplDEjGQ6UScPTCdWw+fhlHL1yHIAgwkUkhlUggkwIyiQRSqQQyqeTv36u3/f28iVRy+zW1j7rfq+tIJejZUYGnhnQx6Ct/i0EQBJzJKsZ3SVexM/UaCv9xCK63lx0eDvHE2F5uLb6OlEol4OTVQhw8l4eDafk4da1I43lHazMM7eaM4f5OGNzVSZTrVl29WY7wZQchk0qQsvg+UU84IDJEDEitjAGJSByVNUr8fLb2ENyh9Hz1ITi5iRSRAa54KMQDg3wdGx1Ci8qr8UtG7SzR4fR8XC+r0ng+yEOBYX61s0SBHRV6cYHL4e8ewsWCMnzyeAgiA1zFbofIoDT253frr0olItIhuYkMYwLdMCbQDXklFfghJQvbkjKRnluKXSezsOtkFlxtzTGxT0c8FOKBzloXVBQEAWm5Jfj5XB4OnctH0pWb6pAF1K5HG9LNCcP8nDDMzxlONvp3Y9hwX0dcLCjDkYwCBiSiVsIZpGbiDBKR/qi7ce53SVfxQ2qW+gKNQO2FKB8K8YCjtRwH0/Jw6FwesooqNF7f1dkaI/ydMczPGX197PX+9Pl9Z3Lw1FdJ6ORohYMvDBO7HSKDwhkkImo3JBIJennYoZeHHV4b2x0H/szDd0mZOJyej6TLN5F0+abGeLmJFAO7OKhDUWNvGKsvwro4QCaV4GJBGa7eLIeHvWH1T2QIGJCIyKjITWQY28sNY3u5Ia+4AjtSrmFHyjVUVCsxpJsThvs5I6yLg0FfoNPW3BRBHgokXynEkYwCTO3vJXZLREaHAYmIjJazrTmeGtoFTw3tInYrOhfe1QnJVwrx63kGJKLWoN8H2omIqEFDujoCAI6eL4BKxaWkRLrGgEREZICCPO1gLTfBzfJqjZsyE5FuMCARERkgU5kUAzo7AAB+PZ8vcjdExocBiYjIQA2+fZjt1/QCkTshMj4MSEREBir8dkBKunwTt6qUIndDZFwYkIiIDFRnRyu4K8xRpVTh+MXrYrdDZFQYkIiIDJREIlHPIh3J4GE2Il1iQCIiMmDhXZ0AAEfOMyAR6RIDEhGRARvUpfZMtnM5JcgrqbjHaCJqLAYkIiID5mAtR4B77Q03f+MsEpHOMCARERm4unVIv3IdEpHOMCARERm4wb631yFlFEAQeNsRIl1gQCIiMnB9fewhN5Eir6QSGXmlYrdDZBQYkIiIDJy5qQz9O3UAwMNsRLrCgEREZAQGq6+HxPuyEekCAxIRkREIv70O6fjFG6iqUYncDZHhY0AiIjIC/q42cLQ2Q3mVEslXbordDpHBY0AiIjICUqkEg3zrTvfnYTailmJAIiIyEuG+vC8bka4wIBERGYnBt+/L9se1IhSWV4ncDZFhY0AiIjISrgpz+DpbQxCAoxeui90OkUFjQCIiMiLhvrztCJEuMCARERkR9fWQznOhNlFLMCARERmR0M4OMJFKkHnjFi5fLxO7HSKDxYBERGRErOUm6ONlD4CH2YhaggGJiMjIhHfl6f5ELcWARERkZOoC0tELBVCqBJG7ITJMDEhEREamV0cFbMxNUFxRgz+uFordDpFB0ouAtHr1avj4+MDc3ByhoaFITEy86/ht27bB398f5ubmCAwMxN69ezWeFwQBixcvhpubGywsLBAREYGMjAyNMenp6ZgwYQIcHR1ha2uL8PBwHDx4UOfvjYiorZnIpBjYxQEAD7MRNZfoAWnr1q1YuHAhlixZguTkZAQFBSEyMhJ5eXkNjj969CgeeeQRzJo1CykpKYiKikJUVBROnz6tHrN8+XJ89NFHWLt2LY4fPw4rKytERkaioqJCPeaBBx5ATU0Nfv75ZyQlJSEoKAgPPPAAcnJyWv09ExG1trqrav96ngGJqDkkgiCIeoA6NDQU/fr1w6pVqwAAKpUKnp6emDdvHl555ZV646dMmYKysjLs3r1bvW3AgAEIDg7G2rVrIQgC3N3d8fzzz+OFF14AABQVFcHFxQUbNmzA1KlTUVBQACcnJ/zyyy8YPHgwAKCkpAS2traIi4tDREREvf1WVlaisrJS/XVxcTE8PT1RVFQEW1tbnX5PiIha6vL1MgxdcQimMglSF4+CldxE7JaI9EJxcTEUCsU9f36LOoNUVVWFpKQkjUAilUoRERGBhISEBl+TkJBQL8BERkaqx1+8eBE5OTkaYxQKBUJDQ9VjHBwc4Ofnhy+//BJlZWWoqanBJ598AmdnZ4SEhDS435iYGCgUCvXD09OzRe+diKg1eTtYwbODBaqVAo5f5G1HiJpK1IBUUFAApVIJFxcXje0uLi53PNSVk5Nz1/F1v95tjEQiwYEDB5CSkgIbGxuYm5vjvffeQ2xsLOzt7Rvc76JFi1BUVKR+ZGZmNv0NExG1oXDf2sNsv6TzMBtRU4m+BkkMgiBgzpw5cHZ2xq+//orExERERUVh3LhxyM7ObvA1crkctra2Gg8iIn32921HGJCImkrUgOTo6AiZTIbc3FyN7bm5uXB1dW3wNa6urncdX/fr3cb8/PPP2L17N7755hsMGjQIffr0wf/+9z9YWFhg48aNOnlvRERiG9jFARIJcD6vFNlFt8Ruh8igiBqQzMzMEBISgvj4ePU2lUqF+Ph4hIWFNfiasLAwjfEAEBcXpx7fqVMnuLq6aowpLi7G8ePH1WPKy8sB1K53+iepVAqVStXyN0ZEpAfsLM3Qq6MCAE/3J2oq0Q+xLVy4EJ9++ik2btyIs2fPYvbs2SgrK8PMmTMBANOnT8eiRYvU4+fPn4/Y2FisXLkS586dw9KlS3HixAnMnTsXQO36ogULFuCtt97Crl27cOrUKUyfPh3u7u6IiooCUBuy7O3tER0djZMnTyI9PR0vvvgiLl68iLFjx7b594CIqLWE8zAbUbOIft7nlClTkJ+fj8WLFyMnJwfBwcGIjY1VL7K+cuWKxkzPwIEDsWXLFrz++ut49dVX0bVrV+zcuRM9e/ZUj3nppZdQVlaGJ598EoWFhQgPD0dsbCzMzc0B1B7ai42NxWuvvYYRI0aguroaAQEB+OGHHxAUFNS23wAiolYU7uuE1Qcv4LfzBVCpBEilErFbIjIIol8HyVA19joKRERiqqxRIvjfcbhVrcTeZwejhzv/vaL2zSCug0RERK1LbiJDaOcOAIAj5/NF7obIcDAgEREZuXDf2nVIv3KhNlGjMSARERm5uvuyJV68gYpqpcjdEBkGBiQiIiPXzcUazjZyVNaokHT5ptjtEBkEBiQiIiMnkUjUp/vzMBtR4zAgERG1A3/fdoQLtYkagwGJiKgdGHR7ofaZrGLcKKsSuRsi/ceARETUDjjbmMPf1QaCAPzGq2oT3RMDEhFRO/H36f48zEZ0LwxIRETthPq+bBkF4E0UiO6OAYmIqJ0I7eQAM5kUWUUV+KugTOx2iPQaAxIRUTthYSZDiLc9gNpZJCK6MwYkIqJ2hNdDImocBiQionak7npIx/66jmqlSuRuiPQXAxIRUTsS4K6AnaUpSitrcDKzUOx2iPQWAxIRUTsik0owqAsPsxHdCwMSEVE7oz7dnxeMJLojBiQionam7oKRqZmFKK6oFrkbIv3EgERE1M54drBEJ0crKFUCjl24LnY7RHqJAYmIqB2qm0XiYTaihjEgERG1Q/+87QgR1ceARETUDoV1cYBMKsFfBWW4VnhL7HaI9A4DEhFRO2RrboogDwUA4EhGvsjdEOkfBiQionYqvKsTAOAXHmYjqsdE7AaIiEgcg7s64qP4DBw9X4CLBWVQqlSoVgqoUQqoUalQo6r/+7oxSpWAaqWq9leVAKWydky19hiVChDEfqdkqEb3dEVvL3tR9s2ARETUTgV72sFaboKb5dUY/u4hsdshqsfbwYoBiYiI2papTIqZg3zwxZGLkEokkMkkMJFKYSKVwEQmuf3r31/LpFKYSiWQSSUwlUn/HiOVQiaT3H5OClPZ32NkUgmkErHfKRkqfzcb0fYtEQSBk5/NUFxcDIVCgaKiItja2ordDhERETVCY39+c5E2ERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0mIidgOGShAEAEBxcbHInRAREVFj1f3crvs5ficMSM1UUlICAPD09BS5EyIiImqqkpISKBSKOz4vEe4VoahBKpUKWVlZsLGxgUQi0Vnd4uJieHp6IjMzE7a2tjqry/ri1mZ98Wqzvni1Db2+Ifdu6PVbs7YgCCgpKYG7uzuk0juvNOIMUjNJpVJ4eHi0Wn1bW9tW+QvN+uLWZn3xarO+eLUNvb4h927o9Vur9t1mjupwkTYRERGRFgYkIiIiIi0MSHpGLpdjyZIlkMvlrN/G9Q25d0Ovb8i9G3p9Q+69tesbcu+GXr+1e28MLtImIiIi0sIZJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUDSM6tXr4aPjw/Mzc0RGhqKxMREndT95ZdfMG7cOLi7u0MikWDnzp06qVsnJiYG/fr1g42NDZydnREVFYW0tDSd1F6zZg169eqlvmBYWFgYfvrpJ53Ubsg777wDiUSCBQsW6KTe0qVLIZFINB7+/v46qQ0A165dw2OPPQYHBwdYWFggMDAQJ06c0EltHx+fer1LJBLMmTNHJ/WVSiXeeOMNdOrUCRYWFujSpQv+85//3PMeSY1VUlKCBQsWwNvbGxYWFhg4cCB+//33Zte71+dIEAQsXrwYbm5usLCwQEREBDIyMnRSe/v27Rg1ahQcHBwgkUiQmpqqs96rq6vx8ssvIzAwEFZWVnB3d8f06dORlZWlk/pA7efA398fVlZWsLe3R0REBI4fP66T2v/09NNPQyKR4IMPPtBZ7zNmzKj3GRg9erTO6gPA2bNnMX78eCgUClhZWaFfv364cuWKTuo39BmWSCRYsWJFi2uXlpZi7ty58PDwgIWFBXr06IG1a9c2qu/G1M/NzcWMGTPg7u4OS0tLjB49utGfqZZiQNIjW7duxcKFC7FkyRIkJycjKCgIkZGRyMvLa3HtsrIyBAUFYfXq1TrotL7Dhw9jzpw5OHbsGOLi4lBdXY1Ro0ahrKysxbU9PDzwzjvvICkpCSdOnMCIESMwYcIEnDlzRgeda/r999/xySefoFevXjqtGxAQgOzsbPXjyJEjOql78+ZNDBo0CKampvjpp5/w559/YuXKlbC3t9dJ/d9//12j77i4OADAww8/rJP6y5Ytw5o1a7Bq1SqcPXsWy5Ytw/Lly/Hxxx/rpP6//vUvxMXF4auvvsKpU6cwatQoRERE4Nq1a82qd6/P0fLly/HRRx9h7dq1OH78OKysrBAZGYmKiooW1y4rK0N4eDiWLVum897Ly8uRnJyMN954A8nJydi+fTvS0tIwfvx4ndQHgG7dumHVqlU4deoUjhw5Ah8fH4waNQr5+fktrl1nx44dOHbsGNzd3Rvdd2Prjx49WuOz8PXXX+us/oULFxAeHg5/f38cOnQIf/zxB9544w2Ym5vrpP4/+87OzsYXX3wBiUSCSZMmtbj2woULERsbi02bNuHs2bNYsGAB5s6di127drW4d0EQEBUVhb/++gs//PADUlJS4O3tjYiICJ38bLkngfRG//79hTlz5qi/ViqVgru7uxATE6PT/QAQduzYodOa2vLy8gQAwuHDh1ulvr29vfDZZ5/ptGZJSYnQtWtXIS4uThg6dKgwf/58ndRdsmSJEBQUpJNa2l5++WUhPDy8VWo3ZP78+UKXLl0ElUqlk3pjx44VnnjiCY1tEydOFKZNm9bi2uXl5YJMJhN2796tsb1Pnz7Ca6+91uL62p8jlUoluLq6CitWrFBvKywsFORyufD111+3qPY/Xbx4UQAgpKSkNKPre9evk5iYKAAQLl++3Cr1i4qKBADCgQMHdFL76tWrQseOHYXTp08L3t7ewvvvv9+kunerHx0dLUyYMKFZ9RpTf8qUKcJjjz3WavW1TZgwQRgxYoROagcEBAhvvvmmxrbmfsa066elpQkAhNOnT6u3KZVKwcnJSfj000+bXL+pOIOkJ6qqqpCUlISIiAj1NqlUioiICCQkJIjYWfMUFRUBADp06KDTukqlEt988w3KysoQFham09pz5szB2LFjNf4MdCUjIwPu7u7o3Lkzpk2b1uip83vZtWsX+vbti4cffhjOzs7o3bs3Pv30U53U1lZVVYVNmzbhiSee0NkNmgcOHIj4+Hikp6cDAE6ePIkjR45gzJgxLa5dU1MDpVJZ73/hFhYWOpvB+6eLFy8iJydH4++PQqFAaGiowX6GJRIJ7OzsdF67qqoK69atg0KhQFBQUIvrqVQqPP7443jxxRcREBCggw7rO3ToEJydneHn54fZs2fj+vXrOqmrUqmwZ88edOvWDZGRkXB2dkZoaKjOl0HUyc3NxZ49ezBr1iyd1Bs4cCB27dqFa9euQRAEHDx4EOnp6Rg1alSLa1dWVgKAxmdYKpVCLpe3ymdYGwOSnigoKIBSqYSLi4vGdhcXF+Tk5IjUVfOoVCosWLAAgwYNQs+ePXVS89SpU7C2toZcLsfTTz+NHTt2oEePHjqpDQDffPMNkpOTERMTo7OadUJDQ7FhwwbExsZizZo1uHjxIgYPHoySkpIW1/7rr7+wZs0adO3aFfv27cPs2bPx7LPPYuPGjTroXNPOnTtRWFiIGTNm6KzmK6+8gqlTp8Lf3x+mpqbo3bs3FixYgGnTprW4to2NDcLCwvCf//wHWVlZUCqV2LRpExISEpCdna2D7jXVfU6N4TNcUVGBl19+GY888ohObxS6e/duWFtbw9zcHO+//z7i4uLg6OjY4rrLli2DiYkJnn32WR10Wd/o0aPx5ZdfIj4+HsuWLcPhw4cxZswYKJXKFtfOy8tDaWkp3nnnHYwePRr79+/Hgw8+iIkTJ+Lw4cM66F7Txo0bYWNjg4kTJ+qk3scff4wePXrAw8MDZmZmGD16NFavXo0hQ4a0uLa/vz+8vLywaNEi3Lx5E1VVVVi2bBmuXr3aKp9hbSatvgdqd+bMmYPTp0/rNOH7+fkhNTUVRUVF+O677xAdHY3Dhw/rJCRlZmZi/vz5iIuLa/Qx/6b452xIr169EBoaCm9vb3z77bct/l+cSqVC37598d///hcA0Lt3b5w+fRpr165FdHR0i2pr+/zzzzFmzJgmr++4m2+//RabN2/Gli1bEBAQgNTUVCxYsADu7u466f+rr77CE088gY4dO0Imk6FPnz545JFHkJSUpIPujVN1dTUmT54MQRCwZs0andYePnw4UlNTUVBQgE8//RSTJ0/G8ePH4ezs3OyaSUlJ+PDDD5GcnKyzmU1tU6dOVf8+MDAQvXr1QpcuXXDo0CGMHDmyRbVVKhUAYMKECXjuuecAAMHBwTh69CjWrl2LoUOHtqi+ti+++ALTpk3T2b91H3/8MY4dO4Zdu3bB29sbv/zyC+bMmQN3d/cWz8abmppi+/btmDVrFjp06ACZTIaIiAiMGTNGZydy3A1nkPSEo6MjZDIZcnNzNbbn5ubC1dVVpK6abu7cudi9ezcOHjwIDw8PndU1MzODr68vQkJCEBMTg6CgIHz44Yc6qZ2UlIS8vDz06dMHJiYmMDExweHDh/HRRx/BxMREJ/9L/Cc7Ozt069YN58+fb3EtNze3eiGxe/fuOjuEV+fy5cs4cOAA/vWvf+m07osvvqieRQoMDMTjjz+O5557TmczeV26dMHhw4dRWlqKzMxMJCYmorq6Gp07d9ZJ/X+q+5wa8me4LhxdvnwZcXFxOp09AgArKyv4+vpiwIAB+Pzzz2FiYoLPP/+8RTV//fVX5OXlwcvLS/35vXz5Mp5//nn4+PjopnEtnTt3hqOjo04+w46OjjAxMWmTz/Gvv/6KtLQ0nX2Ob926hVdffRXvvfcexo0bh169emHu3LmYMmUK3n33XZ3sIyQkBKmpqSgsLER2djZiY2Nx/fr1VvkMa2NA0hNmZmYICQlBfHy8eptKpUJ8fLzO19q0BkEQMHfuXOzYsQM///wzOnXq1Kr7U6lU6uPTLTVy5EicOnUKqamp6kffvn0xbdo0pKamQiaT6WQ/dUpLS3HhwgW4ubm1uNagQYPqXU4hPT0d3t7eLa79T+vXr4ezszPGjh2r07rl5eWQSjX/GZLJZOr/VeuKlZUV3NzccPPmTezbtw8TJkzQaX0A6NSpE1xdXTU+w8XFxTh+/LhBfIbrwlFGRgYOHDgABweHVt+nLj7Hjz/+OP744w+Nz6+7uztefPFF7Nu3T0edarp69SquX7+uk8+wmZkZ+vXr1yaf488//xwhISE6WfcF1P6dqa6ubpPPsEKhgJOTEzIyMnDixIlW+Qxr4yE2PbJw4UJER0ejb9++6N+/Pz744AOUlZVh5syZLa5dWlqq8b+dixcvIjU1FR06dICXl1eL68+ZMwdbtmzBDz/8ABsbG/WaC4VCAQsLixbVXrRoEcaMGQMvLy+UlJRgy5YtOHTokM7+8bOxsam3VsrKygoODg46WUP1wgsvYNy4cfD29kZWVhaWLFkCmUyGRx55pMW1n3vuOQwcOBD//e9/MXnyZCQmJmLdunVYt25di2vXUalUWL9+PaKjo2Fiott/MsaNG4e3334bXl5eCAgIQEpKCt577z088cQTOqm/b98+CIIAPz8/nD9/Hi+++CL8/f2b/Zm61+dowYIFeOutt9C1a1d06tQJb7zxBtzd3REVFdXi2jdu3MCVK1fU1yaq+4Hq6uraqBmqu9V3c3PDQw89hOTkZOzevRtKpVL9Ge7QoQPMzMxaVN/BwQFvv/02xo8fDzc3NxQUFGD16tW4du1aoy4Zca/vjXaYMzU1haurK/z8/O5Z+171O3TogH//+9+YNGkSXF1dceHCBbz00kvw9fVFZGRki+t7eXnhxRdfxJQpUzBkyBAMHz4csbGx+PHHH3Ho0CGd1Adqw/q2bduwcuXKRtVsbO2hQ4fixRdfhIWFBby9vXH48GF8+eWXeO+993RSf9u2bXBycoKXlxdOnTqF+fPnIyoqSieLwO+p1c+Toyb5+OOPBS8vL8HMzEzo37+/cOzYMZ3UPXjwoACg3iM6Olon9RuqDUBYv359i2s/8cQTgre3t2BmZiY4OTkJI0eOFPbv39/ypu9Cl6f5T5kyRXBzcxPMzMyEjh07ClOmTBHOnz+vk9qCIAg//vij0LNnT0Eulwv+/v7CunXrdFZbEARh3759AgAhLS1Np3UFQRCKi4uF+fPnC15eXoK5ubnQuXNn4bXXXhMqKyt1Un/r1q1C586dBTMzM8HV1VWYM2eOUFhY2Ox69/ocqVQq4Y033hBcXFwEuVwujBw5stHft3vVXr9+fYPPL1mypMX16y4d0NDj4MGDLa5/69Yt4cEHHxTc3d0FMzMzwc3NTRg/fryQmJiok++Ntqae5n+3+uXl5cKoUaMEJycnwdTUVPD29hb+7//+T8jJydFJ/Tqff/654OvrK5ibmwtBQUHCzp07dVr/k08+ESwsLJr89/9etbOzs4UZM2YI7u7ugrm5ueDn5yesXLmy0ZcCuVf9Dz/8UPDw8BBMTU0FLy8v4fXXX9fZvw/3IhGENljpRERERGRAuAaJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiKiZJBIJdu7cKXYbRNQKGJCIyCDNmDEDEomk3mP06NFit0ZERoA3qyUigzV69GisX79eY5tcLhepGyIyJpxBIiKDJZfL1Xezr3vY29sDqD38tWbNGowZMwYWFhbo3LkzvvvuO43Xnzp1CiNGjICFhQUcHBzw5JNPorS0VGPMF198gYCAAMjlcri5uWHu3LkazxcUFODBBx+EpaUlunbtil27dqmfu3nzJqZNmwYnJydYWFiga9eu9QIdEeknBiQiMlpvvPEGJk2ahJMnT2LatGmYOnUqzp49CwAoKytDZGQk7O3t8fvvv2Pbtm04cOCARgBas2YN5syZgyeffBKnTp3Crl274Ovrq7GPf//735g8eTL++OMP3H///Zg2bRpu3Lih3v+ff/6Jn376CWfPnsWaNWvg6OjYdt8AImo+gYjIAEVHRwsymUywsrLSeLz99tuCIAgCAOHpp5/WeE1oaKgwe/ZsQRAEYd26dYK9vb1QWlqqfn7Pnj2CVCoVcnJyBEEQBHd3d+G11167Yw8AhNdff139dWlpqQBA+OmnnwRBEIRx48YJM2fO1M0bJqI2xTVIRGSwhg8fjjVr1mhs69Chg/r3YWFhGs+FhYUhNTUVAHD27FkEBQXByspK/fygQYOgUqmQlpYGiUSCrKwsjBw58q499OrVS/17Kysr2NraIi8vDwAwe/ZsTJo0CcnJyRg1ahSioqIwcODAZr1XImpbDEhEZLCsrKzqHfLSFQsLi0aNMzU11fhaIpFApVIBAMaMGYPLly9j7969iIuLw8iRIzFnzhy8++67Ou+XiHSLa5CIyGgdO3as3tfdu3cHAHTv3h0nT55EWVmZ+vnffvsNUqkUfn5+sLGxgY+PD+Lj41vUg5OTE6Kjo7Fp0yZ88MEHWLduXYvqEVHb4AwSERmsyspK5OTkaGwzMTFRL4Tetm0b+vbti/DwcGzevBmJiYn4/PPPAQDTpk3DkiVLEB0djaVLlyI/Px/z5s3D448/DhcXFwDA0qVL8fTTT8PZ2RljxoxBSUkJfvvtN8ybN69R/S1evBghISEICAhAZWUldu/erQ5oRKTfGJCIyGDFxsbCzc1NY5ufnx/OnTsHoPYMs2+++QbPPPMM3Nzc8PXXX6NHjx4AAEtLS+zbtw/z589Hv379YGlpiUmTJuG9995T14qOjkZFRQXef/99vPDCC3B0dMRDDz3U6P7MzMywaNEiXLp0CRYWFhg8eDC++eYbHbxzImptEkEQBLGbICLSNYlEgh07diAqKkrsVojIAHENEhEREZEWBiQiIiIiLVyDRERGiasHiKglOINEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLS8v9tQd8dF5Z0JwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Training part\n",
    "print(\n",
    "    f\"Performing MNIST task with {quantization_bits}-bits in quantization and a \"\n",
    "    f\"sparsity of {sparsity}\"\n",
    ")\n",
    "\n",
    "if do_training:\n",
    "    print(\"\\n1. Training\")\n",
    "    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=0.0001)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=gamma)\n",
    "    test_loss = 1e10\n",
    "    loss_values = []\n",
    "\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        train(model, device, train_loader, optimizer, epoch, criterion)\n",
    "        cur_loss = test(model, device, test_loader, criterion)\n",
    "\n",
    "        scheduler.step()\n",
    "\n",
    "        loss_values.append(cur_loss)\n",
    "\n",
    "    model.prune(sparsity, False)\n",
    "\n",
    "    # Plot the loss\n",
    "    fig = plt.figure()\n",
    "    plt.plot(loss_values)\n",
    "    fig.suptitle(\"Loss during QAT\")\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(\"Loss\")\n",
    "    plt.xticks(range(len(loss_values)))\n",
    "\n",
    "    # Export to ONNX\n",
    "    print(\"\\n2. Exporting to ONNX\")\n",
    "    inp = torch.rand((1, 784)).to(device)\n",
    "    torch.onnx.export(model, inp, \"mnist.qat.onnx\", opset_version=14)\n",
    "\n",
    "else:\n",
    "    print(\"\\n1. Loading pre-trained model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "858488a8",
   "metadata": {},
   "source": [
    "Then, we reload the model, which is useful if we've set do_training = False to avoid to redo the training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "221d9405",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reload the model\n",
    "model = onnx.load(\"mnist.qat.onnx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce19d042",
   "metadata": {},
   "source": [
    "We prepare the ground truth, for final check of the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d3736e4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 313/313 [00:01<00:00, 208.36it/s]\n"
     ]
    }
   ],
   "source": [
    "# Prepare tests\n",
    "list_inputs = []\n",
    "list_targets = []\n",
    "\n",
    "for inputs in test_loader:\n",
    "    inputs_var, targets = inputs\n",
    "    list_inputs.append(inputs_var.detach().cpu().numpy())\n",
    "    list_targets.append(targets.detach().cpu().numpy())\n",
    "\n",
    "np_inputs = np.concatenate(list_inputs, axis=0)\n",
    "np_targets = np.concatenate(list_targets, axis=0)\n",
    "\n",
    "test_data = np.zeros((len(test_loader.dataset), 784))\n",
    "test_target = np.zeros((len(test_loader.dataset), 1))\n",
    "idx = 0\n",
    "\n",
    "for data, target in tqdm(test_loader):\n",
    "    target_np = target.cpu().numpy()\n",
    "    for idx_batch, im in enumerate(data.numpy()):\n",
    "        test_data[idx] = im\n",
    "        test_target[idx] = target_np[idx_batch]\n",
    "        idx += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14025865",
   "metadata": {},
   "source": [
    "### Measuring accuracy in VL and FHE\n",
    "\n",
    "Finally, we measure:\n",
    "- accuracy in VL for the full test set\n",
    "- accuracy in VL for the reduced test set\n",
    "- accuracy in FHE for the reduced test set   \n",
    "\n",
    "and we check that accuracy in VL and in FHE are equivalent on the reduced test set. \n",
    "\n",
    "Running FHE computations on a too large reduced test set may be prohibitive since we're 100% on CPU for the time being. Later, we'll have HW accelerators, to make these computations several order of magnitude faster. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eb014141",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "3. Compiling with the Virtual Library\n",
      "\n",
      "4. Checking accuracy with the Virtual Library (length 10000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:58<00:00, 170.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy in VL with length 10000: 9282/10000 = 0.9282, in 6 bits\n",
      "\n",
      "5. Compiling with the Virtual Library\n",
      "\n",
      "6. Checking accuracy with the Virtual Library (length 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 117.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy in VL with length 2: 2/2 = 1.0000, in 6 bits\n",
      "\n",
      "7. Compiling in FHE\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "8. Checking accuracy in FHE (length 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [08:43<00:00, 261.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy in FHE with length 2: 2/2 = 1.0000, in 6 bits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Test in the VL and in FHE\n",
    "accuracy = {}\n",
    "current_index = 3\n",
    "\n",
    "for use_virtual_lib, use_full_dataset in [(True, True), (True, False), (False, False)]:\n",
    "    test_data_length = test_data_length_full if use_full_dataset else test_data_length_reduced\n",
    "\n",
    "    correct_fhe, test_data_shape_0, max_bit_width = compile_and_test(\n",
    "        model,\n",
    "        use_virtual_lib,\n",
    "        np_inputs,\n",
    "        quantization_bits,\n",
    "        test_data,\n",
    "        test_data_length,\n",
    "        test_target,\n",
    "        show_mlir,\n",
    "        current_index,\n",
    "    )\n",
    "\n",
    "    current_index += 2\n",
    "    current_accuracy = correct_fhe / test_data_shape_0\n",
    "\n",
    "    print(\n",
    "        f\"Accuracy in {'VL' if use_virtual_lib else 'FHE'} with length {test_data_length}: \"\n",
    "        f\"{correct_fhe}/{test_data_shape_0} = \"\n",
    "        f\"{current_accuracy:.4f}, in {max_bit_width} bits\"\n",
    "    )\n",
    "\n",
    "    if (use_virtual_lib, use_full_dataset) == (True, True):\n",
    "        accuracy[\"VL full\"] = current_accuracy\n",
    "    elif (use_virtual_lib, use_full_dataset) == (True, False):\n",
    "        accuracy[\"VL short\"] = current_accuracy\n",
    "    else:\n",
    "        assert (use_virtual_lib, use_full_dataset) == (False, False)\n",
    "        accuracy[\"FHE short\"] = current_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bf39f067",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check that accuracy in FHE and in VL is the same\n",
    "assert (\n",
    "    accuracy[\"VL short\"] == accuracy[\"FHE short\"]\n",
    "), \"Error, accuracy in VL and in FHE are not the same\"\n",
    "\n",
    "# Check that accuracy is random-looking\n",
    "assert accuracy[\"VL full\"] > 0.8, \"Error, accuracy is too bad\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5060f56",
   "metadata": {},
   "source": [
    "### Conclusion\n",
    "\n",
    "We've shown how to train a model with Brevitas in quantization-aware training. Then, we've seen that its accuracy was pretty correct (92% is not too hard to obtain), even if lower than the state-of-the-art on cleartexts. \n",
    "\n",
    "Brevitas is not the only third-party package one can use for QAT. Other examples with other frameworks will certainly be added in our repository.\n"
   ]
  }
 ],
 "metadata": {
  "execution": {
   "timeout": 10800
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}