federated-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "import syft as sy\n",
    "import torchvision.datasets as datasets\n",
    "from torchvision import transforms\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import math\n",
    "\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_workers = 100\n",
    "n_workers_round = int(n_workers * 0.2)\n",
    "\n",
    "lr = 0.1\n",
    "batch_size = 16\n",
    "test_batch_size = 100\n",
    "\n",
    "dp_t = 5 # max number of non-colluding parties\n",
    "dp_eps = 1. # epsilon for (ϵ,δ)-DP\n",
    "dp_delta = 1e-5 # delta for (ϵ,δ)-DP\n",
    "dp_clipping = 1.\n",
    "dp_sigma = 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(X_mnist):\n",
    "    return X_mnist.view(-1, 28, 28).float() / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_list(l, batch_size):\n",
    "    return [l[i*batch_size:(i+1)*batch_size] for i in range(int(math.ceil(len(l) / batch_size)))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_train = datasets.MNIST(root='./data', train=True, download=True, \n",
    "                             transform=None)\n",
    "mnist_test = datasets.MNIST(root='./data', train=False, download=True, \n",
    "                             transform=transforms.Compose([\n",
    "                                 transforms.ToTensor()                             \n",
    "                             ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loader = torch.utils.data.DataLoader(\n",
    "    mnist_test,\n",
    "    batch_size=test_batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "    mnist_train,\n",
    "    batch_size=test_batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples_train_per_worker = len(mnist_train.targets) // n_workers\n",
    "n_samples_train = len(mnist_train.targets)\n",
    "\n",
    "n_samples_test_per_worker = len(mnist_test.targets) // n_workers\n",
    "n_samples_test = len(mnist_test.targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Put data on different workers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Declare the hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "hook = sy.TorchHook(torch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Declare the workers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "workers = [sy.VirtualWorker(hook, id=f\"bob{i}\") for i in range(n_workers)]\n",
    "crypto_provider = sy.VirtualWorker(hook, id=\"crypto_provider\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Move the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_ptr_worker = [transform(mnist_train.data[n_samples_train_per_worker*i:n_samples_train_per_worker*(i+1)]).send(workers[i]) for i in range(n_workers)]\n",
    "Y_train_ptr_worker = [mnist_train.targets[n_samples_train_per_worker*i:n_samples_train_per_worker*(i+1)].send(workers[i]) for i in range(n_workers)]\n",
    "\n",
    "X_test_ptr_worker = [transform(mnist_test.data[n_samples_test_per_worker*i:n_samples_test_per_worker*(i+1)]).send(workers[i]) for i in range(n_workers)]\n",
    "Y_test_ptr_worker = [mnist_test.targets[n_samples_test_per_worker*i:n_samples_test_per_worker*(i+1)].send(workers[i]) for i in range(n_workers)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([100, 28, 28])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_ptr_worker[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 20, 5, 1)\n",
    "        self.conv2 = nn.Conv2d(20, 50, 5, 1)\n",
    "        self.fc1 = nn.Linear(4*4*50, 64)\n",
    "        self.fc2 = nn.Linear(64, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 1, 28, 28)\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.max_pool2d(x, 2, 2)\n",
    "        x = x.view(-1, 4*4*50)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return F.log_softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = torch.nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net()\n",
    "\n",
    "# We send all initial models to the clients, and never take them back in clear (only for secure aggregation)\n",
    "model_worker = [model.copy().send(workers[i]) for i in range(n_workers)] # Important: starting from the same initialization\n",
    "\n",
    "opt_worker = [optim.SGD(params=model_worker[i].parameters(), lr=lr) for i in range(n_workers)]\n",
    "\n",
    "list_loss_test_worker = [list() for _ in range(n_workers)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[], [], [], [], []]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_loss_test_worker[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Expected 4-dimensional input for 4-dimensional weight 20 1 5 5, but got 1-dimensional input of size [0] instead",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPureFrameworkTensorFoundError\u001b[0m             Traceback (most recent call last)",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/frameworks/torch/tensors/interpreters/native.py\u001b[0m in \u001b[0;36mhandle_func_command\u001b[0;34m(cls, command)\u001b[0m\n\u001b[1;32m    289\u001b[0m             new_args, new_kwargs, new_type, args_type = hook_args.unwrap_args_from_function(\n\u001b[0;32m--> 290\u001b[0;31m                 \u001b[0mcmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_args_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    291\u001b[0m             )\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook_args.py\u001b[0m in \u001b[0;36munwrap_args_from_function\u001b[0;34m(attr, args, kwargs, return_args_type)\u001b[0m\n\u001b[1;32m    156\u001b[0m         \u001b[0;31m# Try running it\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 157\u001b[0;31m         \u001b[0mnew_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook_args.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    349\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 350\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlambdas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook_args.py\u001b[0m in \u001b[0;36mseven_fold\u001b[0;34m(lambdas, args, **kwargs)\u001b[0m\n\u001b[1;32m    565\u001b[0m         \u001b[0mlambdas\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 566\u001b[0;31m         \u001b[0mlambdas\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    567\u001b[0m         \u001b[0mlambdas\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook_args.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(i)\u001b[0m\n\u001b[1;32m    327\u001b[0m         \u001b[0;31m# Last if not, rule is probably == 1 so use type to return the right transformation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m         \u001b[0;32melse\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mforward_func\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    329\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrules\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# And do this for all the args / rules provided\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(i)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"child\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m     \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0m_\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPureFrameworkTensorFoundError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m     \u001b[0mAutogradTensor\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mget_child\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/frameworks/torch/hook/hook_args.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"child\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m     \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0m_\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPureFrameworkTensorFoundError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m     \u001b[0mAutogradTensor\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mget_child\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mPureFrameworkTensorFoundError\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-b79b37447176>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi_sample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m                     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi_sample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m                     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_worker\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mworker_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_ptr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_sample\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi_sample\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m                     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_ptr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi_sample\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi_sample\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-8a99c0de106c>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2d_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mconv2d_forward\u001b[0;34m(self, input, weight)\u001b[0m\n\u001b[1;32m    340\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[1;32m    341\u001b[0m         return F.conv2d(input, weight, self.bias, self.stride,\n\u001b[0;32m--> 342\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/trace.py\u001b[0m in \u001b[0;36mtrace_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     81\u001b[0m                 \u001b[0msyft\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook.py\u001b[0m in \u001b[0;36moverloaded_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    584\u001b[0m                 \u001b[0mhandle_func_command\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msyft\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle_func_command\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhandle_func_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/frameworks/torch/tensors/interpreters/native.py\u001b[0m in \u001b[0;36mhandle_func_command\u001b[0;34m(cls, command)\u001b[0m\n\u001b[1;32m    322\u001b[0m             \u001b[0;31m# in the execute_command function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m                 \u001b[0;31m# Change the library path to avoid errors on layers like AvgPooling\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/frameworks/torch/tensors/interpreters/native.py\u001b[0m in \u001b[0;36m_get_response\u001b[0;34m(cmd, args, kwargs)\u001b[0m\n\u001b[1;32m    344\u001b[0m         \u001b[0mcommand_method\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"native_{command}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcommand_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcommand_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected 4-dimensional input for 4-dimensional weight 20 1 5 5, but got 1-dimensional input of size [0] instead"
     ]
    }
   ],
   "source": [
    "n_rounds = 20\n",
    "n_local_epochs = 1\n",
    "\n",
    "for i_round in range(n_rounds):\n",
    "    sample_indices = list(np.random.permutation(range(n_samples_train_per_worker)))\n",
    "    list_workers_round = np.random.choice(range(n_workers), n_workers_round, replace=False)\n",
    "    \n",
    "    for i_worker, worker_index in enumerate(list_workers_round):\n",
    "        loss_worker = 0\n",
    "        i_iter = 0\n",
    "        \n",
    "        # Update client        \n",
    "        for i_local_epoch in range(n_local_epochs):  # n_local_epochs=E\n",
    "            list_batch_indices = batch_list(sample_indices, batch_size)\n",
    "            for i_batch, batch_indices in enumerate(list_batch_indices):\n",
    "                batch_indices_tensor = torch.tensor(batch_indices).send(workers[worker_index])\n",
    "                X_ptr = X_train_ptr_worker[worker_index][batch_indices, :, :]\n",
    "                Y_ptr = Y_train_ptr_worker[worker_index].gather(0, batch_indices_tensor)\n",
    "\n",
    "                pred = model_worker[worker_index](X_ptr)\n",
    "                loss = loss_fn(pred, Y_ptr)\n",
    "\n",
    "                opt_worker[worker_index].zero_grad()\n",
    "                loss.backward()\n",
    "                opt_worker[worker_index].step()\n",
    "\n",
    "#                 model_worker[worker_index] = model_worker[worker_index].get()\n",
    "                loss_worker += loss.get().data.numpy()\n",
    "#                 model_worker[worker_index].get()\n",
    "                i_iter += 1\n",
    "\n",
    "                clear_output(wait=True)\n",
    "                print(f\"Round {i_round + 1}/{n_rounds}\")\n",
    "                print(f\"Worker {i_worker+1}/{n_workers_round}\")\n",
    "                print(f\"Batch {i_batch} / {len(mnist_train.data) // n_workers // batch_size}: {loss_worker / (i_iter):.4f}\")\n",
    "                if i_round >= 1:\n",
    "                    print(f\"Test loss {np.mean(list_loss_test_worker, axis=0)[-1]:.4f}\")\n",
    "                del batch_indices_tensor\n",
    "    \n",
    "    # Update global model\n",
    "    print(\"Update...\")\n",
    "    with torch.no_grad():\n",
    "        new_param = [0.] * len(list(model_worker[0].parameters()))\n",
    "        for i_param, param_worker in enumerate(zip(*(model_worker[i].parameters() for i in range(n_workers)))):\n",
    "            print(f\"{i_param} \", end=\"\\r\")\n",
    "            for i_worker in list_workers_round:\n",
    "                new_param[i_param] += param_worker[i_worker].copy().get().fix_precision().share(*workers, crypto_provider=crypto_provider)\n",
    "            new_param[i_param] = new_param[i_param].get().float_precision() / n_workers_round\n",
    "               \n",
    "        for i_worker in range(n_workers):\n",
    "            model_worker[i_worker].get()\n",
    "            for i_param, param in enumerate(model_worker[i_worker].parameters()):\n",
    "                param.set_(torch.tensor(new_param[i_param]))\n",
    "            model_worker[i_worker].send(workers[i_worker])\n",
    "            \n",
    "\n",
    "    # Test clients\n",
    "    print(\"Test...\")\n",
    "    for i_worker in range(n_workers):\n",
    "        X_ptr = X_test_ptr_worker[i_worker]\n",
    "        Y_ptr = Y_test_ptr_worker[i_worker]\n",
    "        pred = model_worker[i_worker](X_ptr)\n",
    "        test_loss = loss_fn(pred, Y_ptr).get().data.numpy()\n",
    "        list_loss_test_worker[i_worker].append(test_loss)\n",
    "#         model_worker[worker_index] = model_worker[worker_index].get()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test global model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 9758/10000 (98%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i_worker in range(n_workers):\n",
    "    model_worker[i_worker].get()\n",
    "\n",
    "model_worker[0].eval()\n",
    "correct = 0\n",
    "for X, Y in test_loader:\n",
    "    output = model_worker[0](X)\n",
    "    pred = output.argmax(1, keepdim=True)\n",
    "    correct += pred.eq(Y.view_as(pred)).sum().item()\n",
    "\n",
    "print('Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "    correct, len(test_loader.dataset),\n",
    "    100. * correct / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAAFlCAYAAAAnA02CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de3Rc9Xnv/88zM7pLlmTLlo0tyVwcG0wACQXqJD2RSQImvzbk5EcCpCchOeFAcsJJmrPSNvRCski7kkDarqS5AE1oLk0wTdo0NDVQEqLcuMTGNmAbGxvw/Y4t2bKuo3nOH7Mlj2TJGkmj2aPR+7WW1uzZe8/o0cMg9OG79/dr7i4AAAAAyAeRsAsAAAAAgEwh4AAAAADIGwQcAAAAAHmDgAMAAAAgbxBwAAAAAOQNAg4AAACAvBELu4CR1NTU+OLFi8MuQ5J06tQplZWVhV3GjEPfs4+eZx89Dwd9zz56Hg76nn30PLueffbZo+4+d/j+nAw4ixcv1rp168IuQ5LU2tqqlpaWsMuYceh79tHz7KPn4aDv2UfPw0Hfs4+eZ5eZ7RppP5eoAQAAAMgbBBwAAAAAeYOAAwAAACBvEHAAAAAA5A0CDgAAAIC8QcABAAAAkDcIOAAAAADyBgEHAAAAQN4g4AAAAADIGwQcAAAAAHmDgAMAAAAgbxBwAAAAAOSNWNgF5LIb7ntKbW1damkJuxIAAAAA6WAEBwAAAEDeIOAAAAAAyBsEHAAAAAB5g4ADAAAAIG8QcAAAAADkDQIOAAAAgLxBwAEAAACQNwg4AAAAAPIGAQcAAABA3iDgAAAAAMgbBBwAAAAAeYOAAwAAACBvEHDOor2rT+09HnYZAAAAANIUC7uAXHbkZI/au1zuLjMLuxwAAAAAY2AE5yzKi2KKu3SgvTvsUgAAAACkgYBzFuXFyQGu9buPh1wJAAAAgHQQcM6itDAqk7Rhd1vYpQAAAABIAwHnLCJmKo4xggMAAABMFwScMRRHpc37Tqgn3h92KQAAAADGQMAZQ0nM1Nuf0Ob9J8IuBQAAAMAYCDhjKAkm0l6/i8vUAAAAgFzHOjhjiEVMC6uKtWEPEw0AAAAAuY4RnDQ01ldpAyM4AAAAQM4j4KShsb5a+9u7dZAFPwEAAICcRsBJQ1N9lSRpA9NFAwAAADltzIBjZg+Y2WEz2zTK8RYzazezjcHXnSnHVpnZNjPbYWafzmTh2bT8nEoVxiLchwMAAADkuHRGcL4tadUY5/za3S8Lvu6SJDOLSvqapGslXSTpJjO7aDLFhqUwFtHF58xiJjUAAAAgx40ZcNz9V5KOTeC9r5C0w91fcfdeSaslXTeB9wnNQ7et0B1XlkiSmuqr9fy+dvXGEyFXBQAAAGA05u5jn2S2WNJP3f3iEY61SPpXSXsl7Zf0KXffbGbXS1rl7rcE571f0pXufvso3+NWSbdKUm1t7eWrV6+eyM+TcR0dHSovL9fvDsb19Y09unNFsc6rjIZdVt4b6Duyh55nHz0PB33PPnoeDvqeffQ8u1auXPmsuzcP35+JdXDWS2pw9w4ze4ekf5e0ZLxv4u73S7pfkpqbm72lpSUDpU1ea2urWlpatLS9S1/f+ISs5jy1vOncsMvKewN9R/bQ8+yj5+Gg79lHz8NB37OPnueGSc+i5u4n3L0j2F4jqcDMaiTtk1SXcuqiYN+0tKCyRPNnFWvDbiYaAAAAAHLVpAOOmc03Mwu2rwje8zVJayUtMbNzzaxQ0o2SHp7s9wtTU0OV1jNVNAAAAJCzxrxEzcwelNQiqcbM9kr6jKQCSXL3eyVdL+mjZhaX1CXpRk/e2BM3s9slPSYpKukBd988JT9FljTVV2vNCwd1+GS35lUUh10OAAAAgGHGDDjuftMYx78q6aujHFsjac3ESss9jYMLfrbpmuXzQ64GAAAAwHCTvkRtJll+TqUKosZlagAAAECOIuCMQ3FBVBedU8lEAwAAAECOIuCMU1N9lZ7f26a+fhb8BAAAAHINAWecmuqr1d2X0LaDJ8MuBQAAAMAwBJxxGphogPtwAAAAgNxDwBmnhVUlmldRpPW7CDgAAABAriHgjJOZqbG+Shv2MNEAAAAAkGsIOBPQVF+tXa916mhHT9ilAAAAAEhBwJmAxvpqSdJGposGAAAAcgoBZwIuWVSpWIQFPwEAAIBcQ8CZgOSCn7NY8BMAAADIMQScCWqsq9Jze9sUZ8FPAAAAIGcQcCaoqaFanb392naIBT8BAACAXEHAmaDGuuREA1ymBgAAAOQOAs4E1c0uUU15IRMNAAAAADmEgDNBZqbL6qqZKhoAAADIIQScSWhqqNIrR0/p+KnesEsBAAAAIALOpDQNLPi5h1EcAAAAIBcQcCbhkkWVirLgJwAAAJAzCDiTUFoY07L5FQQcAAAAIEcQcCapsb5Kz+1pV3/Cwy4FAAAAmPEIOJPUVF+tjp64th9mwU8AAAAgbAScSRqYaIAFPwEAAIDwEXAmqWFOqWaXFWr9Lu7DAQAAAMJGwJkkM1NjXZU2MFU0AAAAEDoCTgY01ldpx+EOtXf2hV0KAAAAMKMRcDJg8D6cPVymBgAAAISJgJMBl9RVKWJMNAAAAACEjYCTAeVFMb2ulgU/AQAAgLARcDKkqaFaG/e0KcGCnwAAAEBoCDgZ0lhXpZPdcb18pCPsUgAAAIAZi4CTIU0NLPgJAAAAhI2AkyHnzilTZUkB9+EAAAAAISLgZEgkYmqsryLgAAAAACEi4GRQY121th/u0IluFvwEAAAAwkDAyaCmhiq5S8/t4T4cAAAAIAxjBhwze8DMDpvZplGO/5GZPW9mL5jZk2Z2acqxncH+jWa2LpOF56JL66pkLPgJAAAAhCadEZxvS1p1luOvSnqLu79e0uck3T/s+Ep3v8zdmydW4vQxq7hAS+aVcx8OAAAAEJIxA467/0rSsbMcf9LdB/6if1rSogzVNi011Vdrw24W/AQAAADCYO5j/yFuZosl/dTdLx7jvE9JWubutwTPX5V0XJJLus/dh4/upL72Vkm3SlJtbe3lq1evTvNHmFodHR0qLy9P+/xf7u3TP23q1effXKIF5dziNFHj7Tsmj55nHz0PB33PPnoeDvqeffQ8u1auXPnsSFeJxTL1DcxspaQPS3pzyu43u/s+M5sn6XEz2xqMCJ0hCD/3S1Jzc7O3tLRkqrRJaW1t1XhqWXjopP5p068Um79ELc11U1dYnhtv3zF59Dz76Hk46Hv20fNw0Pfso+e5ISNDDGZ2iaRvSrrO3V8b2O/u+4LHw5J+LOmKTHy/XHb+3HJVFMe0gZnUAAAAgKybdMAxs3pJ/ybp/e7+Usr+MjOrGNiWdLWkEWdiyyeRiOmyuiqt38VEAwAAAEC2jXmJmpk9KKlFUo2Z7ZX0GUkFkuTu90q6U9IcSV83M0mKB9fC1Ur6cbAvJukH7v7oFPwMOaepvlr/8MR2dfTEVV6UsasAAQAAAIxhzL++3f2mMY7fIumWEfa/IunSM1+R/xrrq5Rw6fk9bXrjBTVhlwMAAADMGEzzNQUa66olifVwAAAAgCwj4EyBytICnT+3TBt2M9EAAAAAkE0EnCnSVF+tDXvalM46QwAAAAAyg4AzRZoaqnXsVK92vdYZdikAAADAjEHAmSKN9VWSuA8HAAAAyCYCzhRZMq9C5UUx7sMBAAAAsoiAM0WiEdOldZWM4AAAAABZRMCZQk311dp68KQ6e+NhlwIAAADMCAScKdRYX6X+hOv5ve1hlwIAAADMCAScKcSCnwAAAEB2EXCmUHVZoc6rYcFPAAAAIFsIOFPssvoqbdh9nAU/AQAAgCwg4EyxpvpqHe3o1d7jXWGXAgAAAOQ9As4UY8FPAAAAIHsIOFNsaW2FSgujWr+LgAMAAABMNQLOFItFI7p0UZU27GGiAQAAAGCqEXCyoLG+Slv2n1B3X3/YpQAAAAB5jYCTBU311YonXC/sY8FPAAAAYCoRcLLgsoGJBrgPBwAAAJhSBJwsqCkvUsOcUhb8BAAAAKYYASdLGuuqtJ4FPwEAAIApRcDJkqaGah0+2aN9bSz4CQAAAEwVAk6WNNVXSxKXqQEAAABTiICTJUvnV6i4IKL1u5loAAAAAJgqBJwsKYhGdMmiKkZwAAAAgClEwMmixvoqbd7fzoKfAAAAwBQh4GRRU321+vpdm/efCLsUAAAAIC8RcLKoMVjwcwP34QAAAABTgoCTRfMqirWouoSJBgAAAIApQsDJsqb6aiYaAAAAAKYIASfLGuurdKC9WwfaWfATAAAAyDQCTpax4CcAAAAwdQg4WXbhglkqikW0fhf34QAAAACZRsDJssJYRK9fWMlEAwAAAMAUIOCEoKmhWpv2n1BPnAU/AQAAgEwi4ISgsa5KvfGEtrDgJwAAAJBRaQUcM3vAzA6b2aZRjpuZfcXMdpjZ82bWlHLsZjPbHnzdnKnCp7OmBiYaAAAAAKZCuiM435a06izHr5W0JPi6VdI3JMnMZkv6jKQrJV0h6TNmVj3RYvNF7axinVNZzH04AAAAQIalFXDc/VeSjp3llOskfdeTnpZUZWYLJF0j6XF3P+buxyU9rrMHpRmjsYEFPwEAAIBMy9Q9OAsl7Ul5vjfYN9r+Ga+xrkr72rp0+ER32KUAAAAAeSMWdgEDzOxWJS9vU21trVpbW8MtKNDR0TE1tbQlZ1D77prfqHl+zvxjyBlT1neMip5nHz0PB33PPnoeDvqeffQ8N2TqL+t9kupSni8K9u2T1DJsf+tIb+Du90u6X5Kam5u9paVlpNOyrrW1VVNRy4p4v+5e+1/qm7VQLS0XZvz9p7up6jtGR8+zj56Hg75nHz0PB33PPnqeGzJ1idrDkj4QzKb2e5La3f2ApMckXW1m1cHkAlcH+2a8olhUyxfOYqIBAAAAIIPSGsExsweVHImpMbO9Ss6MViBJ7n6vpDWS3iFph6ROSR8Kjh0zs89JWhu81V3ufrbJCmaUpvpq/fPTu9TXn1BBlCWJAAAAgMlKK+C4+01jHHdJHxvl2AOSHhh/afmvsb5K3/rNq3rxwAldsqgq7HIAAACAaY9hgxA11bPgJwAAAJBJBJwQLagsVu2sIu7DAQAAADKEgBMiM1NTfTUBBwAAAMgQAk7ImuqrtedYl46c7Am7FAAAAGDaI+CErLE+ObnABkZxAAAAgEkj4ITs4oWVKoiaNuxhogEAAABgsgg4ISsuiOqiBbO0fhcjOAAAAMBkEXByQGN9tZ7f2654fyLsUgAAAIBpjYCTA5oaqtXV16+tB0+GXQoAAAAwrRFwckBjHRMNAAAAAJlAwMkBi6pLNLeiSBt2M9EAAAAAMBkEnBxgZmqsq2LBTwAAAGCSCDg5oqmhWjtf69SxU71hlwIAAABMWwScHMF9OAAAAMDkEXByxCWLqhSLGPfhAAAAAJNAwMkRJYVRXbhgFvfhAAAAAJNAwMkhjfVVem5Pm/oTHnYpAAAAwLREwMkhTfXVOtXbr5cOseAnAAAAMBEEnBzSWJ+caIDL1AAAAICJIeDkkPrZpZpTVshEAwAAAMAEEXByiJmpsZ4FPwEAAICJIuDkmMb6ar1y5JTaOlnwEwAAABgvAk6OaaqvliRt2MNlagAAAMB4EXByzCWLKhUxacMuLlMDAAAAxouAk2PKimJaNn8WIzgAAADABBBwclBjfZU27m5TggU/AQAAgHEh4OSgpvpqneyJa8eRjrBLAQAAAKYVAk4OampITjSwnvtwAAAAgHEh4OSgxXNKVV1awHo4AAAAwDgRcHJQcsHPam3YzUQDAAAAwHgQcHJUY12Vth/uUHtXX9ilAAAAANMGASdHDdyH8xzTRQMAAABpI+DkqEsWVcpM3IcDAAAAjAMBJ0dVFBdoaW3FGffh3HDfU7rhvqdCqgoAAADIbQScHJacaOA4C34CAAAAaSLg5LDG+iqd6I7rlaMs+AkAAACkI62AY2arzGybme0ws0+PcPzvzWxj8PWSmbWlHOtPOfZwJovPd031wYKfTBcNAAAApCU21glmFpX0NUlvl7RX0loze9jdtwyc4+6fTDn//0hqTHmLLne/LHMlzxzn1ZRpVnFMG3Yf13ub68IuBwAAAMh56YzgXCFph7u/4u69klZLuu4s598k6cFMFDfTRSIs+AkAAACMRzoBZ6GkPSnP9wb7zmBmDZLOlfREyu5iM1tnZk+b2bsmXOkM1VhfpW2HTupkNwt+AgAAAGMZ8xK1cbpR0o/cvT9lX4O77zOz8yQ9YWYvuPvLw19oZrdKulWSamtr1dramuHSJqajoyPUWmJtcblL3/vPX+miOVG1tXVJUs70Z6qE3feZiJ5nHz0PB33PPnoeDvqeffQ8N6QTcPZJSr0BZFGwbyQ3SvpY6g533xc8vmJmrUren3NGwHH3+yXdL0nNzc3e0tKSRmlTr7W1VWHW0tTdp7999r/UX1WvlpYl+sa25Bo4LS0rQqspG8Lu+0xEz7OPnoeDvmcfPQ8Hfc8+ep4b0rlEba2kJWZ2rpkVKhlizpgNzcyWSaqW9FTKvmozKwq2ayS9SdKW4a/F6GYVF+iCueVav/t42KUAAAAAOW/MgOPucUm3S3pM0ouS/sXdN5vZXWb2zpRTb5S02t1TV6W8UNI6M3tO0i8kfSF19jWkp6m+Whv2tGloawEAAAAMl9Y9OO6+RtKaYfvuHPb8syO87klJr59EfVByooGH1u3Rq0dPhV0KAAAAkNPSWugT4WpqSC74yXTRAAAAwNkRcKaBC+aWq6Ioxn04AAAAwBgIONNAJGK6rL6KERwAAABgDAScaaKxvlpbD55Qf4KJBgAAAIDREHCmicb6KiVc6uiJh10KAAAAkLMIONNEY12VJAIOAAAAcDYEnGmiqrRQ580tI+AAAAAAZ0HAmUaa6qvV0R1nwU8AAABgFAScaaSpvlrxhKsnngi7FAAAACAnEXCmkcZ67sMBAAAAzoaAM428rrZCEZM6ugk4AAAAwEgIONNINGIqL4rpJCM4AAAAwIgIONNMeVFMnb396urtD7sUAAAAIOcQcKaZiuKYJGn12t0hVwIAAADkHgLONFNZUqCqkgJ9/pGt2nrwRNjlAAAAADmFgDPNmJnOm1umWcUF+sSDG9Xdx6VqAAAAwAACzjRUEI3oS++5RNsOndQXHtkadjkAAABAziDgTFMtS+fpQ29arG8/uVO/2HY47HIAAACAnEDAmcb+bNUyLZtfoT/54XM6crIn7HIAAACA0BFwprHigqi+fGOjTnTH9ac/ek7uHnZJAAAAQKgIONPc0vkV+ot3XKhfbDui7z61K+xyAAAAgFARcPLAB1Y06Kpl8/Q3a17UtoMnwy4HAAAACA0BJw+Yme6+/hLNKi7Qxx/cwNTRAAAAmLEIOHmipryIqaMBAAAw4xFw8kjL0nn64BuZOhoAAAAzFwEnz3z62tNTRx/tYOpoAAAAzCwEnDyTOnX0n/yQqaMBAAAwsxBw8hBTRwMAAGCmIuDkqQ+saNDKpXOZOhoAAAAzCgEnT5mZ7nnPpZpVHGPqaAAAAMwYBJw8VlNepHvecylTRwMAAGDGIODkuZVMHQ0AAIAZhIAzA3z62mVaWluhP/nh80wdDQAAgLxGwJkBigui+spNjTrR3cfU0QAAAMhrBJwZYun8Cv35tcuYOhoAAAB5jYAzg9z8xsVMHQ0AAIC8RsCZQVKnjv7EaqaOBgAAQP5JK+CY2Soz22ZmO8zs0yMc/6CZHTGzjcHXLSnHbjaz7cHXzZksHuM3MHX01oMn9cVHmToaAAAA+WXMgGNmUUlfk3StpIsk3WRmF41w6kPuflnw9c3gtbMlfUbSlZKukPQZM6vOWPWYkIGpo//ptzvVytTRAAAAyCPpjOBcIWmHu7/i7r2SVku6Ls33v0bS4+5+zN2PS3pc0qqJlQpJeui2FXrothWTfp+BqaM/xdTRAAAAyCM21pTBZna9pFXufkvw/P2SrnT321PO+aCkz0s6IuklSZ909z1m9ilJxe7+18F5fyWpy92/NML3uVXSrZJUW1t7+erVqzPw401eR0eHysvLwy5jSuw9mdBnn+rSRXOi+mRTkcws7JIG5XPfcxU9zz56Hg76nn30PBz0PfvoeXatXLnyWXdvHr4/lqH3/w9JD7p7j5ndJuk7kq4azxu4+/2S7pek5uZmb2lpyVBpk9Pa2qpcqWUqxGe/qs/+xxbtKT5XH1ixeMLvc8N9T0lSRkaXpPzvey6i59lHz8NB37OPnoeDvmcfPc8N6Vyitk9SXcrzRcG+Qe7+mrsPXOf0TUmXp/tahGtw6uj/fFEvHWLqaAAAAExv6QSctZKWmNm5ZlYo6UZJD6eeYGYLUp6+U9KLwfZjkq42s+pgcoGrg33IEWamu6+/VBXFMX38QaaOBgAAwPQ2ZsBx97ik25UMJi9K+hd332xmd5nZO4PTPm5mm83sOUkfl/TB4LXHJH1OyZC0VtJdwT7kkLkVRbrneqaOBgAAwPSX1j047r5G0pph++5M2b5D0h2jvPYBSQ9MokZkwcplp6eOfsvr5qpl6bywSwIAAADGLa2FPjEzMHU0AAAApjsCDgYVF0T15Zsu04nuPv3Zj57XWFOIAwAAALmGgIMhls2fpTuuXaafbz2s7z29K+xyAAAAgHEh4OAMH3zjYrUwdTQAAACmIQIOzmBmuoepowEAADANEXAwIqaOBgAAwHREwMGoUqeObt12OOxyAAAAgDERcHBWTB0NAACA6YSAg7Ni6mgAAABMJwQcjImpowEAADBdEHCQFqaOBgAAwHRAwEFaBqaOLi9i6mgAAADkLgIO0ja3okj3vOcSbT14Unc/ui3scsZ0w31P6Yb7ngq7DAAAAGQRAQfjctWyWt28okEP/PZVpo4GAABAziHgYNzueMeFTB0NAACAnETAwbgxdTQAAAByFQEHE5I6dfQ/M3U0AAAAcgQBBxP2wTcu1lteN1d//Z8vqrM3HnY5AAAAAAEHE2dm+tJ7klNHv3zklBIJLlUDAABAuAg4mJSBqaM7e/u15eAJ/XjDXtbIAQAAQGgIOJi0q5bV6tyaMsX7XZ986Dn93ud/rr/+6Ra9fKQj7NIAAAAww8TCLgD5YV5FkeaWF+rjb32dvv/MLn37yZ365m9e1e+dN1vvu7JB1yyvVVEsGnaZAAAAyHMEHGSMmenNS2r05iU1OnyyWz9ct1cP/m63Pv7gBs0pK9T1zYt00xvqtbimLOxSAQAAkKcIOJgS8yqK9bGVF+ijbzlfv9p+RD94Zre++etXdd8vX9GbL6jR+66s19svqlVBlKskAQAAkDkEHEypSMTUsnSeWpbO08H2bj20do8eWrtb//v761VTXqT3Ni/STVfUq252adilAgAAIA8QcJA18yuL9Ym3LdHtV12g1m2H9YNnduveX76sb/zyZf23JXP1vivr9dZl8xRjVAcAAAATRMBB1kUjprdeWKu3XlirfW1dg6M6t33vWdXOKtINzXVq6E+EXSYAAACmIQIOQrWwqkT/9+2v08evukA/35oc1fmHX+yQXFpzaK3ed2W9WpbOUzRiYZcKAACAaYCAg5wQi0Z0zfL5umb5fO051qkv/ug3enpvu36+dZ3OqSzWDW+o1w1vqNP8yuKwS520G+57SpL00G0rQq4EAAAg/3CzA3JO3exSXf+6Qj11x1X6xh816by55fr7n72kN33xCf2v767TL7YdVn/Cwy4TAAAAOYgRHOSsgmhE175+ga59/QLtPHpKD67drR+t26vHtxzSouoS3XRFvd7TvEjzKqb/qA4AAAAygxEcTAuLa8p0x7UX6sk7rtI/3NSouupS3fPYNr3x80/oo//8rH69/YgSjOoAAADMeIzgYFopikX1h5eeoz+89By9fKRDDz6zWz9av1ePbDqohjmluvENyVGdmvKisEsFAABACAg4mLbOn1uuv/yDi/Spa5bq0U0H9YNnduuLj27V3z2+Tdcsn6/2rj7NKuYjno4b7ntKbW1damkJuxIAAIDJ4a8/THvFBVG9q3Gh3tW4UNsPndQPfrdb//rsXp3ojqsoFtEXHtmqay+er0sWVcqM6aYBAADyGQEHeWVJbYU+84fL9WerlukdX/61jnb06B9//Yru/eXLOqeyWNdcPF+rls9X8+LZrK0DAACQh9IKOGa2StKXJUUlfdPdvzDs+P+VdIukuKQjkv6nu+8KjvVLeiE4dbe7vzNDtQOjKi6Iam5FkeZWFOm+91+un714WI9uOqDvP7Nb//TbnaopL9TbL5qvay+erxXnz1FBlPk2AAAA8sGYAcfMopK+JuntkvZKWmtmD7v7lpTTNkhqdvdOM/uopLsl3RAc63L3yzJcN5C2qtJCXX/5Il1/+SJ19MT1i62H9ejmg/rJxn168He7Nas4prddVKtrL16g319So+KCaNglAwAAYILSGcG5QtIOd39FksxstaTrJA0GHHf/Rcr5T0v6H5ksEsiU8qLY4Cxs3X39+vX2o3pk0wH9bMsh/dv6fSotjGrlsnlatXy+Vi6bp/IiruIEAACYTsz97GuHmNn1kla5+y3B8/dLutLdbx/l/K9KOujufx08j0vaqOTla19w938f5XW3SrpVkmpray9fvXr1xH6iDOvo6FB5eXnYZeS8zz/TJUm648qSjLxfJvo+npriCdfWY/1ad6hf6w/FdaJXikWki+dE1Tw/qsvmxlRemJl7djLdq0z4/DNd6u/v11++MXc+67nYp0zj90s46Hv20fNw0Pfso+fZtXLlymfdvXn4/oz+72kz+x+SmiW9JWV3g7vvM7PzJD1hZi+4+8vDX+vu90u6X5Kam5u9JUfmq21tbVWu1JLLvrHtKUlSS8uKjLxfJvo+3preFjz2J1zP7jquRzYd0GObDuqbL3QrFunTivPn6Jrl83X18lrNqyjOWl3Z8I1tT6mtrS2nPuu52KdM4/dLOOh79tHzcND37KPnuSGdgLNPUl3K80XBviHM7G2S/kLSW9y9Z2C/u+8LHl8xs1ZJjZLOCDhArohGTFecO1tXnDtbd/7BRXp+b7se3XxQj246qL/89036q59sUnNDtVZdvEDXLK/VourSsEtGFt1wXzJ4PZWbtF8AABUZSURBVHRb/gYvAACms3QCzlpJS8zsXCWDzY2S3pd6gpk1SrpPyUvZDqfsr5bU6e49ZlYj6U1KTkAATAtmpkvrqnRpXZX+9JqleulQhx7ZdECPbjqoz/10iz730y26ZFGlrlmenJHtvLkMSwMAAIRpzIDj7nEzu13SY0pOE/2Au282s7skrXP3hyXdI6lc0g+DhRQHpoO+UNJ9ZpaQFFHyHpwtI34jIMeZmZbOr9DS+RX647e9Tq8ePaXHNh/UI5sO6p7Htumex7bpdbXlWnXxAq1aPl8XLqhgYVEAAIAsS+seHHdfI2nNsH13pmy/7YwXJfc/Ken1kykQ08NMvFzn3JoyfeQt5+sjbzlf+9u69FhwGdtXn9iur/x8uxrmlGpVsLDopYuqFGFhUQAAgCnHHLhABpxTVaIPvelcfehN5+poR48e33JIj2w6qG/9+lXd98tXtKCyWNcsn69VF8+XuzOyg4ziviAAAE4j4AAZVlNepJuuqNdNV9SrvbNPP996SI9uOqgHf7db335yp2IRU0lhVJ98aKMWVpXonKoSLawu0cKqYp1TVaLSQv61BAAAmCj+kgKmUGVpgd7dtEjvblqkUz1x/fKlI7rzJ5vU05fQ7149poMnutWfGLoWVXVpgRZWl+icyoHgUzIkCM0pK2QECAAAYBQEHCBLyopiesfrF+g7T+6UlLycKN6f0OGTPdrX1qX9bV3aezz5uK+tSztfO6Xf7jiqU739Q96nKBYZDDznVBVrYVVp8jEIQwsqS1QYi4TwEwIAAISPgAOEKBaNBEGlZMTj7q4TXXHtbevU/rZu7Tveqf3t3dp3PBmCfrHtiI6c7BnyGjNpbnlRchSoqkSLBkZ/UkaBZhXHGAXCjHTDfU+pra1LrMMHAPmLgAPkMDNTZWmBKksrtfycyhHP6Yn360Bbd3IEKBgJ2ne8S/vbu7Rl/wk9vuWQeuOJIa8pL4oFoz/J0LO/rUvxXtdze9rUMKdUVaWF2fjxkMeY+AAAEBYCDjDNFcWiWlxTpsU1ZSMeTyRcr53qHbwMbmD0Z+D5hj1tauvskyRd97XfSpJmFcdUP6dUDbPLgsdS1c8uVf2cUi2oLFGUKa8BAECOIuAAeS4SMc2tKNLciiJdVlc14jnXf+NJHT3epj+/rkm7j3Vq12ud2nWsU5v3t+uxzQcVT5kIoTAa0aLqEtXPCULP7FI1zClTw5xS1VWXqqQwmq0fDcgLjHYBQGYRcAAoGjEVRU1XL59/xrH+hGt/W9dg8Nl9rFO7j53Srtc69ezO4zrZEx9y/ryKIjXMKVX97LLgsXRwFGg2M8AB0wbBC8B0RcABcFbRiKludqnqZpfqTRcMPebuOt7ZF4SfU9odjPzsfq1Tv91xVP+6vnvI+eVFsZRRn9LBy+Aa5pRqQWWxYlFmfwMAAJNDwAEwYWam2WWFml1WOOLlb919/dozZOQnGYReOnxST2w9rN7+05MfxCKmhdUlg+HnQHuXCqMR/WzLIcWipsJoRAWxiGIRU0E0osJh2wXRyOnzohHuEwLyEKNKANJBwAEwZYoLolpSW6EltRVnHOtPuA6d6A7Cz6nB+372HOvUfzx3QO1dyYkPbvnuugl9bzMlw080ooKoKTZsO3lsYNsGz42NsF2Qcs7e452KRSL6ycZ9mlNWpNllhaopL1R1WaEKGIECZhxCF5B7CDjIW/zHJrdFIza4BtCK8+eccfz///pv1duf0OfffYl6+xPqiycUT/jgdl+/K55IqDfY7utPBF/pbA97Hnd1xOOD232J0/uHnz8w4cInVm88o+ZZxTHVlCdDz5zyQs0uK1JNeeHgKNeQY0zFDWAKseZT+gip+YeAAyAnxaIRxaIRXbxw5PV/wvLee59UPOG6+/pL9VpHj46d6tXRU7061tGrY6d6BrdfPXpKz+46rmOnepUyCd0QZQVS7bOtmlNWmBwNKi9UTRCGZpcXJbfLk8eqSwu4RwnAtEaQQLYQcABgHMxMBVHTBfPKdcG88jHPTyRcbV19yfDT0atjp3r12qlevdbRoxdeelXFVbP0WkePXjnaobU7e3W8c+RAZCZVlRQkR4DKioIRokLNKS/SwRPdKoiYfr39iKpKClVVWqCq0gKVF8WYtQ4ApiHC4OQQcABgCkUipydiuGDe0GOtsf1qaWkasq8/4Wrv6tNrHT1BEEqODJ3e7tXRjh7tONyhY6d6dayzVx4Eovd/63dD3isaMVWVFKiytEDVpYWD21UlhaoOQlBlsL+6tDB4XqAKghEAYJjpFLoIOACQQ6IpgWhJGuf3J1zvufdJ9fW7PvOHF6mts0/HO3vV3tU3uN3W1af2zj4dOtmtrQdPqr2rTx3D1i8aXsPpMFSgqiD8DIwOVacEo6ogPBGMAAC5goADANNYNDIw05vUvHh22q/r60+orbNP7V29auvsO2swOnyyW9vSDEaVQeg5fKJH0Yjplu+slWQykyImmUyRSPJRJkXMZFJwfGA7eb4p2Gcj74sEYSr1tZFI8nG09957vFM9Pa7vPbVTZUUxlRXFVBE8lhXFVFGcfCwtiCrCVOMAMC0RcABgBiqIRjS3okhzK4rG9bp0g9HRk0cU70/oQHu3Ep5cFFaSEu5yDx4lKWV74JgH5w/Zp2BfsJ36Phph32jvPeCvfrL5rD+nmVRWGFNZUVTlRTGVBwFoyHZxsF0YVXlxgcqLoiOeU1YYY10mAMgiAg4AIG3pBqNcvVb7vfc+qeNtbfrBx96qUz1xdQRfZ2x3x9XR03/G/t2nOoec39c/yhR5w5QURAcDUTL8nA5Orx49pYiZ7n5065A1l2JD1mpK2R8xFcSCtZqC7YJIRAUxUywyfA2noWs5cQkhgJmAgAMAmDHMTBGzCY1ejaQn3q9TQRA62R3Xqd4gEHWPEJp6+oeEo/1t3TrVGw+mEnf9469fSTswTVQsYmcsYBuLRFQYC8JSSih66dBJmZk+/O21yf2x5LHCwdcmQ1VB5PT2kGNRU2Esub31UFy+7XDK8WBB3VhkxCA2sNgulwkCmAgCDpBlufZ/tAFMXFEsqqJYVLPLJr5wa+pol7srnvChi8ueZfHaeH9CvannJDxYCPf0djwx+qK38X4f+vqUY+5Svyd08ET3yAvnxpPfuzc4d0wb1o67N8l7zE4HnhPdfYqY6aovtZ4OWEEwKkwNSrFhzwfD1Ajnx1Kfn953tjA2eCwWUcJdxDAgtxBwAADIEQPrLBXkwKKu47nMsD9xZoDqjZ9+/tQzv9MljU1B4EqGqmQQC84deO3AvkRCffHT7zlw/LFNB+WSLjpn1hnfq7M3PsL7DXve7+ofbeXdSTr/z9ck/9lFksEndURsxMsMU44Nv6wwFhl2/vBjsYgKIqePFQaXNA5uR0wnu+Pqjrs2729XLBJRNGKKRSz5GA0eh+8PHrmUEdMdAQcAAExKNGKKRqIqLoiOePxAZVRN9dWT/j7bD3VIkr76vqYxzhxdahiLDwlEZwazwWNnCWPfeWqn3F3valw46ohY6ijawDmpgWxg1O706xNDjk0mlP1/X/nNuF8THRZ4ko+RoQHJ7PR50WHHR3pd1LTjcPKf3x+v3hBMHJIygYgUTBjiSiSSj6ePJw8OPz/1uQafp0xWkvIa+fD3PP0e2w+fVESm9/3j0ykjfkNH6tIaEYyZth+Iq2fzwXGNCHJJZuYRcAAAwIwxVhgbr5+9eEiS9CfXLMvI+40kkQhGtYZdljgYiAZGvBKJwVGyu/5jszo6TunOdzepP+FBUEq+7vTzlP0JV3//KPsHnvefub8/Mfz9kgGwqy943n96/8A08+t3tw1O+2428tTuFkwvr5RzklPGnz6WfF3qa4adG0xHb5IsIpkiZ7yHlLw3zV3qjSd0qieu3tTLMeOJoc+D8HlWzz07oX/OqZdkdvX2y0z6/bufGHKf2/B71VIvt4wNPj99qebASN/pSzGHjigOBK+B0b+CUe6LK4xG1NefkJnJ3XN+lI+AA0AP3bZCra2tYZcBABhBJGIqikRVNI6/2qpKC6XeTl2zfP7UFTZOuTq74njrcvch96QNjgDGE/rt08/osqbLTx+PDx0hPD1KOOz1wy7J/M/nD8hdekPD7MFRw4FA29efUHdfQie740PeZyCMxROnRxh7+xMZ71c84SqIEnAAAACAvGBmyUvYYmfeK7ezPKLl51RO+nts2X9CkvR3N1w2qfcZmLgkNRwN3p+WOL2dGqKG3rd2Oozd/8tX5HLFpsGldAQcAAAAIA+dnrhEKtHkLst8eOP+wffMdQQcADkp1y5hGJCrdQEAgKTw56EEAAAAgAwh4AAAAADIG1yiBgDTXC5eNpeLNQEAZgZGcAAAM8ZDt63QHVeWhF0GAGAKMYIDAECIGO0CgMwi4AAAgDPkYvDKxZoA5B4CDgAAwATlauh66LYVam1tDbuMIXK5V8gvaQUcM1sl6cuSopK+6e5fGHa8SNJ3JV0u6TVJN7j7zuDYHZI+LKlf0sfd/bGMVQ8AAADkmVwMXblY02jGnGTAzKKSvibpWkkXSbrJzC4adtqHJR139wsk/b2kLwavvUjSjZKWS1ol6evB+wEAAABAxqUzi9oVkna4+yvu3itptaTrhp1znaTvBNs/kvRWM7Ng/2p373H3VyXtCN4PAAAAADIunUvUFkrak/J8r6QrRzvH3eNm1i5pTrD/6WGvXTjSNzGzWyXdKkm1tbU5c91oR0dHztQyk9D37KPn2UfPw0Hfs4+eh4O+Zx89zw05M8mAu98v6X5Jam5u9paWlnALCrS2tipXaplJ6Hv20fPso+fhoO/ZR8/DQd+zj57nhnQuUdsnqS7l+aJg34jnmFlMUqWSkw2k81oAAAAAyIh0As5aSUvM7FwzK1Ry0oCHh53zsKSbg+3rJT3h7h7sv9HMiszsXElLJP0uM6UDAAAAwFBjXqIW3FNzu6THlJwm+gF332xmd0la5+4PS/qWpO+Z2Q5Jx5QMQQrO+xdJWyTFJX3M3fun6GcBAAAAMMOldQ+Ou6+RtGbYvjtTtrslvWeU1/6NpL+ZRI0AAAAAkJZ0LlEDAAAAgGmBgAMAAAAgbxBwAAAAAOQNAg4AAACAvEHAAQAAAJA3CDgAAAAA8oYl1+PMLWZ2RNKusOsI1Eg6GnYRMxB9zz56nn30PBz0PfvoeTjoe/bR8+xqcPe5w3fmZMDJJWa2zt2bw65jpqHv2UfPs4+eh4O+Zx89Dwd9zz56nhu4RA0AAABA3iDgAAAAAMgbBJyx3R92ATMUfc8+ep599Dwc9D376Hk46Hv20fMcwD04AAAAAPIGIzgAAAAA8gYBJ2Bmq8xsm5ntMLNPj3C8yMweCo4/Y2aLs19l/jCzOjP7hZltMbPNZvaJEc5pMbN2M9sYfN0ZRq35xsx2mtkLQU/XjXDczOwrwWf9eTNrCqPOfGFmS1M+wxvN7ISZ/fGwc/isZ4CZPWBmh81sU8q+2Wb2uJltDx6rR3ntzcE5283s5uxVPb2N0vN7zGxr8Pvjx2ZWNcprz/q7CKMbpe+fNbN9Kb9H3jHKa8/69w5GNkrPH0rp904z2zjKa/msZxmXqEkys6iklyS9XdJeSWsl3eTuW1LO+d+SLnH3j5jZjZL+u7vfEErBecDMFkha4O7rzaxC0rOS3jWs5y2SPuXufxBSmXnJzHZKanb3EefpD/6j+H8kvUPSlZK+7O5XZq/C/BX8rtkn6Up335Wyv0V81ifNzP6bpA5J33X3i4N9d0s65u5fCP6Yq3b3Pxv2utmS1klqluRK/j663N2PZ/UHmIZG6fnVkp5w97iZfVGShvc8OG+nzvK7CKMbpe+fldTh7l86y+vG/HsHIxup58OO/62kdne/a4RjO8VnPasYwUm6QtIOd3/F3XslrZZ03bBzrpP0nWD7R5LeamaWxRrzirsfcPf1wfZJSS9KWhhuVQhcp+QvcHf3pyVVBYEUk/dWSS+nhhtkjrv/StKxYbtTf3d/R9K7RnjpNZIed/djQah5XNKqKSs0j4zUc3f/L3ePB0+flrQo64XluVE+6+lI5+8djOBsPQ/+HnyvpAezWhRGRcBJWihpT8rzvTrzj+3Bc4Jf3O2S5mSlujwXXO7XKOmZEQ6vMLPnzOwRM1ue1cLyl0v6LzN71sxuHeF4Ov8+YGJu1Oj/AeSzPjVq3f1AsH1QUu0I5/CZnzr/U9Ijoxwb63cRxu/24NLAB0a5HJPP+tT4fUmH3H37KMf5rGcZAQehMrNySf8q6Y/d/cSww+slNbj7pZL+QdK/Z7u+PPVmd2+SdK2kjwXD7phiZlYo6Z2SfjjCYT7rWeDJa7K5LjtLzOwvJMUlfX+UU/hdlFnfkHS+pMskHZD0t+GWM6PcpLOP3vBZzzICTtI+SXUpzxcF+0Y8x8xikiolvZaV6vKUmRUoGW6+7+7/Nvy4u59w945ge42kAjOryXKZecfd9wWPhyX9WMlLFlKl8+8Dxu9aSevd/dDwA3zWp9ShgUssg8fDI5zDZz7DzOyDkv5A0h/5KDf7pvG7COPg7ofcvd/dE5L+USP3k896hgV/E75b0kOjncNnPfsIOElrJS0xs3OD/8t6o6SHh53zsKSBmXWuV/IGSv5P4AQF16t+S9KL7v53o5wzf+A+JzO7QsnPK6FyEsysLJjUQWZWJulqSZuGnfawpA9Y0u8pedPkAWGyRv0/fHzWp1Tq7+6bJf1khHMek3S1mVUHl/VcHezDBJjZKkl/Kumd7t45yjnp/C7COAy7V/K/a+R+pvP3DsbnbZK2uvvekQ7yWQ9HLOwCckEw08vtSv4HLSrpAXffbGZ3SVrn7g8r+cf498xsh5I3md0YXsV54U2S3i/phZRpFf9cUr0kufu9SgbJj5pZXFKXpBsJlZNWK+nHwd/SMUk/cPdHzewj0mDf1yg5g9oOSZ2SPhRSrXkj+I/a2yXdlrIvted81jPAzB6U1CKpxsz2SvqMpC9I+hcz+7CkXUreCCwza5b0EXe/xd2PmdnnlPzjT5LucveJ3MA944zS8zskFUl6PPhd83QwA+k5kr7p7u/QKL+LQvgRpqVR+t5iZpcpeRnmTgW/b1L7PtrfOyH8CNPOSD13929phHsr+ayHj2miAQAAAOQNLlEDAAAAkDcIOAAAAADyBgEHAAAAQN4g4AAAAADIGwQcAAAAAHmDgAMAAAAgbxBwAAAAAOQNAg4AAACAvPH/AC6sIIZTSKeWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1008x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = (14, 6)\n",
    "\n",
    "plt.errorbar(range(n_rounds), np.mean(list_loss_test_worker, axis=0), np.std(list_loss_test_worker, axis=0))\n",
    "plt.grid()\n",
    "# plt.scatter(range(n_rounds), np.mean(list_loss_test_worker, axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean everything (in case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(n_workers):\n",
    "    workers[i].clear_objects()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_ptr_worker = [transform(mnist_train.data[n_samples_train_per_worker*i:n_samples_train_per_worker*(i+1)]).send(workers[i]) for i in range(n_workers)]\n",
    "Y_train_ptr_worker = [mnist_train.targets[n_samples_train_per_worker*i:n_samples_train_per_worker*(i+1)].send(workers[i]) for i in range(n_workers)]\n",
    "\n",
    "X_test_ptr_worker = [transform(mnist_test.data[n_samples_test_per_worker*i:n_samples_test_per_worker*(i+1)]).send(workers[i]) for i in range(n_workers)]\n",
    "Y_test_ptr_worker = [mnist_test.targets[n_samples_test_per_worker*i:n_samples_test_per_worker*(i+1)].send(workers[i]) for i in range(n_workers)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net()\n",
    "model_worker = [model.copy().send(workers[i]) for i in range(n_workers)] # Important: starting from the same initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([10, 30])\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "too many indices for tensor of dimension 1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-97-dc5981f4841e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/trace.py\u001b[0m in \u001b[0;36mtrace_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     81\u001b[0m                 \u001b[0msyft\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook.py\u001b[0m in \u001b[0;36moverloaded_native_method\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    456\u001b[0m                 \u001b[0;31m# Send the new command to the appropriate class and get the response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m                 \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_self\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mnew_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mnew_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    459\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m                 \u001b[0;31m# For inplace methods, just directly return self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook.py\u001b[0m in \u001b[0;36moverloaded_pointer_method\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    619\u001b[0m             \u001b[0mcommand\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mattr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 621\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mowner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    622\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    623\u001b[0m             \u001b[0;31m# For inplace methods, just directly return self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/base.py\u001b[0m in \u001b[0;36msend_command\u001b[0;34m(self, recipient, message, return_ids)\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    582\u001b[0m             \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensorCommandMessage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomputation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 583\u001b[0;31m             \u001b[0mret_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrecipient\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    584\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mResponseSignatureError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0mret_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/base.py\u001b[0m in \u001b[0;36msend_msg\u001b[0;34m(self, message, location)\u001b[0m\n\u001b[1;32m    285\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0;31m# Step 2: send the message and wait for a response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m         \u001b[0mbin_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbin_message\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m         \u001b[0;31m# Step 3: deserialize the response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/virtual.py\u001b[0m in \u001b[0;36m_send_msg\u001b[0;34m(self, message, location)\u001b[0m\n\u001b[1;32m     13\u001b[0m             \u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage_pending_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_recv_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_recv_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbin\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mbin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/virtual.py\u001b[0m in \u001b[0;36m_recv_msg\u001b[0;34m(self, message)\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_recv_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbin\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mbin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;34m\"\"\"receive message\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_msg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/base.py\u001b[0m in \u001b[0;36mrecv_msg\u001b[0;34m(self, bin_message)\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    319\u001b[0m         \u001b[0;31m# Step 1: route message to appropriate function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_message_router\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    322\u001b[0m         \u001b[0;31m# Step 2: Serialize the message to simple python objects\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/base.py\u001b[0m in \u001b[0;36mexecute_tensor_command\u001b[0;34m(self, cmd)\u001b[0m\n\u001b[1;32m    432\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexecute_tensor_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmd\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensorCommandMessage\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mPointerTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mComputationAction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 434\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_computation_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    435\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_communication_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/workers/base.py\u001b[0m in \u001b[0;36mexecute_computation_action\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m    466\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    467\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 468\u001b[0;31m                     \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_self\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    469\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m                     \u001b[0;31m# TODO Andrew thinks this is gross, please fix. Instead need to properly deserialize strings\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/trace.py\u001b[0m in \u001b[0;36mtrace_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     81\u001b[0m                 \u001b[0msyft\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook.py\u001b[0m in \u001b[0;36moverloaded_native_method\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    417\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    418\u001b[0m                     \u001b[0;31m# we can make some errors more descriptive with this method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 419\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mroute_method_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    421\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# means that there is a wrapper to remove\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/privacy-eeg/lib/python3.7/site-packages/syft/generic/frameworks/hook/hook.py\u001b[0m in \u001b[0;36moverloaded_native_method\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    413\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 415\u001b[0;31m                     \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: too many indices for tensor of dimension 1"
     ]
    }
   ],
   "source": [
    "# bob = sy.VirtualWorker(hook, id=\"bob\")\n",
    "\n",
    "# a = torch.tensor([10,20,30])\n",
    "# print(a[[0,2]])\n",
    "\n",
    "# b = torch.tensor([10,20,30]).send(bob)\n",
    "# print(b[[0,2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sy.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}