MFT_Transformer_HSI_Only_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a7d3e1de",
   "metadata": {},
   "source": [
    "## Transformer Model for Hyperspectral Image Classification\n",
    "\n",
    "\n",
    "### The MFT model with only HSI images used for classification. The other multimodal data is NOT used as external CLS token, rather the external CLS token is taken as random "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e5b2e976",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-11T11:17:16.979544Z",
     "start_time": "2022-08-11T11:17:16.971125Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"./../\")\n",
    "import matplotlib.pyplot as plt\n",
    "from torchsummary import summary\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score\n",
    "import math\n",
    "from PIL import Image\n",
    "import time\n",
    "from scipy.io import loadmat as loadmat\n",
    "from scipy import io\n",
    "import random\n",
    "import numpy as np\n",
    "import os\n",
    "import torch \n",
    "import torch.utils.data as dataf\n",
    "import torch.nn as nn\n",
    "from operator import truediv\n",
    "import record\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from mft_model import MFT, Transformer\n",
    "from dataset import Multimodal_Dataset_Train, Multimodal_Dataset_Test\n",
    "import torch.backends.cudnn as cudnn\n",
    "cudnn.deterministic = True\n",
    "cudnn.benchmark = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "199d9d19",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_confusion_matrix(y_test,y_pred, plt_name):\n",
    "    df_cm = pd.DataFrame(confusion_matrix(y_test, y_pred), range(6),range(6))\n",
    "    df_cm.columns = ['Buildings','Woods', 'Roads', 'Apples', 'ground', 'Vineyard']\n",
    "    df_cm = df_cm.rename({0:'Buildings',1:'Woods', 2:'Roads', 3:'Apples', 4:'ground', 5:'Vineyard'})\n",
    "    df_cm.index.name = 'Actual'\n",
    "    df_cm.columns.name = 'Predicted'\n",
    "    sns.set(font_scale=0.9)#for label size\n",
    "    sns.heatmap(df_cm, cmap=\"Blues\",annot=True,annot_kws={\"size\": 16}, fmt='g')\n",
    "    plt.savefig(''+str(plt_name)+'.eps', format='eps')\n",
    "\n",
    "def AA_andEachClassAccuracy(confusion_matrix):\n",
    "    counter = confusion_matrix.shape[0]\n",
    "    list_diag = np.diag(confusion_matrix)\n",
    "    list_raw_sum = np.sum(confusion_matrix, axis=1)\n",
    "    each_acc = np.nan_to_num(truediv(list_diag, list_raw_sum))\n",
    "    average_acc = np.mean(each_acc)\n",
    "    return each_acc, average_acc\n",
    "\n",
    "def reports (xtest,xtest2,ytest,name,model, HSIOnly, iternum):\n",
    "    pred_y = np.empty((len(ytest)), dtype=np.float32)\n",
    "    number = len(ytest) // testSizeNumber\n",
    "    for i in range(number):\n",
    "        temp = xtest[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "        temp = temp.cuda()\n",
    "        temp1 = xtest2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "        temp1 = temp1.cuda()\n",
    "        if HSIOnly:\n",
    "            temp2 = model(temp)\n",
    "        else:\n",
    "            temp2 = model(temp,temp1)\n",
    "        temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "        pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n",
    "        del temp, temp2, temp3,temp1\n",
    "\n",
    "    if (i + 1) * testSizeNumber < len(ytest):\n",
    "        temp = xtest[(i + 1) * testSizeNumber:len(ytest), :, :]\n",
    "        temp = temp.cuda()\n",
    "        temp1 = xtest2[(i + 1) * testSizeNumber:len(ytest), :, :]\n",
    "        temp1 = temp1.cuda()\n",
    "        if HSIOnly:\n",
    "            temp2 = model(temp)\n",
    "        else:\n",
    "            temp2 = model(temp,temp1)\n",
    "        temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "        pred_y[(i + 1) * testSizeNumber:len(ytest)] = temp3.cpu()\n",
    "        del temp, temp2, temp3,temp1\n",
    "\n",
    "    pred_y = torch.from_numpy(pred_y).long()\n",
    "    \n",
    "    oa = accuracy_score(ytest, pred_y)\n",
    "    confusion = confusion_matrix(ytest, pred_y)\n",
    "    each_acc, aa = AA_andEachClassAccuracy(confusion)\n",
    "    kappa = cohen_kappa_score(ytest, pred_y)\n",
    "    get_confusion_matrix(ytest, pred_y, 'test_'+str(iternum)+'')\n",
    "    \n",
    "    return confusion, oa*100, each_acc*100, aa*100, kappa*100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ac5c476f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-11T11:18:10.978783Z",
     "start_time": "2022-08-11T11:18:01.894932Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------- Dataset details for  Trento  ---------------------------------------------\n",
      "\n",
      "\n",
      "HSI Train data shape =  torch.Size([819, 63, 11, 11])\n",
      "LIDAR Train data shape =  1\n",
      "Train label shape =  torch.Size([819])\n",
      "HSI Test data shape =  torch.Size([29395, 63, 11, 11])\n",
      "LIDAR Test data shape =  1\n",
      "Test label shape =  torch.Size([29395])\n",
      "Number of Classes =  6\n",
      "\n",
      "\n",
      "---------------------------------- Model Summary ---------------------------------------------\n",
      "\n",
      "\n",
      "===============================================================================================\n",
      "Layer (type:depth-idx)                        Output Shape              Param #\n",
      "===============================================================================================\n",
      "├─Sequential: 1-1                             [-1, 8, 55, 11, 11]       --\n",
      "|    └─Conv3d: 2-1                            [-1, 8, 55, 11, 11]       656\n",
      "|    └─GroupNorm: 2-2                         [-1, 8, 55, 11, 11]       16\n",
      "|    └─ReLU: 2-3                              [-1, 8, 55, 11, 11]       --\n",
      "├─Sequential: 1-2                             [-1, 64, 11, 11]          --\n",
      "|    └─HetConv: 2-4                           [-1, 64, 11, 11]          --\n",
      "|    |    └─Conv2d: 3-1                       [-1, 64, 11, 11]          31,744\n",
      "|    |    └─Conv2d: 3-2                       [-1, 64, 11, 11]          28,224\n",
      "|    └─GroupNorm: 2-5                         [-1, 64, 11, 11]          128\n",
      "|    └─ReLU: 2-6                              [-1, 64, 11, 11]          --\n",
      "├─Dropout: 1-3                                [-1, 5, 64]               --\n",
      "├─TransformerEncoder: 1-4                     [-1, 64]                  --\n",
      "|    └─ModuleList: 2                          []                        --\n",
      "|    |    └─Block: 3-3                        [-1, 5, 64]               100,736\n",
      "|    |    └─Block: 3-4                        [-1, 5, 64]               100,736\n",
      "|    └─LayerNorm: 2-7                         [-1, 5, 64]               128\n",
      "├─Linear: 1-5                                 [-1, 6]                   390\n",
      "===============================================================================================\n",
      "Total params: 262,758\n",
      "Trainable params: 262,758\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (M): 12.27\n",
      "===============================================================================================\n",
      "Input size (MB): 0.03\n",
      "Forward/backward pass size (MB): 1.00\n",
      "Params size (MB): 1.00\n",
      "Estimated Total Size (MB): 2.03\n",
      "===============================================================================================\n",
      "\n",
      "\n",
      "---------------------------------- Training started for  Trento  ---------------------------------------------\n",
      "\n",
      "\n",
      "Epoch:  0 | train loss: 2.4431 | test accuracy: 13.2846\n",
      "Epoch:  1 | train loss: 1.8137 | test accuracy: 39.9388\n",
      "Epoch:  2 | train loss: 1.2580 | test accuracy: 60.0204\n",
      "Epoch:  3 | train loss: 0.8837 | test accuracy: 77.5608\n",
      "Epoch:  4 | train loss: 0.5837 | test accuracy: 83.5006\n",
      "Epoch:  5 | train loss: 0.4990 | test accuracy: 85.8343\n",
      "Epoch:  6 | train loss: 0.4421 | test accuracy: 90.6175\n",
      "Epoch:  7 | train loss: 0.2673 | test accuracy: 87.5795\n",
      "Epoch:  8 | train loss: 0.2590 | test accuracy: 92.6450\n",
      "Epoch:  9 | train loss: 0.1682 | test accuracy: 88.9335\n",
      "Epoch:  10 | train loss: 0.1247 | test accuracy: 93.4343\n",
      "Epoch:  11 | train loss: 0.1216 | test accuracy: 92.0020\n",
      "Epoch:  12 | train loss: 0.1390 | test accuracy: 94.6862\n",
      "Epoch:  13 | train loss: 0.0527 | test accuracy: 93.8391\n",
      "Epoch:  14 | train loss: 0.0389 | test accuracy: 94.8018\n",
      "Epoch:  15 | train loss: 0.0494 | test accuracy: 93.2472\n",
      "Epoch:  16 | train loss: 0.0514 | test accuracy: 94.1759\n",
      "Epoch:  17 | train loss: 0.0343 | test accuracy: 93.9275\n",
      "Epoch:  18 | train loss: 0.0201 | test accuracy: 94.1453\n",
      "Epoch:  19 | train loss: 0.0972 | test accuracy: 93.7676\n",
      "Epoch:  20 | train loss: 0.0247 | test accuracy: 94.1929\n",
      "Epoch:  21 | train loss: 0.0136 | test accuracy: 93.7540\n",
      "Epoch:  22 | train loss: 0.0368 | test accuracy: 92.4545\n",
      "Epoch:  23 | train loss: 0.0227 | test accuracy: 91.0291\n",
      "Epoch:  24 | train loss: 0.0211 | test accuracy: 92.4885\n",
      "Epoch:  25 | train loss: 0.0148 | test accuracy: 83.6945\n",
      "Epoch:  26 | train loss: 0.1034 | test accuracy: 92.0463\n",
      "Epoch:  27 | train loss: 0.0476 | test accuracy: 94.9923\n",
      "Epoch:  28 | train loss: 0.0422 | test accuracy: 94.5569\n",
      "Epoch:  29 | train loss: 0.0385 | test accuracy: 94.9651\n",
      "Epoch:  30 | train loss: 0.0166 | test accuracy: 93.9752\n",
      "Epoch:  31 | train loss: 0.0089 | test accuracy: 94.5535\n",
      "Epoch:  32 | train loss: 0.0466 | test accuracy: 93.9718\n",
      "Epoch:  33 | train loss: 0.0093 | test accuracy: 94.3086\n",
      "Epoch:  34 | train loss: 0.0094 | test accuracy: 95.1420\n",
      "Epoch:  35 | train loss: 0.0138 | test accuracy: 94.6419\n",
      "Epoch:  36 | train loss: 0.0154 | test accuracy: 93.4343\n",
      "Epoch:  37 | train loss: 0.0150 | test accuracy: 94.2371\n",
      "Epoch:  38 | train loss: 0.0101 | test accuracy: 93.0668\n",
      "Epoch:  39 | train loss: 0.0066 | test accuracy: 94.5127\n",
      "Epoch:  40 | train loss: 0.0105 | test accuracy: 94.7338\n",
      "Epoch:  41 | train loss: 0.0088 | test accuracy: 94.0670\n",
      "Epoch:  42 | train loss: 0.0119 | test accuracy: 92.2266\n",
      "Epoch:  43 | train loss: 0.0084 | test accuracy: 94.3630\n",
      "Epoch:  44 | train loss: 0.0096 | test accuracy: 95.4550\n",
      "Epoch:  45 | train loss: 0.0120 | test accuracy: 93.9275\n",
      "Epoch:  46 | train loss: 0.0156 | test accuracy: 94.2439\n",
      "Epoch:  47 | train loss: 0.0098 | test accuracy: 93.7132\n",
      "Epoch:  48 | train loss: 0.0253 | test accuracy: 93.4138\n",
      "Epoch:  49 | train loss: 0.0075 | test accuracy: 94.2643\n",
      "Epoch:  50 | train loss: 0.2419 | test accuracy: 94.4412\n",
      "Epoch:  51 | train loss: 0.0185 | test accuracy: 94.8733\n",
      "Epoch:  52 | train loss: 0.0131 | test accuracy: 94.8937\n",
      "Epoch:  53 | train loss: 0.0102 | test accuracy: 93.3220\n",
      "Epoch:  54 | train loss: 0.0077 | test accuracy: 94.3222\n",
      "Epoch:  55 | train loss: 0.0136 | test accuracy: 94.3766\n",
      "Epoch:  56 | train loss: 0.0071 | test accuracy: 93.8527\n",
      "Epoch:  57 | train loss: 0.0112 | test accuracy: 94.6828\n",
      "Epoch:  58 | train loss: 0.0096 | test accuracy: 93.6622\n",
      "Epoch:  59 | train loss: 0.0084 | test accuracy: 94.5025\n",
      "Epoch:  60 | train loss: 0.0090 | test accuracy: 93.9956\n",
      "Epoch:  61 | train loss: 0.0090 | test accuracy: 92.7641\n",
      "Epoch:  62 | train loss: 0.0100 | test accuracy: 94.4616\n",
      "Epoch:  63 | train loss: 0.0088 | test accuracy: 94.6964\n",
      "Epoch:  64 | train loss: 0.0088 | test accuracy: 93.6554\n",
      "Epoch:  65 | train loss: 0.0083 | test accuracy: 92.5532\n",
      "Epoch:  66 | train loss: 0.0088 | test accuracy: 94.1351\n",
      "Epoch:  67 | train loss: 0.0120 | test accuracy: 93.8969\n",
      "Epoch:  68 | train loss: 0.0117 | test accuracy: 94.2133\n",
      "Epoch:  69 | train loss: 0.0102 | test accuracy: 94.2303\n",
      "Epoch:  70 | train loss: 0.0103 | test accuracy: 93.8731\n",
      "Epoch:  71 | train loss: 0.0103 | test accuracy: 93.1315\n",
      "Epoch:  72 | train loss: 0.0107 | test accuracy: 94.2303\n",
      "Epoch:  73 | train loss: 0.0099 | test accuracy: 94.6113\n",
      "Epoch:  74 | train loss: 0.0146 | test accuracy: 94.9379\n",
      "Epoch:  75 | train loss: 0.0137 | test accuracy: 94.9277\n",
      "Epoch:  76 | train loss: 0.0191 | test accuracy: 94.9889\n",
      "Epoch:  77 | train loss: 0.0072 | test accuracy: 94.6658\n",
      "Epoch:  78 | train loss: 0.0087 | test accuracy: 93.3458\n",
      "Epoch:  79 | train loss: 0.0103 | test accuracy: 93.5329\n",
      "Epoch:  80 | train loss: 0.0108 | test accuracy: 95.0400\n",
      "Epoch:  81 | train loss: 0.0121 | test accuracy: 95.0774\n",
      "Epoch:  82 | train loss: 0.0194 | test accuracy: 94.5365\n",
      "Epoch:  83 | train loss: 0.0158 | test accuracy: 94.9141\n",
      "Epoch:  84 | train loss: 0.0122 | test accuracy: 93.8731\n",
      "Epoch:  85 | train loss: 0.0093 | test accuracy: 93.8561\n",
      "Epoch:  86 | train loss: 0.0099 | test accuracy: 94.5365\n",
      "Epoch:  87 | train loss: 0.0090 | test accuracy: 94.5025\n",
      "Epoch:  88 | train loss: 0.0099 | test accuracy: 94.0602\n",
      "Epoch:  89 | train loss: 0.0125 | test accuracy: 94.3290\n",
      "Epoch:  90 | train loss: 0.0147 | test accuracy: 94.3834\n",
      "Epoch:  91 | train loss: 0.0137 | test accuracy: 94.6488\n",
      "Epoch:  92 | train loss: 0.0128 | test accuracy: 93.9820\n",
      "Epoch:  93 | train loss: 0.0105 | test accuracy: 92.6620\n",
      "Epoch:  94 | train loss: 0.0101 | test accuracy: 94.5841\n",
      "Epoch:  95 | train loss: 0.0101 | test accuracy: 94.0976\n",
      "Epoch:  96 | train loss: 0.0098 | test accuracy: 94.1555\n",
      "Epoch:  97 | train loss: 0.0080 | test accuracy: 94.6522\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  98 | train loss: 0.0115 | test accuracy: 94.0092\n",
      "Epoch:  99 | train loss: 0.0092 | test accuracy: 94.8699\n",
      "\n",
      "The train time (in seconds) is: 1223.482893705368\n",
      "\n",
      "\n",
      "Overall Accuracy =  95.45500935533255\n",
      "\n",
      "\n",
      "----------Trento Training Finished -----------\n",
      "\n",
      "The Confusion Matrix on test data\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAEJCAYAAAA5Ekh8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABgO0lEQVR4nO3dd3wURRvA8V8qECCNlkCoCQy9t4AoRVFAadJEpYOINJUmFhRERORFOghSlA4iXUAp0rvUwEBCDUnoKUACae8fuzlSLuQSLrmLmS+f+3A3t+XZy949O7Ozszbx8fEoiqIoiiXZWjoARVEURVHJSFEURbE4lYwURVEUi1PJSFEURbE4lYwURVEUi1PJSFEURbE4e0sHkN0sOR6Y7frCd6jmZekQ0iUuLtt9xNja2lg6hHSLy2aXddjaZL/POLc9Lxx0nhoDTf5DRf47I/t9SDqVjBRFUayZjfkbsIQQ84A2QJCUsrpeVgBYBZQAJNBZSvlICGEDzASaAxFAVynleX2e3sAoIB4YLaVco5fXARYCuYHVUsrP0opJNdMpiqJYM1s70x+m+xV4I1nZZ8BGKWVZ4F9gsF7eCigqpfQBhgE/gSF5jQJqAS8BE4UQefR5ZgHdgHJAAyGEb1oBqZqRoiiKNUtH86QQwhVwNfJWqJQyNOGFlHKvEKJUsmlaAwlJ41dgCTBBL/9Nn2+HEGKRECIv8DqwVUoZDoQLIQ4BjYUQp4DcUsoTekxLgLbAwefFrmpGiqIo1szG1vQHDAWuGHkMNWFNBaSU9/TngUAx/XlR4Gai6YIATyPlCfOkVv5cqmakKIpizdLXceMnYJGR8lAT5rVojxaVjBRFUaxZOjow6E1xoRlc030hRELtyItntZsgktZsigLBennic0FewG4j0ydeVqpUM52iKIo1s7Ex/fFiNgHv68+7AeuTlwshmgHnpZSPgO1ACyGEsxCiMFpi2i2lDAKeCiFqCiFsgfcSLStVqmakKIpizdLXS84kQojfgFeBAkKIQLRecROAVUKIj4BLQGd98k1ASyFEAFrX7ncBpJR3hRATgRNoTXyjpJSR+jwD0Do95AZ+l1IeSCsmG3U/o/RRF71mPnXRa9ZQF71mPrNc9PrSl6Zf9LpvXPb7kHSqZqQoimLNsmESzgiVjF5QwKmjHNi4gjs3rxH16CFOzi54la3EK293o5BXqSTTXvr3MAc2Lif4yiVsbG0p4OFFs679KF2pBgDr50zk9J7tRtdTwLM4AyYvMryOefqU3asXcmb/30Q9ekiRkt40e6cfJStUzaxNBSAkOJhJEydw6OB+4uPjqefbgBEjR+NZtGimrje5WyEhLFwwD79zZ7l0URIVFcXmrX9TtNizWmBQ0E1+mDAeKS/w4P498uTJQxlvH3r06kujl19Jsrzg4CBmzZjKsSNHePDgPkU8PGjevAW9+vQjj5NTlm6btXzGqTl65BCzpk/jvN85cuXKTaOXX+HjYSMoULBgqvN8+80Yfl+9kpat3mL8xElZGK1x+/ftZeEv87gcEEB4eBhu7u5Ur16D/gMG4e3jY+nwksqEERiskUpGLyjyUQSepctR67XW5HV2JezubfZvWM6CMYP44Pv5uBYqAsDxHRvZumg6dZq3pVG794iPiyfkmj/RT6IMy2rU7n1qNXsryfJD74Twx4zxlKuV9ALmjT//yKWTh3i16we4Ffbk6F/rWfb9SHp+Mx2PUpnzZYqMjKRvr+44ODoy7ruJ2NjAjGlT6dOrG6vXbsApC3+0b9y4xl/btlKhYiVq1KzFwQP7U8b7+DGubm58NGgIRYoU4eHDR/zx+yoGf/QBP06ZRrNXmxum69+3JzExMXw4cDCenp6cO3uWObOmc/36NSb+OCXLtsuaPmNjThw/xoB+ffBt0JBJU6YRFhrKzOlT+aBPT5at+h1HR8cU85w8cYItmzaSL18+C0RsXHhYGBUrVaJzl664ubsTHBzEgvnzeL9rJ9as20jRomleFpN1VDJ6MUKIWOCM/jIW+EhKeSiNebagnTQrAKxLGDMp2TQn0a7mfQpMkVJ2Tj5NVqrcoCmVGzRNUlbUuzyzh/Xg/JF/8G3VidA7IWz/dRavdv2Aei3eNkznXa1OkvncixTFvUjSo9/LZ44DUPXl1w1lIdcCOHtgB2/1G071xtqIHiUrVGP2iF7sXrOILsO+Nes2Jli7ZhWBgTdYv2krJUqWBKBsOUHrlq+zZtVKuvXomSnrNaZmrTrs+EdLQGt/X200GXn7lOXrseOTlDV6+RXefONVNqxba0hGJ0+e4Pq1a8yaOx/fBi8BUKdufcLCwvht8QIiIyPJkydPiuVnBmv6jI2ZO3smnp5F+d+0mdjbaz8fpcuU4b0uHVm3dg2dunRNMn10dDTfjh1D734f8PvqlZYI2agWrd6kRas3k5RVqVKVNm+24K/t2+jeo5eFIjPCzvwdGKxRZqbcCClldT2hjAbGpzE9UsqWUsoIUxYupQyydCJKjVM+ZwBs9V4wJ3f/iY2tbYpajynO7P0Lz9LlKJyoye/i8QPY2tlTybexoczWzo5Kvk24fPoYMdFPXyj+1OzetZOqVasZfiQBvLyKU71GTXbv2pEp60yNrW3Gdl17e3vy5c+Pnd2z47Do6GgA8uZNeuSeP39+4uLiIAtP9FvTZ2zMmVOnqOfbwJCIACpVroKrqys7//47xfS/LvyFuNhYulnTj3sqXFxdAbC3th//rOvabVFZ1UznDIQBCCEaA0OllG3114vQakHrhBBXgeqJZxRCOAGLgUrAacBBLy+lz1ddCNEDeFNfTxlgsZRynD7dGKArEALcB9ZLKRcJIb5HG3MpBtgupRz2IhsYFxdLfFwcoXdusXPFfPK5uhtqTDfkWQp4FufcwZ3s/WMJoXdv4VrIg3ot3qZO87apLvOGPMv9Wzd5vfvAJOV3Aq/iWtgDh1y5k5QX8ipFbEw0928FJUle5hLg70/jps1SlHt7+/DX9q1mX5+5xMXFERcXR2joA35fvYprV68yfORow/v16jegRMmSTJ3yI6O//BpPT0/OnjnD8qW/0aFTlyw9Z2Ttn7GdnS0ODg4pyh0cHQnwv5Sk7Pr1a8z/eQ7TZs0xOo81iI2NJS42lqDgIKb+bzIFCxbijZZvpj1jVlLNdC8sv96klhvwAFJ+w0wzALgrpawohGgI7EtlumpATSAOuCSEmA6URUs4VQEntGbD9fpos+2A8lLKeH1wwRey4MuBBF+5CIB7kWK89/mP5HVxAyDiwT0iQu/x97KfadK5N25FinL+0D9sXTSduNjYJE13iZ3eux1bO3sq+yZtBox6FEGevPlTTJ9QFvUw/EU3x6iwsDCcnZ1TlLu4uBAenjnrNIef/jeJ3xYvBMDJyYnvJ02mXv1n5+By5crFwsXLGPbJYDq0ffZD1O7tjowa/WWWxmrtn3HJUqU5c/pUkrKgoJvcvXMnSW0J4Lux39C02WvUqVs/K0NMl/fe6YjfuXMAlChRknkLFlOgQAELR5VMNq/xmCormunKAy2AX/X7YqTXS8ByACnlfrRB/4z5W0oZoV8ZHIA2BEVDtNrTEynlA2CbPm0YEAX8IoRoDzzOQFxJtBkwil5jZ9Bu4Oc45nFi6YQRhN4JASA+Po6nkY9p2ftjajZtRelKNWjZeyje1eqwf8NyjF3rFfP0KX6H/qFszfo4Obu8aHg52rvvdWfJitVMnTGbhi+9zOiRw9jzzy7D+0+ePGHk8I+5f/8e3373A/MX/sbHnw5n+9YtTBg/1oKRW5+u73Xj7JnTzJz2E/fv3ePK5ct88dlIbG1tkzSdbt64gXPnzvDJ8JEWjDZt4ydM4rflq/j+h8nkzZePD/r25ObNQEuHlVT6BkrNtrIkeinlQaAgUAitWSzxenOZsAhTGu2fJHoey3NqfVLKGKAusAZoCbxw+0ehYiUp5lOByg2a8v7nP/I0KpL9G5YDkEc/h1SmSq0k85SpUptHYQ94GHovxfLkiQNEPX5ItUbNU7yXO29+Ih+lPLWWUJY7X8oja3NwdnE2enSe2tG8tSji4UGlSlV4+ZUm/DD5J6pUrcaUH38wvL9u7RqOHT3C9Fk/0+qt1tSqXYduPXrzybCRrFm1AikvZFms1v4Zt3zzLfp88CG/LV5Is1ca8nabVhQuXJiGjV6mYKFCADx+/IjJk76nR68+ODo6EhEeTkR4OPFx8cTExBARHm44T2dpZby9qVq1Gi1avcnPvywi8vFjFsz/2dJhJZVDzhllSTISQpQH7IB7wDWgghDCQQjhDrzy3Jm1ZrnO+nJ8gdLpWPV+oLUQwlEI4YJ2p0KEEPkAVynlFuBjoEp6tictufPmw71IMR6EBAGkuN4oORsjRzSn92zHKb8LPtXrpXivkFdJQm+HJOkWDnA38Bp29g4peuSZi7e3T4rzAgCXLwdQxtvKrs14joqVKnPjxnXD60uXLuLs7ELx4iWSTFepinbN1pXLAVkWW3b4jD8aNIRdew+yau16/tq1l+8n/Y8b165RvYZ2sBX64AEP7t9nxtQpvNygruEREhLM9m1/8nKDuuzb84+FtyIlZ2dnipcowY3r19OeOCtlzs31rE5mJqP8QoiT+nmjlUB3KWWslPIG8AdwDu3mTSfSWM4swEMI4Yd250E/UwOQUh4F/gTOog3UdxoIB/IDG4UQp9ES1gt1XkjuYdh97gZdx62IJwDl62jdhQNOH00yXcCpozi7FyKfq3uK+QNOH6Vyg6bY2aes4JWt6UtcbAx+h599oeNiYzl3aDdlqtTC3iHltR7m0LhJU86cPkXgjRuGsps3Azn57wleadL0OXNaj7i4OE7+ewKv4sUNZQULFiQ8PIzr168lmfasfm6kcOEiWRZfdvmM8zg5UbacoEDBguzft5crVy7ToZPWubVAwULMW7A4xaNAgYLUq9+AeQsWU71mrTTWkPXu3b3LlctX8Ep2UGJxOaSZLtM6MEgpU03TUsoRwAgj5aX0p6HoveqklI+BDqksKmGaRcmW0zjRyx+klF8JIZyBQ8BJKWUwWjPdC1v1v6/wKFWWIiXKkCuPE/dCAjm85Xds7eyo36ojAD7V61GqYnW2/DKFyIhwXAt7cv7wP1w+c4zWHwxPscyz+3YQHxdH1ZdTNtEBeJYqS8X6jdn+6yziYmNwLeTJsb83EHonmHYfpXmr+Qxr36ETK5YtZcigAQwcPAQbbJg5fSpFPDzo2DHre9kn9C4776edgN63by9ubm64ublTu05d5syaTlhYGNVr1KRAgYLcu3eXdWvXcPbMab6b+KNhOa3btGPJr4sYNKAfvfv2x9PTE79z55g3dxYVKlaieo2aWbZN1vYZJ3fhvB/79+6hfMVKAJw8cZzFC3+hR68+hs8pV65c1K6bskbvmMuRAgUKGH0vqw0d/BEVKlSknBDkzZuPa9eusuTXRdjb21n8Wq4Usnnzm6n+8wOlCiGWARXRzk3NklJOf5HlJR8odf+G5fgd+ocHt4OIjYnBuUAhSlWoRsM2XXEt5GGY7snjR+xcOZ/zh/cQ+eghBYsWp0Hrd6jSMGUnw7mj+hIfH0//ifNTjSP66RN2rfyFswd2EvX4IUVKeNPsnb6Uqlg9xbTmHCg1OCgo6VA19X0ZPmo0xYqZbx2mDpRao0p5o+W1atdh/sLf2L1rJ8uWLMbf/xIPIyIoULAQ5YSgZ6++KRJMQIA/c2fN4PSpk4SGPqCIhwevNG5Kn779cXZJuwOJOQdKzYrPGDI2UGqA/yW+/WYM/v6XiH76lNJlvOnS9V3atDPeIzSxls2bUqNGrQwPB2TOgVIXzP+Z7du2EnjjOtHR0RTx8KB2nXr07tvPrJ+zWQZKfXOG6QOlbhqYbTPXfz4ZmZsatTvzqVG7s4YatTvzmSUZvTXL9GS0cUD2+5B0amw6RVEUa5bNOyaYSiUjRVEUa5YNa4QZoZKRoiiKNcvmveRMpZKRoiiKNVM1I0VRFMXSbFQyUhRFUSxNJSNFURTF4myy4WUDGaGSkaIoihVTNSNFURTF4lQyUhRFUSxOJSNFURTF8nJGLlLJSFEUxZqpmpFiVHYbdBTg5NVQS4eQLtVLuVo6hBwhuw08GhObvQZ2BcD+xT/jxLdz/y9TyUhRFMWKqZqRoiiKYnk5IxepZKQoimLNVM1IURRFsTiVjBRFURSLU8MBKYqiKBanakaKoiiKxWVWMhJCfAz00V8eAj4ASgLLAXdgH9BbShkrhMgNLAGqAyFARyllsL6cL4AeQDTQT0q5NyPx5IwO7IqiKNmUjY2NyQ9TCSE8gEFALaAyUBR4A5gIfCel9AHsgE76LH2AQL18IfC1vpwqQBugAtAOmJ3R7VTJSFEUxYplRjJC6zBuD+QGHPT/bwGNgA36NL8CbfXnrYHf9OfLgVb687eAFVLKaCnlBSBMCFEhI9upmukURVGsWTpyjBDCFXA18laolDI04YWUMlgIMRm4ATwFVgBXgAdSyjh9skCgmP68KHBTn/exEMJOCOGgl+9LtJ6Eec6bHrVGJSMLCgkOZtLECRw6uJ/4+Hjq+TZgxMjReBYtmmnrPLpvB4f+2c6VSxcID3tAgUJFqNWgMW916kEep7wA3LkVxLBe7YzOP2vl3+TNl9/wevXiWVy5dJ6r/hd4FBFOn6Ff0ui1N58bwyW/04wf0Y/4+HgWbNiPnZ15dsO/tm3lzy2b8Tt3lvv37+Hh6UmzV5vTp98H5M2bzzCdv/8lZk6fyplTJ4l4+JCiRYvRtl173n2/O/b2lv9KWGK/eFHWEvPf27ey7c/N+Pmd48H9e3h4eNLk1dfo1efZPnDe7ywzp/2Ev/9FwkJDyZ/fmfIVKtLngw+pWq1GimWeOXWSubNncObMKWKiYyjm5UXvvv15vUWrFNNmhnQOBzQUGGOk/Bv0pjUAIYQbWq2mFPAQWIfWTGcxlv/m5VCRkZH07dUdB0dHxn03ERsbmDFtKn16dWP12g04OTllynr/XLsU90IedOj+Ie4FC3MtQLJu2XwunD7OFz/OT7Ljv9mpOzXqNUoyf548SeP6e+NqSpQpS/W6L7F/x5Y01x8TE8OiGd/j7OpO2IN75tko3eJFC/D09GTQ0I8pUsSDC+f9mDNrBkePHObXpSuwtbXl9u1b9OnxPoULF2H4qNG4urpx5PAhpkyexP379/n40+FmjSm9LLVfvAhrivm3xQvx8PTko8EfU6RIEeT588ydM4NjR46w8Lfl2NraEhEeQfESJXirTTsKFirE/fv3WfbbIvr27MYvi5dSuUpVw/L27tnNsKGDeKNlK8Z//yMODg5cDvDnydMnWbZN6Wx++wlYZKQ8NNnrV4EAKeU9ACHEBqAS4CaEsNVrR17otSEgCK3GEyKEyAPESimjhRAJ5QkSz5MuKhlZyNo1qwgMvMH6TVspUbIkAGXLCVq3fJ01q1bSrUfPTFnv0DGTcXZxM7wuX6UmefO7MO9/33DhzAkqVqtteK+QRzF8yld57vJmr9qBra0tt4JumJSM/vx9CfHE8/Jrb7Fx1aIMb4cx02bOwd3d3fC6dp26uLi48sXokRw9cph69X3Zs3s3Dx48YNGS5ZQqVRqAevV9uXHjOps2rLd4MrLUfvEirCnmn6bPxi3RPlCrdl2cXVwY88Uojh09Qt169alb35e69X2TzNeg4Us0e9mXLZvWG5LRo0cP+ebL0XTs/A7DRo42TFuvfoOs2ZgE6chFelNcqAmTXgd89cQSBTQBNgP70c4PrQO6Aev16TcB7wPHga7AlkTlvwghpgHegKuUMt1NdGBFHRiEEFOEEAMTvT4khPgx0euVQoi3X2D5Q4UQX79gmGaze9dOqlatZvjyAnh5Fad6jZrs3rUj09abOBElKFNWO9/44N7tdC8vPU0It4ID2bByAd0GjMAuE5rDEieiBJUqa8n09u1bAERHRwOQL1GzHUD+/PmJi4/D0iy1X7wIa4rZ7Tn7wB19HzAmTx4nHB0dkzQZ/719Gw8e3Oe97pY9AMiMDgxSysPAH8AJ4AwQDiwFRgBfCCECgHhglT7LPKCEEMIf6IXe5CelPI2WkC6gJbABGd1Oa6oZHUTrIjhDCJELrYdHrUTv+6K1h/4nBPj707hpsxTl3t4+/LV9a5bGcuHsvwAULV46SfmaRbNYPGMiuXLnRlSuSYfu/SleyifD61k8YyJ1XmpG+co1OH/q2AvFbKpjx44AUKaMNwDNX3+DubNnMGH8OD4eNhxXVzcOHzrIpo0b6P/hR1kS0/NY035hKmuP+fixowCULlMmSXlcXByxsbHcvXuHRb/MA6Dd2x0N75/89zguLi74X7rI4AH9uHrlMgULFqJt+w707vchdnZ2WRJ/Zl1nJKUcQ8rzS/5AbSPTRgLtU1nON2jnpF6INSWjA2h93EFLQvuBGnqPjcJALNBECPEZWsX1d/3DRAjRFTBW3gcYCdwDJFpvEYQQg4H+QAzgJ6XskiVbmEhYWBjOzs4pyl1cXAgPD8+yOO7fvc3aJT9TqXpdSus1JAcHR5q0aEflGvXI7+JKcOA1Nq5axLfD+jLmfwsoWqJ0GktNaf/OP7nqf4Hvh6809yak6tatW8yaMY36vg0MR8cFChbk16UrGTJoAK1efxXQvuz9BwykZ+++WRZbaqxlv0gPa4759q1bzJk5jXr1G1CxUtIm51HDhrLj7+0AuLsXYOrMnynj/exg687t20RFRfH5qGH06fchFSpW4sihg8z/eTYRERF8OuKzLNkGNQJDFpNSBurdBT3QakGH0JoRa6BdFXwFGA/UQatS/iOE+Aetepha+edoiS1SX94VfXWjgNJSyid6V8gcKSryMVPHDcfOzo4+H39hKHd1L0iPgaMMr0XlGlSp5cvoD7uwYeUi+g9P30HQw4gwls+fSofuH+LsmrIZJTM8fvSIoYM+xN7OjrHfTjCU379/n0+GDiRPnjxMnjINF1dXjhw+xLy5c3B0dKRXn35ZEp+S+R4/fsQnQwZgZ2/HmLHfpXh/8CfD6d6rL7dCglm1YhlDB/Vn9s8LDEkrLj6OJ0+eMGDQUN7rpjXV1a5Tj9DQUFatWEa/DweSP3/+FMs1NzU2nWUcREtEvsBwtGTki5aM1gPVpJR3AYQQK9Au0MoP7DJS7qyX39fLf+fZObLTwFIhxDq0ds4s5+zibPSoMbWjTHN7+iSKKd98yp2QID6bOBv3gkWeO32BQkUoV6kaVy75pXtdv/86F1f3AtRt1IxHDyMAiH76FIDIR49wcHQkV+486d+IVERFRTHoo/4E3ghkweLfKOLhYXhv0YJ5BN28yda/duHs4gJAnbr1iIuLY+b0abR7uwNublmTMI2x9H6REdYYc1RUFEMHfcjNwEB+XvBrkn0ggZdXcby8ilOpchUavdKYTu3fYtb0qcyYMx8AVxdXIGWHhfoNGvL76hVcDrhEteo1M31bVM3IMg4CDdBqLVeEEHbAOKA0EJaB5aV2n+JWaAnrLWC0EKKqlDImIwFnlLe3DwH+l1KUX74ckKSpIDPExMQw47vPuOp/geHfTkvXeaCMfDGCblzhxhV/PurSPMV7H73TnJr1X2bIl5PSvVxjoqOjGfbxYPzOnWXu/IWULSeSvH/p4kWKlyhpSEQJKlepQkxMNDeuX7doMrLkfpFR1hZzdHQ0Iz4dwvlzZ5k1d0GKfcAYBwdHypYVaIMIaMr4lH3uPLY2WdP/K6ckI6vpTac7ALyNdlUwUkp/tDGPygELgMZCCHchhCPQGdgDHHlOeRMhhJs+yF87ACGELVBcSrkb7XxSfiBp16os0LhJU86cPkXgjRuGsps3Azn57wleadI009YbFxfHnElf4Xf6OIO/+CHNrtsJ7t0O4eK5U5QuVzHd6+za92NGTZiV5PFSM+2CwRHjZ/D2+/3TvUxj4uLiGD1yGEcOH+Kn6bOoWq16imkKFizEjevXCA9Lemxz5vRpAAoXfn4NMbNZar94EdYUc1xcHF98NpxjRw4x+aeZVDGyDxgTGRmJn99ZvIoXN5Q1bqJ1yjh4YF+SaQ/u30uuXLnwLvv8ZGUuNjamP7Iza6sZ/Qt4Aj8nKgtEG8rihj467D8866iwCwyjxhor/w4tKd1Da5oDbfC/JUIIF336mYmHycgq7Tt0YsWypQwZNICBg4dggw0zp0+liIcHHTt2zrT1/jp7Ekf37eCtzj3JlTs3/hfOGN5zL1gY94JFWD5/KnFxcfhUqIKzsyvBN6+zadVibG1tad05aTfXC2dOEBH2gNAH9wG44n+e3Hm0Jrc6L2lf5pLe5VLEceHMCQDKV6lhthEYvvv2G7Zv20rffv3JkycPp0+dNLxXpIiH9tl27sKWzRvp37cX3Xv1xtXVjaNHDrN44QKavvoaHp6eZokloyy1X7wIa4r5+/Fj+Xv7Vnr31faBM4n2gcL6PjB+7Fc4u7hQsWJlXN3cCA4KYtWKpdy9c4dx438wTO9TthxvtWnHnFnTiY+Lp3zFihw+dJB1a9fQp9+HOOkjlmS2nFIzsomPT60lSzEmKibVpr90Cw4KSjqESn1fho8aTbFiXuZaBQAnr4Yann/asy13bwcbna5t1z60e7cve7ZvYOeWtdwKCuRJ1GPy5XehQrXatO3aB0+vkknmmTDqQ0NiSW7x5sOpxvTH0nmsWzbf6HBA1Uu5mrZhybR4rSlBQcYv/u4/YCAffjQIgNOnTjJ39kwunD/Pw0facEAtWraiW49e5M6dO0PrNqes2i/MKStijolN+6v35htNCQ4KMvpev/4f8cGAQaz/43fWrV3NtatXiIyMpFDhIlSuUpWevfulaNKLjn7KvDmz2LRhHffu3aNosaJ07PwuXd/rZlLM+XK9eCYRI7eZ/JsjJ76ebTOXSkbpZM5klFUSJ6PsIKPJSPlvMyUZWRtzJKPyo0xPRhe+z77JyNqa6RRFUZREbFXXbkVRFMXScsgpI5WMFEVRrFlO6cCgkpGiKIoVyyG5SCUjRVEUa5bOm+tlWyoZKYqiWDFVM1IURVEsTp0zUhRFUSwuh+QilYwURVGsmaoZKYqiKBaXQ3KRSkaKoijWTI3AoCiKolicaqZT/jOy28CjT6LjLB1CuuVyyBnXgliSvV3O+FFOLofkIpWMFEVRrJmqGSmKoigWl0NykUpGiqIo1kx1YFAURVEsTjXTKYqiKBankpGiKIpicTkkF6lkpCiKYs1UzUhRFEWxuBySi1QyUhRFsWaqN52iKIpicbY5pGqkkpEFhQQHM2niBA4d3E98fDz1fBswYuRoPIsWtXRoqbKWmA8e2MevC+dz5XIAEeFhuLm5U6VaDfr2/4gy3j5Jpt2/9x8WL5yHPH8eW1sbipcsxaChw6hTtz4A5/3OMXvGTwRcukhYWCj58jtTvnwFevX7kKrVamTpdoH1fMbpYc0x3woJYcEv8/A7d5aL8gJRUVFs2b6DYsW8kkw37af/ce7cWc6fO0dYWChjv51Am3btLRT1MzkkF2ETHx9v6RiylagYzPKBRUZG0ql9GxwcHRk4eCg2NjBj2lSioiJZvXYDTk5O5liNWWVVzKaMTbftz83I835UqlIVNzc3QkKCWbxgHrdvhbBs9Xo8ixYDYO2alUz6/ls6du5Kw5deJi4+novyPGW8fWj0chMAjhw+yD87/6ZajZoULFiI+/fvs3zJYs77nWPewiVUqlI1zXjMNTad2i/M7+iRw4wY9jEVK1YiNjaOgwf2GU1GvnVqIMpXwMurOBs3rDNLMsptzwunktdnHTb5N2fbgHrZNnWpmpGFrF2zisDAG6zftJUSJUsCULacoHXL11mzaiXdevS0cIQpWVPMr7doxestWiUpq1S5Kh3btmTn39t5t1tPgm7eZMqkCQweOox33utumM63wUtJ5qtbz5e69XyTlPk2bETzxr5s2bzBpGRkLtb0GZvK2mOuVbsOu/YcAGDtmtUcPLDP6HT7Dx/H1taW69eusXHDuiyM8PlyyCkj601GQohY4AzgAFwG3pdShr7gMl2Bk1LKUi8a34vavWsnVatWM3x5Aby8ilO9Rk1279ph8S+wMdYes4uLKwB2dnYAbFz/OzY2trTv2CXdy8qTJw+Ojo6GZWUVa/+MjbH2mG1tTau1mjpdVsusDgxCCHfgF6ACEA90BCKB5YA7sA/oLaWMFULkBpYA1YEQoKOUMlhfzhdADyAa6Cel3JuReKzz09dESCmrSykrAaHARxaOx6wC/P3xLlsuRbm3tw+XA/wtEFHarDHm2NhYoqOfcv3aVSZ8O4YCBQvSXK8xnfz3BKVKl+avrVto92ZzfGtVpv1br7N6xVKjy4qLiyMmOpqQ4CAmTRgHQNv2HbNsW8A6P+O0ZMeYsxObdPxLp2nAOilleaAmcA2YCHwnpfQB7IBO+rR9gEC9fCHwNYAQogrQBi2htQNmZ3Q7rbZmlMx+oBqAEKI6MBfIA5wH+kgpI4QQfYG+QC7AD+gupXwqhPABlgJOwJaEBQohKqF9qI5oSfltKeWlrNqgsLAwnJ2dU5S7uLgQHh6eVWGkizXG3PP9zlzwOwdA8eIlmPXzItzdCwBw985t7t65zbSfJjFg4FCKFS/Bjr+2Mun7b4mNjaXLu92SLGv0iI/Z+fd2ANzdCzBlxtwUnSEymzV+xmnJjjFnJ+mpGOmtP65G3gpN3LIkhHAB6ksp3wOQUkYKIWyARjxLQL+iJaHlQGvgM718OfCN/vwtYIWUMhq4IIQIE0JUkFKeNz1qjTXXjAAQQtgBrwEb9aJfgU+klFWBm8Bwvfx3KWVdKWU1IBh4Ry//CfiflLIKEJZo0f2BqVLK6kBtIDAzt0PJHN98O5EFv61g3IQfyZsvH4P69ybo5k1Aq+k8evSIz774mrZvd6JO3fqM+vxrfBs2YtGCeSTvvDNo6DAWLVnFxMlTKeNTlk8GfYjfubOW2CxFMbCxsTH5AQwFrhh5DE222NLAHSHEb0KIf4UQM4HCwAMpZUIPokCgmP68KNrvLVLKx4CdEMIhcbmRedLFmpNRfiHESeAW4AFs17N5Pinlfn2axWiZHKCaEGKfEOIM0AGoqJfXA1brz5ckWv5BYLQQYhRQUkoZmXmbkpKzi7PRo8bUjjKtgTXGXLqMN5WrVOP1Fq2YOXchjx8/ZvHCeQC4uLoCULd+wyTz1KvfgPv37nL3zp0k5cW8ilOxchWaNGvO1JlzcXN3Z87MqVmyHQms8TNOS3aMOTuxsTH9gXbwXdrI46dki7VHOwj/Ca2Jzh7tvI/FWHMyitBrLSUBG2BAGtMvQDt5VgX4Aa25LlVSymVoVc9HwCYhRNMXjjgdvL19CPBP2Sp4+XJAljcNmcraY87v7IxXiRIE3rgGkGZMNs9p/3BwcKRsOUHgjetmjTEt1v4ZG5MdY85ObG1sTH5IKUOllFeNPEKTLTYQuCalPC6ljAfWAVUBNyFEQl7w4lmtJwi9xiOEyAPE6k1zhnIj86RvOzMyU1aSUj4CBgOfoiWOCCFEQj/c94E9+vN8wG0hhCPPmugADgNv68+7JhQKIcoAl6WU04E/0P4QWaZxk6acOX2KwBs3DGU3bwZy8t8TvNIkS/Oiyaw95nv37nLtyhW8vEoA0LjJqwAcStaV9+CBfRQu4kHBgoVSXVZUZCTnz52lmFfxzAvYCGv/jI3JjjFnJ7a2NiY/TCWlDAGC9HPqAI3RzrXvRztIB+gGrNefb0L7vQXtd3RLovIuQggHIUR5wDUj54sgm3RgkFIe05vfOgHdgTlCCCfgAtBbn+wr4Chas96/iWYfCiwTQnxJog4M+rLeF0I81efpShZq36ETK5YtZcigAQwcPAQbbJg5fSpFPDzo2LFzVoZiMmuKefjHAylfoSI+ZQV58+Xj+rWrLF+yGDs7O7p26wFAw0avUKtOPSZ8O4bQ0AcUK1acHX9t5fDB/Xz1zXeGZU0YNwZnFxcqVKyMq6srwcFBrF6xjLt37/D1+IlZul3W9BmbKjvE/Ne2rQD4+WnnAPfv3YObmztu7u7UrlMXgGNHj/Dg/n3u3r0LwLlzZw0X7L72+hsWiFqTiSMwDAFW6AfwF4CegKdeNhktMa3Sp50HLBVC+KP9XnYEkFKeFkJs0uePBj7IaDBqBIZ0MtcIDADBQUFJh1Cp78vwUaNTXBluTbIiZlNGYFi8cB47tm8l8MYNomOiKVLEg1q169K9Vz+KFnvWavDw4UNmTfsfO//eTnh4OKVKl6Zbz7680fJNwzQb1v3O+rVruHbtClGRkRQqXIRKlavSo3c/fIx0WTbGXCMwgNovMkO1SsJoee06dfll0W8A9O7xPseOHjE63alzMkPrNccIDJ0X/2vyb87K7jWy7SWyqSYjIcQNMPrDawPESylLZGZg1sqcyUgxzpRkZG3MmYyU/w5zJKMu6UhGK7JxMnpeM91Lz3lPURRFyQI5/uZ6UsprWRmIoiiKkpIam04nhCiJ1lW6CpA7oVxKWSYT41IURVHIOTfXM6WhewFaH/R4tC7SfwOLMi8kRVEUJUE6R2DItkxJRm5SyuVAnJTyX7Sue2+mMY+iKIpiBrY2pj+yM1OuM3qq/x+qX9QUDKR+taCiKIpiNtm9xmMqU5LR7/p9L74HDqHVpsZnalSKoigKwIv3Dc8m0kxGUspJ+tPNQojCQG4ppRoXXlEUJQvYZff2NxOZ0psuxeBSQgiklDszJyRFURQlgWqme+bLRM9zo9129gSgkpGiKEomyyG5yKRmuiaJX+t3SB2WaREpiqIoBrY5JBule0AtKeU5oEYmxKIoiqIkk86b62Vb6T1nZAfURxsqXFEURclk6pzRM4nPGcUA/mj3AlKUTJEdR8B26zjf0iGk24PVfSwdgmICO5WMDLpKKYMTFwghPDIpHkVRFCWRHNKz26RzRpuNlG0xUqYoiqKYWY4fDkgI4Qq4A45CiNI8uxDYBXDK/NAURVEUdc4IugNDgaIkvaYoDPhfJsakKIqi6LJ7jcdUz7u53lRgqhBipJRyYhbGpCiKouhySMXIpHNGJ4UQLgkvhBCuQojmmRiToiiKorO3sTH5kZ2Zkoy+l1KGJXodhjaCt6IoipLJcspFr6YkoySbKKWMx7Qu4YqiKMoLsrWxMfmRnZmSjO4JIV5PeCGEaAHcy7yQFEVRlAQ5pWZkSg1nKPCHECJKf+0A9My0iBRFURSDHN+bLoGU8ox+u/GKQGu0sem2AK6ZG9p/362QEBb8Mg+/c2e5KC8QFRXFlu07KFbMy9KhpSokOJhJEydw6OB+4uPjqefbgBEjR+NZtKilQ0uVJWL2LV+E0Z1rULV0AfI42uEfFM6cP/34dcdFwzQlC+djQvd6NKlWFAc7W45dusPoxUc4EXA3xfKKujvxVddavF6zOG75chF8/zGr9wXw1ZJjhmnyONrxaftqdGrkjVeBvNyLiOKfM8GMXX6c63ceZtq2gtovMlNOubmeTXx8/HMnEELUBnoAHYG8QF9go5Qyc/duKxUVw/M/sHQ4euQwI4Z9TMWKlYiNjePggX1WnYwiIyPp1L4NDo6ODBw8FBsbmDFtKlFRkaxeuwEnJ+u7FjqrYk48Nl3lku7smdiaIxdvM2PTWR4/iaWdbyn6vF6BwXP2M2/bedzz5+LolPZEREbz7YoTPH4Sw+DWlanpU5BGIzYgA0MNyytRKB+7JrzF1dsRzNp0jlthkZQsnB9vD2fGLj9umG7Rx415q14pvl1xnOP+dyleKB9fdqlJbFw8dT9ey6OomCQxm2tsOrVfpC63/YvfNXz8Dn+Tf3M+b+aTbTPX80ZgGI524Ws8sBjtpnoHpJTLMzMgIUQXYBlQQkoZmMFlXAWqSylDzRia2dWqXYddew4AsHbNag4e2GfhiJ5v7ZpVBAbeYP2mrZQoWRKAsuUErVu+zppVK+nWw/paby0Rc8eXymBna8Pb3203JICdp25SpZQ77zbxYd628/R9vQKFXfPw6hebuBISAcDuM0H4zenMl11q8t6Pz64zn96/IUH3H/H6l5uJidV+l/adC0myzjyOdrzdsAz/++M0U9adMZTfDo1kw1dv4Fu+CH+fvGn2bQW1X2Q2mxfPZ9nC8zowTABCgHeklD/qg6WarVbwHF2AA0DnLFiXRdnaZq/RqXfv2knVqtUMX14AL6/iVK9Rk927dlgwstRZImZHe1uiY+OIfBqbpDzs0VNDj6e6ojD+weGGRATw+EkMB/xCaFG7hKFpprRHfprXLM6szX6GRGSMna0t9na2REQ+TbbOJwDYZmJTj9ovMleOH5sOKIVWM1orhIhAqx3ZZWYw+sW1NYG3gHnAZCFED6ANUADwAOZKKScLIUqhDeJ6FqgKHAN6SSmjky3zfWAw4AjskFJ+IoTIC6wCvPRtGielXJmZ2/ZfEODvT+OmzVKUe3v78Nf2rRaIKG2WiPm3XZfo+0YFJvfx5YfVJ3n8JIb2DUvTpGoxek/dDUBsXDxPo2NTzPskOhanXPaU8XDmUlAYvuWLABD1NIZNY1rwUiUPHj+JYcvR64xYeIj7EVqyeRgVzdJdlxjQqhJHLt7huP8dShTKx3fd63Hqyj12nQ7KlG0FtV9ktuyeZEz1vOGAAoHxwHghREO080bOQog/geVSyl8zIZ52wCYp5SkhhJMQooxeXheojHZTv2NCiI3AU7ROFb2klIeFEMvQkqeh8V4IUUFfpq+UMkYIsUQI0QrIDQRJKVvp0xlGmFBSFxYWhrOzc4pyFxcXwsPDLRBR2iwRs9/1B7z+5WZWjnyN/i0qAvA0OpZBc/axet9lAC7dDKNZtWK4589lSCg2NlC7bCEA3PPnArSOCwBzBr7Msn/8mbT2pHau6P06lC/uSqMR60k47dtvxh4m9/Fl27hWhliOyNu8+fWfRMfEZcq2gtovMltOGSjVpHYiKeV+KWVftEFTlwLvZVI8XdBqLAAredZUt01K+UDvNLEFaKCXX5FSHtafrwBeSra8ZkA9tAR2Ei2p+QBngNeEED8IIRolG2FCUV6It6czy0e8it+NB7Qfv40WX21h/rYLTO//El1e9gZg3rbz2NrA/MGvUNojPx5uefhfH19KFckPQFyclmESfoj2nA3m458P8M+ZYBb8JRk69wC1fArxWo1nnV2+7lqLd172YdTCw7z6+SZ6/rQb9/y5WPfl6zjlUtepZ1d2tqY/srN07aFSykhgif4wKyFEQeAVoIIQImGUh7vATyQ9VxWf6HVq5QlsgQVSyi+TlSOEqAm0BMYKIXZJKceaYzv+y5xdnI0eNaZ2lGkNLBHz2PdqEx0bR/vx2wzneXafCcI9fy4m9fZl5d4Art6KoOeU3Uzp1wC/2dox14mAu0zfeJaP21Yl5MFjAEOtaeeppJ0P/j6p9e2pVroA208EUqG4K8Pfrk7/GXtYnKj7+NGLtzk7qxM9XxPM3HQuU7ZX7ReZK7uPrGAqazpc6oCWOD5KKBBCnEZrUmuu318pGi2BzNUnKS2EqCOlPIpWi9qVbJk70M55TZVS3hVCFEY7R2QDPJBSLhFC3AX6Z+aG/Vd4e/sQ4H8pRfnlywGU8faxQERps0TMlUq4c+bq/RQdDo7536HLKz4UdsnDrdBI1h26yoYj1yhb1IWnMbFcCYlg6gcNuXHnITfuPgLg/I0Hz11XwqUZlUu6A3DcP+k1SgHB4Tx4+ATh5WqmrUtJ7ReZKzPPGQkhbIHDwE0pZVshhDewHO1edvuA3lLKWCFEbrRKSHW0jm0dE+4ALoT4Au00TjTQT0q5NyOxWFPFrgvwR7KyP4BRaJ0T1gMngflSyoRDv/PAp0KI8/rrJOexpJTngLHADj2xbUL7kKsAh/Wmu7HAt+bemP+ixk2acub0KQJv3DCU3bwZyMl/T/BKk6YWjCx1loj5VmgkVUu542Cf9OtVp2whIp/EcP/hE0NZXFw8MjCUKyEReLo50aFhaX7eet7w/mF5m+D7j3m1RtJrz5rrr49d0pJPyINI4Nk5pwQ+RZ1xy5eLoHuPzLeByaj9InNl8nBAHwIBiV5PBL6TUvqgHbh30sv7AIF6+ULgawAhRBW0DmYJ5+dnZygKTLjo1dL03nTVpZRDk5WXAtZJKatnZTzmvOgV4K9tWs+dw4cPsnrlCj7/cgxubu64ubtTu05dc67qhT1+/JhO7duQK3duBg4egg02zJw+lUePH7Fm7Qac8ua1dIgpZFXMiS96bedbimUjXuWvfwP5easfkU9iebNuCfq3rMS0DWcYufAw9nY2fNetLnvPhRAe+ZSKxd0Y/nY1LodE0GLMliQdDt5tUpb5g19h3rbzrD94FW9PZ75+tzanr9zjja+2AFrX7QM/tqVU4fxMXPOv4aLXUR2qU9AlN3WHrjXUthKY66JXtV+kzhwXvc7cf9Xk35yPGpYyeX1CiCJo13SOR+tx3A6t1uMppYwTQrwG9JFSdhZCbAc+k1IeF0I4ARellF5CiNHAEynlZH2Z+/V5zhtd6XNYUzNdjjTskyFJXo8f9w0AtevU5ZdFv1kipFQ5OTkxb8FiJk2cwOejRmhDqNT3Zfio0Vb5gwOWifmPg1dpM24rn7arxqwBjcjtaMflkAiGzN3P/O0XAIiPB++iLnR62QfXvI7cvPeIxTsu8sOakyl6vi3ddYm4uHg+bV+Vbk3LcT/iCSv+8efLJUcN08TFxdNyzBZGvF2dXq+V58su2nBAhy7cZuzy4ykSkTmp/SJzpafGo5/OcDXyVqiRQQB+AL4AcumvC6CdvkjYAQOBYvrzosBNACnlYyGEnRDCQS9PfLV+wjzpTkZWXzOyNuauGSn/DYlrRtmFuWpGSurMUTP6+dA1k39zJndv/g0wxshb30gpv054IYR4Ge2ymB5CiMZoA2L3AfZJKcvr01QA5kkpXxJCnAVelVKG6O8FAyWAKfo8K/Tylfo8f6d3O1XNSFEUxYql81zQT8AiI+WhyV43QLu85SpaJzFnYDLgJoSw1WtHXui1ISAIrcYTIoTIA8RKKaOFEAnlCRLPky4qGSmKolix9HTt1pviQk2Y7nv0O3Yn1IyklN2FEGvR7s6wDuiG1nEMtM5f7wPHga5o13smlP8ihJgGeAOuGTlfBNbVm05RFEVJJotvrjcC+EIIEYB23WbCIATzgBJCCH+gF3pvOillQi/lC2gJbEBGV6zOGaWTOmekGKPOGSnGmOOc0aKj103+zelRp0S2vUJWNdMpiqJYMTUCg6IoimJxKhkpiqIoFpczUpFKRoqiKFYth1SMVDJSFEWxZjnlfkYqGSmKolixnHL9jUpGiqIoVkx1YFAURVEsTjXTKUbFZcOLhHPKkZUlZccLSN3qDLR0COny4OgMS4dgEaqZTlEURbE4VTNSFEVRLC5npCKVjBRFUaxaDqkYqWSkKIpizexySDZSyUhRFMWK2eSQhjqVjBRFUaxYDqkYqWSkKIpizWxVzUhRFEWxNFUzUhRFUSwup1y0rpKRoiiKFbPNGblIJaPMcCskhIUL5uF37iyXpCQqKorN2/6maDGvJNM9efKEWdOnsmXTRiIiwilXvjxDPh5Grdp1DNNcu3qFlcuXcezIYQIDA8mbNy8VK1dmwMAhiPLls3rTCAkOZtLECRw6uJ/4+Hjq+TZgxMjReBYtmuWxmCq7xXzk8CFmTp/Keb9z5MqVm0avvMKnw0ZSoGBBs63D3hZ2L/6UKmWL4ZTHEdHyK64H308yTS5He8YMeJMuLevgmj8Ppy/e5POp69h/IsAwjU+JwvTv3IiXa5ejtFdBIh5FcdzvOmNnbeLMxZsp1tuzXQOGvN+UUsUKcC3oPtOX7mL+mn2pxlmqWAGOr/4cpzyOPIkBcw3G9de2rfy5ZTN+585y//49PDw9afZqc/r0+4C8efOZaS3mkVN60+WUYY+y1I3r1/hr61acnV2oUbNWqtN989XnrP19NR8OHMTUmXMoVLAwH33QB3nhvGGagwf2c+zIYd5s05apM2fx2Rdf8eDBfbq/2xm/c2ezYnMMIiMj6durO1euXGbcdxMZ//0PXL92jT69uvH48eMsjcVU2S3mE8eP8WG/3uR3dmbyT9MZ8dloThw7Rt/ePXj69KlZ1mFrA3Y28CD8Mfv/DUh1ujlj3qVn+waMm72Z9kPmEHInjI0zP6JquWKGaV71Lc/LtcuxdNNh3h4yh6ETVlLILR//LP6UGhWKJ1lez3YNmPFFF9btOEXrj2ax9q9/mfpZJ/p2fCnVGKZ+1pmwh5EvvtHJLF60ADs7WwYN/ZhZc+fTqfM7rF65nA/69CIuLs7s63sRNjamP7Izm/hsOPCnOQkhTgJtpZRXTZn+cXTaH1hcXBy2tlqeX7tmNeO+/jJFzUheuECXDm35etx42rR7G4CYmBg6tH2TkqVKM3XGbAAePHiAq6trkvGpIiIiaPV6M15+pQnfTpiYZszmanNe+ttifvzhe9Zv2kqJkiUBCAy8QeuWrzP0k+F069HTLOsxp+wWc7/ePQi6eZN1m/7E3l5ruDh39gxdO3dg9Bdf0fmdd822Lrc6A+nRzpfZX72bomZUpVwxjqz8jH5jlvDbhkMA2NnZcmLN51y8dpuOQ+cCUMA1L/dCHyVZrnO+3FzYPJYte87Q58vfDPNe3j6e7fv96PvVb4Zp54x5l1avVKF089HExCRNAp3fqM3EYe35ccF2Jg3vYNaa0f3793F3d09StnH9Or4YPZKff1lEvfq+ZllPbvsXr9bslvdN3uzGwj3bpqRsUTMSQthZOob0SEhEz/PP7p3Y2zvQ/I2WhjJ7e3tef6MlB/fvMxwFu7m5pRgoMX/+/JQsWYo7t2+ZN/A07N61k6pVqxl+1AG8vIpTvUZNdu/akaWxmCq7xXz61CnqN2hgSEQAlSpXwdXVlZ07/s6yOFq9UoWn0TGs2X7cUBYbG8fqbcd5zbc8jg5afMkTEUD4wyj8r92maCFXQ1n9qqUp7J6f5ZuPJpl22eYjFHTLR4Pq3knKXfPn4ftP2/PZlD8IjTB/zSh5IgLtcwa4ncXfq7TY2pj+yM6s4pyREGIM0BUIAe4D64GvgZVAc2CMECIf8BnauIG/SynH6POGSild9ec9gOpSyqFCiEVAOFAHKAwMlFL+KYRwAhYDlYDTgEPWbGVSAf7+FPMqRp48eZKUe/uUJTo6mhvXr+HtU9bovGFhofj7X6JN23ZZEapBgL8/jZs2S1Hu7e3DX9u3ZmkspspuMdvZ2eLgkHKXdHB0xP/SpSyLo6K3J1dv3iMyKjpJ+fmAYHI5OuBdvCDnL4cYndfN2YmKPp78tv6QoayCtycAfgFBKZYHUKGMJ3uOPdu+8UPbcvHqLZZvPsp7b9Uzyzal5dixIwCUKeOdxpRZK6f0prN4zUgIUQdoDVQF2qIljwRBUsoawDFgPNAEqAk0F0I0NWHxhYAGwDvAOL1sAHBXSlkRmA5UNMNmpFt4WCjOzi4pyp1dtLKwsLBU55343bcQH0/X97tnWnzGhIWF4ezsnKLcxcWF8PDwLI3FVNkt5pKlSnP61KkkZUFBN7l75w5hYaFZFoebsxOh4SnPqd3Xy9xd8qY67/9GdsQGG6Yv25VkeaCdpzK2PDcXJ0NZwxrevPtmXYZMWJnxDUinW7duMWvGNOr7NjDUkKyFTToe2ZnFkxHQEFgnpXwipXwAbEv03mr9/zrALinlXSnlU2AF0MiEZa+TUsYDx4FSetlLwHIAKeV+4MqLb0LW+WXeXP7cvImRo7+kRImSac+gZCvvvt+Ns2dOM2PqFO7du8eVywF8PmoEtra2JjX/WtqwXs3p0rIOH09cxeUbd9M9v4O9HdO/6ML0pbu4kErNy9weP3rE0EEfYm9nx9hvJ2TJOtPD1sbG5Ed2Zu17tyndnRKf3MuV7L0nAFLKWJI2SVq814azswvh4SlrP+F6jcjFJWWtafXKFcyYOoWPBg2lbfu3Mz3G5JxdnI3WJlKrfViD7BZzqzdb0/eDD/l18UKavtyAdq1bUbhwEV5q9DIFCxXKsjhCwx/j6uyUotxdL7sflvJcUZ8OLzFuUGvGzNjIr4ma6ABCI/QaULJlJizvQZj2/qB3m+Ca34lZy//BJV8eXPLlwSm344tvUCqioqIY9FF/Am8EMvvnXyji4ZFp68qonFIzsoZzRvuBWUKIiUAetHNEe5NNcwSYIoRwBx4CnYHP9ffuCSG8gatozX1pNazv0+ffK4TwBUqbYyPSq4yPDzt3/E1kZGSS80aXA/xxcHCgeLJaz6YN65nw7Te8370nfT7on9XhAtp5lgD/lB/v5csBlPH2sUBEacuOMQ8cPJReffpxM/AG7u4FKFCwIG3favHcywTMze9yMK2bViNPbock543Kl/HkydNoApLVet5pVYepn3Xip1938MMv25IvDr+Ec0PenoTcfXZwUL6Mdi7p/OVg/bUHnoVcuLx9fIpl5LKHuHh4Gvvi2wcQHR3NsI8H43fuLHPnL6RsOWGeBZtbds8yJrJ4zUhKeRT4EziL1nHhNFrHg8TTBANfAP8AJ4C/pJQJDdKfAdvREth1E1Y5C/AQQvgBgwE/M2xGur3SuAkxMdFJTqLHxMSwfeuf+DZoiKPjs6PBnX//xddfjqbd2x34ZPhIS4QLQOMmTTlz+hSBN24Yym7eDOTkvyd4pYkpp/CyXnaMGcDJyYmy5QQFChZk/949XLl8mY6dumTZ+rf8cxZHB3vav1bTUGZnZ0uH5jX5++AFnkbHGMpbN6nKz1+/x8I/DvLZlD+MLu/w6SvceRBBlxZ1kpS/06oO90IfcfDkZQB+XPgXzftMTfL4ceF2QEtC0WZKRHFxcYweOYwjhw/x0/RZVK1W3TwLzgQ5pZnOGmpGAD9IKb8SQjgDh4CTUspSiSeQUi4DliWfUUq5mmfnlhKX90j22lX//zHQwVyBpyYhyZz3OwfAvr17cXN3w83Nndp16lK+QkWav9GSHydOICY6hmJeXqxeuZybNwMZP3GSYTnHjx3lsxGfUk4I3mrTjtOnThrec3R0pHyFrOt/0b5DJ1YsW8qQQQMYOHgINtgwc/pUinh40LFj5yyLIz2yW8znz/uxf+8eKuh/139PHGfRwl/o0asP1WvUTGNu09naQLtXq1OjQgkAXn+pIncfPOTOg4fsO+7PKRnI6m3HmTTsbRzsbbl68x79OjaiVLEC9Px8kWE5DWt6s3hCT05fvMmSjYeoW6WU4b0nT2M4JQMBiImJY+yszUz9rBNBt0PZeVjSuG45urepzycT1xAdo2WZi1dvcfFq0q7VJYtq3bDj483Xvv7dt9+wfdtW+vbrT548eZJ8r4oU8bCq5rrsnWJMZxUXvQohlqH1assFzJJSTrdwSKky5aJXgBqVjQ/VU6t2HeYv0i76i4qKYsa0KWzdvFkbDkiUZ8jHn1K77rOurHNmTmfu7JlGl+VZtChbtu9MMxZzHjEFBwUlHVqnvi/DR42mWLKhjqxJdorZ3/8S477+igD/Szx9+pTSZbx55933aNvOvOcIc6dyGLrn2CVe7ztVmyaXA98MfItOb9TGNX8ezly8yedT17P3+LNmz88/aMkX/VsaXda1oHuUbzUmSVnvtxsy5P1mlPB040bIA6Yv2cXPq5O3yif13lv1mDf2fbNe9NritaYEBaUcrgig/4CBfPjRILOsxxwXvR69EmbyZtcp7ZJtc5dVJKPsxNRkZE2ye/VdyRxudQZaOoR0eXB0hqVDSDdzJKNjV8JN/s2pXdo5237ZraWZTlEURTEipxxLqmSkKIpixXJILlLJSFEUxZolH5vSHIQQ5YAFgBsQC4yTUq7WL5NZDrijXQbTW0oZK4TIDSwBqqMN29ZR7+WMEOILoAcQDfSTUj7/JGAqLN61W1EURUldJt1C4gla4qiEdm3nVCFEfmAi8J2U0gewAzrp0/cBAvXyhWhjhyKEqAK0ASoA7YDZGd1OlYwURVGsWGaMwCClvCal9NOfhwB3gIJow6xt0Cf7FW28UNAGFEi498dyoJX+/C1ghZQyWkp5AQgTQlRI5yYCqplOURTFuqUjywghXAFXI2+FSilDU5mnDtrdCx4CD6SUCTeWCgQS7qRYFLgJ2rWaQgg7IYSDXp74Vr0J85wnnVTNSFEUxYrZpOMfMBRt8Ofkj6HGli2EKIxWA+qNhcfsVMlIURTFiqXznNFPaONtJn/8lHy5+r3dNgBjpZQHgXuAmxAiIS94odeGgCD0WpIQIg8QK6WMTlxuZJ50Uc10iqIoViw9HRP0prjQtKbT7569AvhDSplwS514IcR+tPND64BuaOOFAmwC3ke7HU9XYEui8l+EENMAb8BVSpnuJjpQyUhRFMWq2WTOlUYt0DohlBBCvKOXvQ+MAFYIISaj3VFhlf7ePGCpEMIfuAV0BJBSnhZCbAIuoHXt/iCjAanhgNJJDQek/Feo4YAynzmGA/ILemTyb07Fonmz7Zdd1YwURVGsWLbNLumkakbpFBVj+bvEpldEZEzaE1mR/HnUMZKS0pkbKe+MbO3MMYr2+WDTa0YVPFXNSFEURckEOaWZXSUjRVEUK5YzUpFKRoqiKNYth2QjlYwURVGsWCZ17bY6KhkpiqJYsRxyykglI0VRFGuWQ3KRSkaKoijWLDNurmeNVDJSFEWxYjkkF6lkpCiKYs1ySC5SyUhRFMWq5ZBspJKRBYUEBzNp4gQOHdxPfHw89XwbMGLkaDyLFs3SOAb268HJE0eNvlfXtyH/m/4zAOHhYcyaOpm9u3fw5MkTKlWtxuBPRuLtUy7JPE+ePGH+nOls37KRiIcRlC1Xng8HfUL1mrUzdTtuhYSw4Jd5+J07y0V5gaioKLZs30GxYl6Gac6dPcPvq1dx/PhRQoKDcXV1o2atWnw0eCheXsUzNT5TWct+kR6WiPnI3h0c3L2dy5fOEx76gAKFilCnYRNad+lBHqe8AFy5dJ5Vi2YTeDWAh+FhOOXLRymf8rR9pxdlK1ZNddkLpk1g55Y/aNDkDQaMHJvkvadPn7Bm8Vz27/yTx48eUrJMWbr0Hkj5KjUzZTtzStduNTZdOplrbLrIyEg6tW+Dg6MjAwcPxcYGZkybSlRUJKvXbsDJyckcqwHSHpvuymV/Hj16lKTs3OmTTJ/yA5+M/IL2Hd8hPj6eAX3eJyQ4iAGDPyW/szNLFs3nSoA/C5f9TuEiHoZ5v/liBAf37WHAkE8pWqw4a1cv59CBvcxdsJSyokKa8WZ0bLqjRw4zYtjHVKxYidjYOA4e2JciGU2eNJHTJ/+l5Ztv4e1Tltu3bvHz3Fncv3efVb+vw8PTM0PrNpes3C/MJatiTj423ZihvShQqAi1fF/BvWBhrgVI1i6Zh2fxUoz533xsbW05++8Rjh/4h3KVquHqXpDw0Pts/WM5ly+d56vJ8/AWlVKs5+K5U0z8fDC2trbUqNcoRTKaNfFLTh7Zzzt9BlPIoyh/b1zDqWMH+XrKL5T0TnpgZo6x6a7ff2Lyb04J91zZNnOpmpGFrF2zisDAG6zftJUSJUsCULacoHXL11mzaiXdevTMslhKl/FJUbbxjzU4ODjwavMWAOz7ZxdnTv3LtDkLqFm7HgCVq1anY+vmLPt1AUOHjwbg0sUL/LV1M5999S2tWrcDoHrN2rzfqQ3z58xg4pSZmbYdtWrXYdeeAwCsXbOagwf2pZimZ+++uLu7JymrXrMmLZs34/c1q/ho0JBMi88U1rRfmMpSMX/69WScXd0MrytUrUne/M7M/fEbzp8+TqXqdahcoy6Va9RNMl/V2r582Lk5+3ZsSZGMYmJiWDBtAm269GTnlj9SrPPa5Ysc2LWNvp98ySvN3zKsd2S/Lqz5dS6ffjPZ7Ntpm23TS/pk6m3HhRC7hRDNkpWNE0KMFUI0ycx1m0II4SqEuGqJde/etZOqVasZvrwAXl7FqV6jJrt37bBESAZRUZHs2rGNho0a4+ziCsC+PbsoWKiwIREB5MuXn4aNGrP3n52Gsv17dmFvb0+z5m8Yyuzt7Xn19RYcObSfp0+fZlrctrZp787JExFA0aLFcHN35/btW5kRVrpY836RGkvFnDgRJShTriIAD+7eSXW+XLnzYO/giJ2dXYr3Nq/5jbi4OFp2eM/ovCcO7sXO3p76L79mKLOzs8e3cXPOnDhEdKbs3zbpeGRfmZqM0G5r2ylZWWdgmZRyVyavOwX9VrtWIcDfH++y5VKUe3v7cDnA3wIRPbNn1w4eP3rEG2+2MZRduexPGe+UNajSZXy4FRLM48daM9+VgAA8i3qRO3eeJNOVKuNDdHQ0gTeuZ27wGXA5IID79+5Rpoy3pUOx6v0iNdYU84UzJwAoWqJUkvK4uDhiYmK4ezuExbMmAdDkjbZJpgkJusH65QvoMXAE9vbGG41uXr9MoSJFyZU7d5LyYiVLExMdza2gG+bZkERsbEx/ZGeZ3Uy3BhgjhPhIShkjhKgBRAKjhBDrpJTr9JrJYrT7rgO0k1JeFUIUAuYCJdBuZzsA8Af+BYSUMlYI4QHslFJWFEL0BfoCuQA/oLuU8qkQYhEQBdQEtgkhFgNLASee3cc9y4WFheHs7Jyi3MXFhfDwcAtE9MzWzetxcy9A/QaNDGXh4WF4ehZLMa2ziwsAEeHhODnlJTw8jPxGtsvZOWE667onTUxMDN+OHYObuzvt2newdDhWvV+kxlpivn/3Nr//+jOVa9Q11JASTP9uNEf3aTV4Z1d3ho+dQrGSZZJMs2j6RGo3aELFaql3tHkYEU7e/Cm3NV8+bf9++ND825vNc4zJMrVmJKW8C5wCEprqOqPVlpILkVLWAJYDCfdC/gn4XkpZG+gOzJVSRqDdlz2hDeg9tMQC8LuUsq6UshoQDCTc1x2gMFBfSvmlvtz/SSmrANb1y2gF7t65zbEjh2j+RqtUjw7/SyaMH8upk//y3feTDIlVyX6iIh8z5Zth2NrZ0feTL1O8/07vQXwzdRFDvpiIV6kyTB7zCZcv+hne37fjTy5f9OPdfpY9Z2hMTqkZZXYzHSRtqusIrDQyzVr9/+NAKf35q8DPQoiTwCqgiF7+C9BDf/4+8Kv+vJoQYp8Q4gzQAUh8aLRGShmnP68HrNafL0n/5piHs4uz0aPG1I4ys8q2LRuJi4tL0kQHkD+/MxERKXN3eJhWllAbyu/sTISR7QoPT5jOen7wf/rfj/y+ehVfjxtPg4YvWTocwHr3i+exdMxPn0Qxecyn3A4OYuT4aRQoVCTFNIU9i+EtKlLnpSaMGDcVZ1c3Vi+eA2iJbNnPP/Fmx27YOzjy6GEEjx5GEB8fR2xsDI8eRhATo/VIzZsvP48iUm7rw4fa/p0vn/m318bGxuRHdpYVyegPoKUQoiFwT0p52cg0T/T/Y3nWdGgL1JVSVtcfJQGklHuA8kKI14DbUsqERtoFQD+9xvMDWnNdgsfm3aQX5+3tQ4D/pRTlly8HGD03k1X+3Lwen3KCsuXKJykvXcaHK5cDUkx/9UoARTw8cdKv6yhdxpvgoECioiKTTnc5AAcHB7yKl8i84NNh3tzZLPxlHiM/+4K3Wre1dDgG1rpfPI8lY46JiWHqt59x5dJ5ho+bQvHSaa/P3sGB4qXLcisoEICIsFDCwx6watEsPujQzPC4d+cWh/f8zQcdmnHyiNYz06tkGe7cCuJJVFSSZd68dgV7BweKFDX/tWo5o/tCFiQjKWUYcBiYhfFaUWp2AB8lvBBCVEv03jK080yLEpXlA24LIRxJ2kSX3GHgbf1513TEY1aNmzTlzOlTBN54dsLz5s1ATv57gleaNLVITBf8znL1cgAtWrVJ8d5LrzThzu1b/Hv82cWxjx4+ZP/e3bz08rOOkQ0bNSEmJoZdf28zlMXExLDzr63Uqd8AR0fHTN0GUyxd8iszpv3EoCEf8867xntNWYo17hdpsVTMcXFxzJr4JX6njjH0qx/wqVDFpPmeREVx5eJ5iujnQF3cCzB64uwUDxc3dyrXqMvoibMRlaoDUKNeI2JjYji892/D8mJjYzi852+q1KyHQybs3zmlmS6rTgqsQEsg6UlGg4A5QoieaHFuRDv/BNq5pVE8a94D+Ao4CtxC6+SQmqHAMiHEl1iwA0P7Dp1YsWwpQwYNYODgIdhgw8zpUyni4UHHjp0tEtPWzRuws7OneYs3U7z30stNqFy1OuO+GmW46PW3hfOJj4+na7dehunKla9As9daMHXyRGJiYvAs6sW6NSsIDgrkq28nZvo2/LVtKwB+fmcB2L93D25u7ri5u1O7Tl3+3LKZSd9/R8OXGlG3Xn1OnzppmDdv3nx4+1i29mGN+0VaLBXz4pk/cGTvDtp06Unu3HnwP3/G8J5bwcIUKFSEX6ZOIF9+Z0qXq0B+Z1fu3g7mrw2rCX1wl/4jvgbA0TEXFavVSrF8B4dcOLu6J3mvlI+g/iuvsWTuFGJjYijkUZQdm9dyJySID0eMTbEMc1AjMFgxIURXoJGU8sOsXre5RmAACA4KSjqESn1fho8anWTEAHNIawQGgJiYaNq+0YSKVaryw5RZRqcJDwtlxk8/svefHTx98pTKVasx8OMRKZr0nkRF8fOsqfy1bTMPIyLwLiv4cNAn1Kxd1+hyk8voCAwA1SoJo+W169Tll0W/8eXoUWxYn/JixsTTWFpW7RfmlBUxJx+BYWi3Nty9HWx02nbv9uHt9/vxz7YN7N66nuDAazyJisKtYCG8RSVad+6RZpPe0G5tKFepWsrhgJ5EsWrRbA7u3sbjhw8pUaYsnXsNNJrQzDECw52HMSb/5hTKZ59tM1e2S0ZCiNlovfOaSymvZvX6zZmMsoopyciavEgyUv67kiej7MAcyehuOpJRwWycjLLdt94StSFFURRLsc3uJ4NMlO2SkaIoSk6SQ3JRlnTtVhRFUZTnUjUjRVEUK5ZTakYqGSmKolixnNK1WyUjRVEUK6ZqRoqiKIrFqWSkKIqiWJxqplMURVEsTtWMFEVRFIvLrFwkhGgJTAHsgKlSyumZtCqTqOuMFEVRrFkm3ENCCGEPTEO7b1xVYIAQwqKDH6qakaIoihVLz3BAQghXwNXIW6FSytBEr+sC5xPuByeEWAu8BczOaJwvSiWjdMptn/3OJubOr/7MSvZXp7T13CU4K6XzN+drYIyR8m/09xIUBW4meh0IFEtvbOakfqUURVH+O34i6U1HE4RmaRQZoJKRoijKf4TeFBdqwqRBJK0JeaHVjixGJSNFUZSc5whQUQhRHLgLtAeaWzIg1ZtOURQlh5FSxgBDgR3AWWBOQmcGS8l2d3pVFEVR/ntUzUhRFEWxOJWMFEVRFItTyUhRFEWxOJWMFEVRFItTXbszQAgRC5zRX8YCH0kpD6UxzxagM1AAWCelrG5kmpNAW+ApMEVK2dl8URuNaQoQIKWcob8+BOyTUg7TX68EVkkpf8/g8ocCrlLKr80TcZJlJ/wNHIDLwPvJhjvJyDJdgZNSylIvGp+RZXcBlgElpJQZup5DCHEVqP6i22nNEr4DUsqrqby/GxgnpdyRqGwcEA/sklLuyoIwU5WZ+9B/naoZZUyElLK6nlBGA+PTmkFK2VJKGWHKwqWUQZmdiHQHAV8AIUQutB/2Wone9wUOZEEcGZHwN6iEdpHfRxaOJy1d0D7LrPi7ZiohhJ0FV78C6JSsrDOwzBKJyMKfxX+Kqhm9OGcgDEAI0RgYKqVsq79ehFYLWpdwVJt4RiGEE7AYqAScRksGCCFK6fNVF0L0AN7U11MGWCylHKdPNwboCoQA94H1UspFQojvgdZADLA9oaZjxAFgov68FrAfqCGEcAAKo9X6mgghPkMbE/h3KeUYfd1dAWPlfYCRwD1AAlf08sFAfz0mPyllFxM+W1PtB6rp66kOzAXyAOeBPlLKCCFEX6AvkAvwA7pLKZ8KIXyApYATsCVhgUKISsBCwBHtoO1tKeWljAQnhHABaqINRDkPmKz/Xdug1ZQ9gLlSysn6334z2rUfVYFjQC8pZXSyZb4PDNbj2yGl/EQIkRdYhXY1vR1aDWJlBuJNsV+hjWu2Eu3CyDFCiHwY//uHSild9ec90GpyQ/XvQjhQB23fGiil/DO178BzrNHX/5GUMkYIUQOIBEYJIRJ/1xajfQcA2kkprwohCqHtGyWAaGAA4A/8CwgpZawQwgPYKaWs+Jx9ZhEQhfY33SaEWIyRfUhJH1Uzypj8QoiTQogLaD8uadaMUjEAuCulrAhMByqmMl014G39/w+FEK5CiDpoX7aqaE17dQCEEAWAdkAlKWVV4NvUVq43F9npX0Bf4BBwCqgBNEBLJOOBJmhfvOZCiKZCiKLPKf8cqKe/Vz3R6kYBNfSY+pv6AaVFPzJ9DdioF/0KfKKv5yYwXC//XUpZV0pZDQgG3tHLfwL+J6Wsgn5QoeuPdo+X6kBtXmyolHbAJinlKcBJCFFGL6+LlpBqAn2FEOX08op6TBXQkkr3ZNtcQV+mr749hYUQrYA3gCApZTUpZWVga3oDTW2/0gVJKWugJcgUf38TFl8Ibb96Bxinl5n6HQBASnkXbR9tphd1RqstJReix7ocGKiX/QR8L6WsjfaZztVbK/ajfXYA76ElFkh9nwEtodaXUn5J6vuQkg4qGWVMQhNReaAF8KsQIiOjeb+E9mVBSrkfvRZhxN9Syggp5SMgAO3ItyFa7emJlPIBsE2fNgztqO0XIUR74HEaMSQ01fnqzw8ler0erR3+rpTyKdqXvhHaD5Sx8rp6+X0pZSSQ+FzTaWCpEOI9tNrRi8qvn1+4hVaz2K7XQPLpnyVoR8eN9OfVhBD7hBBngA48+9GrB6zWny9JtPyDwGghxCigpL49GdUFrcYCWu0ioalum5TygZTyIdoRdQO9/IqU8rD+fAXafpJYMz3uY/pnUBfwQTuH9poQ4gchRCMpZUZ+GFPbr+DZ55Ta3z8t66SU8cBxoJReZup3ILHETXUd0T7T5Nbq/yde16vAz/pntgooopf/AvTQn7+PdkADqe8zAGuklHH689T2ISUdVDJ6QVLKg0BBtKO+GJJ+prlMWIQpQ2A8SfQ8luc0r+rDfNRFa85oSdpHxwfRfgRLSymvoCWj+nqZqwmxJZfa9rQCZqDVuo7oN/d6ERF6raUkWlPRgDSmXwD0049efyCNv42UchlaDeERsMnEI/8UhBAFgVeAxXrz0Qc8S0aJP6v4RK9TK09gCyxIOG8ppSwnpZwqpbyIVlM5DYwVQnyVkZifI60Dm4R4EyT/jJ8ASCmT78PpHQbmD6ClEKIhcE9KednINAnfmcTrsgXqJvrcSurx7AHKCyFeA24nGhbnefuMKZ+Fkg4qGb0gIUR5tKaUe8A1oIIQwkEI4Y72I/Q8+9B/mIQQvkDpdKx6P9BaCOGo1wia68vJh9aDbQvwMVAljeUcQGsCvAEgpfQHKgDl0L6MjYUQ7kIIRz3WPWiDLKZW3kQI4SaEyI3WlIQQwhYoLqXcjXY+KT+QLx3bmiq9tjgY+BQtcUTonyVoR7l79Of5gNt6vImbWw7r2w/aeRL0mMsAl6V2K+Y/0JqtMqIDWuIoKaUsJaX0Qvve5UZr3nLVz/W0RDswACitN5eB9tnuS7bMHUAnPdEhhCgshPDUm0kjpZRL0M4F1sxAvEb3q2RS+/sD3BNCeOvNp62NzJtcur8Deo3vMDAL47Wi1OwgUUcXIUS1RO8tQ6tJL0pUlto+k5zRfUhJH5WMMibhnNFJtC9DdyllrH5E9QdwDq26fiKN5cwCPIQQfmg/qH6mBiClPAr8iXaiez3a0XA42g/9RiHEabQfltQ6LyT4F/BEqxElCARO69vzBfCPvi1/SSl3SSmDUykPAr5D+7HarccEWrJeojd3nARmSjN2T5ZSHkNrouqEdi5gir79JYAf9cm+Ao6i/WieTjT7UGC4Pr1rovJOwFkhxL9o575+JWO6oO0Tif2Bdg7tGNrf7iQwX6/ZgNbx4lMhxHn9dZJ1SynPAWOBHXrcmwB3tAOPw/p+OZbnnC9MzXP2q8TTGP37629/BmwH9gLXTVhlRr8DK9C2Nz3JaBDwkhDitL6+xIljOZCXZ817kPo+k9xQjO9DSjqogVKzMSFEPinlQyGEM1oyeTOVJgvFyiTuaZasvBSpXIeWVXLifqX3Dm0kpfzQ0rHkVKprd/b2sxCiIlpb9qz/+g+GkmVy1H4lhJiN1inEovfzyelUzUhRFEWxOHXOSFEURbE4lYwURVEUi1PJSFEURbE41YFByRGEEPFow8jYAHHAMJlo5OcMLvMq8KqU0l9oo7J/kOiCSWPTDwV+lVLez8C64gEH/aJmRfnPUTUjJSeprY8z9jWwUr8Y1+BFRoWQ2qjsqSYi3VC064EURUlG1YyUnOgvtNGyCwghVqNd+NsA8BdC9AK+RxujLRfaCAGD9RGdq6CN5J1bLzeMR5isllQFmKqvA7QLXKsBRYENQoinaCMuPAKmoY14kRtYLZ+NyN4YbfikWFJeNKso/zmqZqTkRJ2AG1LKO/prL7QRsN9FG64oUEpZF23khTxAb326xcAEfUTs7WgjPCQhtNtvrAN+1GthNYBDUsrvgSCgtT4uWhAwBdigr6sm0EgI0Vxo95ZaCvTUlxGefD2K8l+jakZKTnJMCAHafXraJCpfmmgE5reAfEK7XxBoyeiePk6bj9TveiulXCuECDWyDgE80ccGRF/ug1TieQuoJYT4Un+dDyiPNhL5fX1oHtBGlZ6cri1VlGxGJSMlJ6mdSgeAh4me26DdzO5g4gn0ZGRuNsAb+lhviddVjfSPZK0o2ZpqplOUpDYCn+gjNSOEKCCEKK2PFO0vhEgYibwtxgfFlICDEKKlPp2tEMJNfy8c7Y69idc1MqEjhRDCS7/R4QXAXQiRcAv4nubcQEWxRioZKUpSE4DLwHF9FObtQDH9ve7A5/ro469jZFRqqd0evB0wQp/uBNrN1wBmAsv1Ed+LAkPQRoo+pU+7Cu32H0/Q7ji6WAhxCsiMWpmiWBU1Np2iKIpicapmpCiKolicSkaKoiiKxalkpCiKolicSkaKoiiKxalkpCiKolicSkaKoiiKxalkpCiKolicSkaKoiiKxf0fPB/BcAHDoKoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "datasetNames = [\"Trento\"]\n",
    "data2Name = 'LIDAR'\n",
    "\n",
    "patchsize = 11\n",
    "batchsize = 64\n",
    "testSizeNumber = 500\n",
    "EPOCH = 100\n",
    "BandSize = 1\n",
    "LR = 5e-4\n",
    "FM = 16\n",
    "HSIOnly = True\n",
    "FileName = 'MFT_HSI'\n",
    "ntokens = 4\n",
    "token_type = 'channel' \n",
    "num_heads = 8\n",
    "mlp_dim = 512\n",
    "depth = 2\n",
    "train_loss = []\n",
    "\n",
    "def set_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "for BandSize in [1]:\n",
    "    for datasetName in datasetNames:\n",
    "            print(\"---------------------------------- Dataset details for \",datasetName,\" ---------------------------------------------\")\n",
    "            print('\\n')\n",
    "            try:\n",
    "                os.makedirs(datasetName)\n",
    "            except FileExistsError:\n",
    "                pass\n",
    "            \n",
    "            train_dataset = Multimodal_Dataset_Train(Filename=datasetName, MM_Data=data2Name)\n",
    "            test_dataset = Multimodal_Dataset_Test(Filename=datasetName, MM_Data=data2Name)\n",
    "            NC = train_dataset.hs_ims.shape[1]\n",
    "            NCLidar = train_dataset.lid_ims.shape[1]\n",
    "            Classes = len(torch.unique(train_dataset.lbs))\n",
    "\n",
    "            train_loader = dataf.DataLoader(train_dataset, batch_size=batchsize, shuffle=True, num_workers= 4)\n",
    "            print(\"HSI Train data shape = \", train_dataset.hs_ims.shape)\n",
    "            print(data2Name + \" Train data shape = \", train_dataset.lid_ims.shape[1])\n",
    "            print(\"Train label shape = \", train_dataset.lbs.shape)\n",
    "\n",
    "            print(\"HSI Test data shape = \", test_dataset.hs_ims.shape)\n",
    "            print(data2Name + \" Test data shape = \", test_dataset.lid_ims.shape[1])\n",
    "            print(\"Test label shape = \", test_dataset.lbs.shape)\n",
    "\n",
    "            print(\"Number of Classes = \", Classes)\n",
    "            \n",
    "            TestPatch1 = test_dataset.hs_ims\n",
    "            TestPatch2 = test_dataset.lid_ims\n",
    "            TestLabel1 = test_dataset.lbs\n",
    "            \n",
    "            KAPPA = []\n",
    "            OA = []\n",
    "            AA = []\n",
    "            ELEMENT_ACC = np.zeros((3, Classes))\n",
    "\n",
    "            set_seed(42)\n",
    "            for iterNum in range(1):\n",
    "                print('\\n')\n",
    "                print(\"---------------------------------- Model Summary ---------------------------------------------\")\n",
    "                print('\\n')\n",
    "                if HSIOnly:    \n",
    "                    model = Transformer(FM=FM, NC=NC, Classes=Classes, ntokens=ntokens, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n",
    "                    summary(model, [(NC, patchsize**2)])\n",
    "                                        \n",
    "                else:\n",
    "                    model = MFT(FM=FM, NC=NC, NCLidar=NCLidar, Classes=Classes, ntokens=ntokens, token_type=token_type, num_heads=num_heads, mlp_dim=mlp_dim, depth=depth).cuda()\n",
    "                    summary(model, [(NC, patchsize**2),(NCLidar,patchsize**2)])                    \n",
    "                    \n",
    "                optimizer = torch.optim.Adam(model.parameters(), lr=LR,weight_decay=5e-3)\n",
    "                loss_func = nn.CrossEntropyLoss()  # the target label is not one-hotted\n",
    "                scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.9)\n",
    "                BestAcc = 0\n",
    "\n",
    "                torch.cuda.synchronize()\n",
    "                print('\\n')\n",
    "                print(\"---------------------------------- Training started for \",datasetName,\" ---------------------------------------------\")\n",
    "                print('\\n')\n",
    "                start = time.time()\n",
    "                # train and test the designed model\n",
    "                for epoch in range(EPOCH):\n",
    "                    for step, (b_x1, b_x2, b_y) in enumerate(train_loader):\n",
    "\n",
    "                        # move train data to GPU\n",
    "                        b_x1 = b_x1.cuda()\n",
    "                        b_y = b_y.cuda()\n",
    "                        if HSIOnly:\n",
    "                            out1 = model(b_x1)\n",
    "                            loss = loss_func(out1, b_y)\n",
    "                        else:\n",
    "                            b_x2 = b_x2.cuda()\n",
    "                            out= model(b_x1, b_x2)\n",
    "                            loss = loss_func(out, b_y)\n",
    "\n",
    "                        optimizer.zero_grad()  # clear gradients for this training step\n",
    "                        loss.backward()  # backpropagation, compute gradients\n",
    "                        optimizer.step()  # apply gradients\n",
    "\n",
    "                        if step % 50 == 0:\n",
    "                            model.eval()\n",
    "                            pred_y = np.empty((len(TestLabel1)), dtype='float32')\n",
    "                            number = len(TestLabel1) // testSizeNumber\n",
    "                            for i in range(number):\n",
    "                                temp = TestPatch1[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "                                temp = temp.cuda()\n",
    "                                temp1 = TestPatch2[i * testSizeNumber:(i + 1) * testSizeNumber, :, :]\n",
    "                                temp1 = temp1.cuda()\n",
    "                                if HSIOnly:\n",
    "                                    temp2 = model(temp)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n",
    "                                    del temp, temp2, temp3\n",
    "                                else:\n",
    "                                    temp2 = model(temp, temp1)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[i * testSizeNumber:(i + 1) * testSizeNumber] = temp3.cpu()\n",
    "                                    del temp, temp1, temp2, temp3\n",
    "\n",
    "                            if (i + 1) * testSizeNumber < len(TestLabel1):\n",
    "                                temp = TestPatch1[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n",
    "                                temp = temp.cuda()\n",
    "                                temp1 = TestPatch2[(i + 1) * testSizeNumber:len(TestLabel1), :, :]\n",
    "                                temp1 = temp1.cuda()\n",
    "                                if HSIOnly:\n",
    "                                    temp2 = model(temp)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n",
    "                                    del temp, temp2, temp3\n",
    "                                else:\n",
    "                                    temp2 = model(temp, temp1)\n",
    "                                    temp3 = torch.max(temp2, 1)[1].squeeze()\n",
    "                                    pred_y[(i + 1) * testSizeNumber:len(TestLabel1)] = temp3.cpu()\n",
    "                                    del temp, temp1, temp2, temp3\n",
    "\n",
    "                            pred_y = torch.from_numpy(pred_y).long()\n",
    "                            accuracy = torch.sum(pred_y == TestLabel1).type(torch.FloatTensor) / TestLabel1.size(0)\n",
    "\n",
    "                            print('Epoch: ', epoch, '| train loss: %.4f' % loss.data.cpu().numpy(), '| test accuracy: %.4f' % (accuracy*100))\n",
    "                            train_loss.append(loss.data.cpu().numpy())\n",
    "                            # save the parameters in network\n",
    "                            if accuracy > BestAcc:\n",
    "\n",
    "                                BestAcc = accuracy\n",
    "                                \n",
    "                                torch.save(model.state_dict(), datasetName+'/net_params_'+FileName+'.pkl')\n",
    "                                \n",
    "\n",
    "                            model.train()\n",
    "                    scheduler.step()\n",
    "                torch.cuda.synchronize()\n",
    "                end = time.time()\n",
    "                print('\\nThe train time (in seconds) is:', end - start)\n",
    "                Train_time = end - start\n",
    "\n",
    "                # load the saved parameters\n",
    "                \n",
    "                model.load_state_dict(torch.load(datasetName+'/net_params_'+FileName+'.pkl'))\n",
    "\n",
    "                model.eval()\n",
    "                \n",
    "                confusion, oa, each_acc, aa, kappa = reports(TestPatch1,TestPatch2,TestLabel1,datasetName,model, HSIOnly, iterNum)\n",
    "                KAPPA.append(kappa)\n",
    "                OA.append(oa)\n",
    "                AA.append(aa)\n",
    "                ELEMENT_ACC[iterNum, :] = each_acc\n",
    "                torch.save(model, datasetName+'/best_model_'+FileName+'_BandSize'+str(BandSize)+'_Iter'+str(iterNum)+'.pt')\n",
    "                print('\\n')\n",
    "                print(\"Overall Accuracy = \", oa)\n",
    "                print('\\n')\n",
    "            print(\"----------\" + datasetName + \" Training Finished -----------\")\n",
    "            print(\"\\nThe Confusion Matrix on test data\")\n",
    "            record.record_output(OA, AA, KAPPA, ELEMENT_ACC,'./' + datasetName +'/'+FileName+'_BandSize'+str(BandSize)+'_Report_' + datasetName +'.txt')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8d7a95c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss = np.asarray(train_loss)\n",
    "np.save('HSIOnly_train_loss.npy', train_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e19924b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "26407125db06bdd9abe40e82cf041582bb19887fa16dd38638e528b7039723e8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}