tesis_ujiindoorloc_heterogeneous_gnn_pyg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vp0qGc5z4AOk"
   },
   "source": [
    "# UJIINDOORLOC - Heterogeneous GNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oDifkXbM4UUX"
   },
   "source": [
    "Dataset: UJIINDOORLOC\n",
    "\n",
    "Modelo: GNN con grafo heterogéneo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n4qV8TIQYl62",
    "tags": []
   },
   "source": [
    "## Importar Datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 27989,
     "status": "ok",
     "timestamp": 1635510361213,
     "user": {
      "displayName": "Facundo Lezama",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjvX7z6H8HKQ9Q4HSRF20NaPjvwtJ3pQh0TGed2XQ=s64",
      "userId": "08104556030349101598"
     },
     "user_tz": 180
    },
    "id": "OjHf3j75Agti"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from itertools import combinations\n",
    "from copy import deepcopy\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "import torch\n",
    "import networkx as nx\n",
    "from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\n",
    "\n",
    "from torch_geometric.data import Data, HeteroData\n",
    "from torch_geometric.nn.conv.dna_conv import Linear\n",
    "from torch_geometric.utils import to_networkx, is_undirected, to_undirected\n",
    "\n",
    "import torch.nn.functional as F\n",
    "from torch_geometric.nn import GCNConv, ChebConv, SAGEConv, TAGConv, GraphConv, to_hetero, GATConv, Linear, BatchNorm, HeteroConv\n",
    "from torch_geometric.loader import DataLoader\n",
    "import torch_geometric.transforms as T\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "from zipfile import ZipFile\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score, classification_report\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 1270,
     "status": "ok",
     "timestamp": 1635454213295,
     "user": {
      "displayName": "Facundo Lezama",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjvX7z6H8HKQ9Q4HSRF20NaPjvwtJ3pQh0TGed2XQ=s64",
      "userId": "08104556030349101598"
     },
     "user_tz": 180
    },
    "id": "jXw5-YfeAgtp"
   },
   "outputs": [],
   "source": [
    "# Descarga de datos\n",
    "!kaggle datasets download -d giantuji/UjiIndoorLoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 383
    },
    "executionInfo": {
     "elapsed": 1105,
     "status": "ok",
     "timestamp": 1635454214395,
     "user": {
      "displayName": "Facundo Lezama",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjvX7z6H8HKQ9Q4HSRF20NaPjvwtJ3pQh0TGed2XQ=s64",
      "userId": "08104556030349101598"
     },
     "user_tz": 180
    },
    "id": "OIhn7QSoAgtq",
    "outputId": "37bf3f47-b29c-4af5-aec4-a23a675b0eef"
   },
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df = pd.read_csv(zip_file.open('TrainingData.csv'))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gfqVFAK32Th4",
    "tags": []
   },
   "source": [
    "## Preprocesamiento"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1635454214397,
     "user": {
      "displayName": "Facundo Lezama",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjvX7z6H8HKQ9Q4HSRF20NaPjvwtJ3pQh0TGed2XQ=s64",
      "userId": "08104556030349101598"
     },
     "user_tz": 180
    },
    "id": "q0BCpSLFAgts"
   },
   "outputs": [],
   "source": [
    "def preprocess_dataset(dataset_path, filter_std, dataset_percentage=None):\n",
    "    \n",
    "    zip_file = ZipFile(dataset_path)\n",
    "    df = pd.read_csv(zip_file.open('TrainingData.csv'))\n",
    "    \n",
    "    df['CLASS'] = df['BUILDINGID'].astype(str) + df['FLOOR'].astype(str)\n",
    "    \n",
    "    df_X = df.iloc[:,:520]\n",
    "    df_y = df['CLASS']\n",
    "\n",
    "    df_X.values[df_X.values==100] = -105\n",
    "\n",
    "\n",
    "    # keep those APs where std > filter_std\n",
    "    ap = (df_X.describe().iloc[2]>filter_std).index\n",
    "    values = (df_X.describe().iloc[2]>filter_std).values\n",
    "    filtered_aps = [ap[i] for i in range(len(values)) if values[i]==True]\n",
    "    df_X = df_X[filtered_aps]\n",
    "        \n",
    "    # take minimum -105 to 0\n",
    "    df_X.iloc[:,:] = 105 + df_X.values\n",
    "    df_X['CLASS'] = df_y.values \n",
    "    \n",
    "    if dataset_percentage:\n",
    "        df_X = df_X.sample(frac=dataset_percentage)      \n",
    "    \n",
    "    # apply ordinal encoder to the classes and split X, y\n",
    "    enc = OrdinalEncoder(dtype=int)\n",
    "    y = enc.fit_transform(df_X['CLASS'].values.reshape(-1,1))\n",
    "    X = df_X.iloc[:,:-1].values    \n",
    "\n",
    "    dfaux = pd.DataFrame(X)\n",
    "\n",
    "    number_aps = len(dfaux.columns)\n",
    "    dfaux[str(number_aps)] = y\n",
    "    subsample = dfaux.sample(frac=1, random_state=99)\n",
    "    y = subsample.iloc[:, -1].values.reshape(-1,1)\n",
    "    X = subsample.iloc[:, :-1].values\n",
    "    \n",
    "    # split 80-20\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "    \n",
    "    print(\"X_train shape: \", X_train.shape)\n",
    "    return X_train, X_test, y_train, y_test, number_aps, len(df_X['CLASS'].value_counts()), filtered_aps, enc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fmC2Dsazdx7i"
   },
   "source": [
    "## Grafo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cF2ZViCq3a_Q"
   },
   "source": [
    "En las siguientes celdas se describe un poco el dataset y se muestran las distribuciones de potencia por AP."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1635454218455,
     "user": {
      "displayName": "Facundo Lezama",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjvX7z6H8HKQ9Q4HSRF20NaPjvwtJ3pQh0TGed2XQ=s64",
      "userId": "08104556030349101598"
     },
     "user_tz": 180
    },
    "id": "xHfjTzU5E2eq"
   },
   "outputs": [],
   "source": [
    "def ap_graph_creator(X_G, th=10, prune_th=0):\n",
    "    \"\"\"\n",
    "    Dado un dataset y un threshold se arma un grafo basado en las medidas de RRSI\n",
    "    \"\"\"\n",
    "    df_data_train = pd.DataFrame(X_G)\n",
    "    df_G = pd.DataFrame(columns = ['from', 'to', 'weight']) \n",
    "\n",
    "    columns = df_data_train.columns.to_list()\n",
    "    for ap in columns:\n",
    "        # para cada AP me quedo con las instancias donde el RSSI esta en el rango\n",
    "        # (max-th) intentando estimar las instancias mas cercanas al AP\n",
    "        max_val = df_data_train[ap].max()\n",
    "        df_aux_i = df_data_train[df_data_train[ap]  > (max_val - th)]\n",
    "        df_aux_i = df_aux_i.drop(ap, axis=1) \n",
    "        # df_aux_i.head()\n",
    "\n",
    "        for k, v in df_aux_i.mean().items():\n",
    "            # armo las aristas con el promedio de RSSI que ven las instancias \n",
    "            # filtradas al resto de los APs\n",
    "            # weight = v\n",
    "            # if df_G.loc[(df_G['from'] == k) & (df_G['to'] == ap)].weight.any():\n",
    "            #     weight = np.mean([float(df_G.loc[(df_G['from'] == k) & (df_G['to'] == ap)].weight), weight])\n",
    "            #     df_G.loc[(df_G['from'] == k) & (df_G['to'] == ap)] = k, ap, weight\n",
    "            if v > prune_th:\n",
    "                df_G = df_G.append({'from':ap, 'to': k, 'weight': v}, ignore_index=True)\n",
    "        \n",
    "\n",
    "    edge_index_first_row = []\n",
    "    edge_index_second_row = []\n",
    "    edge_attr = []\n",
    "    for index, row in df_G.iterrows():\n",
    "        edge_index_first_row.append(columns.index(row['from']))\n",
    "        edge_index_second_row.append(columns.index(row['to']))\n",
    "        edge_attr.append([float(row.weight)])\n",
    "\n",
    "    \n",
    "    edge_index = torch.tensor([edge_index_first_row, edge_index_second_row], dtype=torch.long)\n",
    "    edge_attr = torch.tensor(edge_attr, dtype=torch.float)                           \n",
    "    edge_index, edge_attr = to_undirected(edge_index, edge_attr, reduce=\"mean\")\n",
    "    return edge_index, edge_attr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zone_ap_graph_creator(X_G, y, prune_th=0, zone_to_remove=None):\n",
    "\n",
    "    df_data_train = pd.DataFrame(X_G)\n",
    "    df_data_train['cls'] = y    \n",
    "    df_G = pd.DataFrame(columns = ['from', 'to', 'weight']) \n",
    "\n",
    "    for zone in np.sort(df_data_train['cls'].unique()):\n",
    "\n",
    "        filtered_instances = df_data_train.loc[df_data_train['cls'] == zone]\n",
    "        means = np.array(filtered_instances.mean())[:-1]\n",
    "\n",
    "        for ap, mean in enumerate(means):\n",
    "            if mean > prune_th:\n",
    "                if zone_to_remove == zone:\n",
    "                    mean = 0                \n",
    "                df_G = df_G.append({'from':zone, 'to': ap, 'weight': mean}, ignore_index=True)\n",
    "\n",
    "    edge_index_first_row = []\n",
    "    edge_index_second_row = []\n",
    "    edge_attr = []\n",
    "    for index, row in df_G.iterrows():\n",
    "        edge_index_first_row.append(row['from'])\n",
    "        edge_index_second_row.append(row['to'])\n",
    "        edge_attr.append([float(row.weight)])\n",
    "\n",
    "\n",
    "    edge_index = torch.tensor([edge_index_first_row, edge_index_second_row], dtype=torch.long)\n",
    "    edge_attr = torch.tensor(edge_attr, dtype=torch.float)                           \n",
    "\n",
    "    return edge_index, edge_attr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(X, y, graph, num_classes, zone_to_remove=None):\n",
    "    dataset = []\n",
    "    one_hot = torch.nn.functional.one_hot(torch.arange(0,num_classes)).float()\n",
    "    \n",
    "    for i in range(len(y)):\n",
    "        if y[i] != zone_to_remove:\n",
    "\n",
    "            data = deepcopy(graph)\n",
    "            data['aps'].x = torch.Tensor(X[i])\n",
    "\n",
    "            # data['zones'].x = torch.ones(data['zones'].num_nodes,1)\n",
    "            data['zones'].x = torch.zeros(data['zones'].num_nodes,1)\n",
    "            # data['zones'].x = torch.randn(data['zones'].num_nodes,5)\n",
    "            # data['zones'].x = one_hot\n",
    "            # data['zones'].x -= data['zones'].x.mean()\n",
    "            # data['zones'].x /= data['zones'].x.std()\n",
    "            data['zones'].y = torch.Tensor(y[i])\n",
    "            data['zones'].train_mask = torch.Tensor([True]*len(y))\n",
    "            data['zones'].val_mask = torch.Tensor([True]*len(y))\n",
    "            data['zones'].test_mask = torch.Tensor([True]*len(y))\n",
    "\n",
    "            dataset.append(data)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_heterodata(ap_edge_index, ap_edge_attr, zone_ap_edge_index, zone_ap_edge_attr, num_classes, num_aps):\n",
    "    data = HeteroData()\n",
    "    data['aps', 'ap_ap', 'aps'].edge_index = ap_edge_index\n",
    "    ap_edge_attr = (ap_edge_attr - ap_edge_attr.mean())/ap_edge_attr.std()    \n",
    "    data['aps', 'ap_ap', 'aps'].edge_attr = ap_edge_attr\n",
    "    data['aps'].num_nodes = num_aps\n",
    "\n",
    "    data['zones', 'zone_ap', 'aps'].edge_index = zone_ap_edge_index\n",
    "    zone_ap_edge_attr = (zone_ap_edge_attr - zone_ap_edge_attr.mean())/zone_ap_edge_attr.std()        \n",
    "    data['zones', 'zone_ap', 'aps'].edge_attr = zone_ap_edge_attr\n",
    "    data['zones'].num_nodes = num_classes\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HeteroGNN(torch.nn.Module):\n",
    "    def __init__(self, hidden_channels, out_channels, hidden_layers):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.hidden_convs = torch.nn.ModuleList()\n",
    "        for _ in range(hidden_layers):\n",
    "            conv = HeteroConv({\n",
    "                ('aps', 'ap_ap', 'aps'): GraphConv((-1, -1), hidden_channels),\n",
    "                ('zones', 'zone_ap', 'aps'): GraphConv((-1, -1), hidden_channels),\n",
    "                ('aps', 'rev_zone_ap', 'zones'): GraphConv((-1, -1), hidden_channels),\n",
    "            }, aggr='mean')\n",
    "            self.hidden_convs.append(conv)\n",
    "        \n",
    "        self.out_convs = torch.nn.ModuleList()\n",
    "        conv = HeteroConv({\n",
    "            ('aps', 'ap_ap', 'aps'): GraphConv((-1, -1), out_channels),\n",
    "            ('zones', 'zone_ap', 'aps'): GraphConv((-1, -1), out_channels),\n",
    "            ('aps', 'rev_zone_ap', 'zones'): GraphConv((-1, -1), out_channels),\n",
    "        }, aggr='mean')\n",
    "        self.out_convs.append(conv)        \n",
    "\n",
    "        # self.lin = Linear(hidden_channels, out_channels)\n",
    "\n",
    "    def forward(self, x_dict, edge_index_dict, edge_attr_dict):\n",
    "        for conv in self.hidden_convs:\n",
    "            x_dict = conv(x_dict, edge_index_dict, edge_attr_dict)\n",
    "            x_dict = {key: x.relu() for key, x in x_dict.items()}\n",
    "        for conv in self.out_convs:\n",
    "            x_dict = conv(x_dict, edge_index_dict, edge_attr_dict)\n",
    "           # x_dict = {key: x.relu() for key, x in x_dict.items()}\n",
    "\n",
    "        # out = self.lin(x_dict['zones'])\n",
    "        \n",
    "        return x_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HeteroGNN_simplified(torch.nn.Module):\n",
    "    def __init__(self, hidden_channels, out_channels, hidden_layers):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.hidden_convs = torch.nn.ModuleList()\n",
    "        for _ in range(hidden_layers):\n",
    "            conv = HeteroConv({\n",
    "                ('aps', 'ap_ap', 'aps'): GraphConv((-1, -1), hidden_channels),\n",
    "                # ('zones', 'zone_ap', 'aps'): GraphConv((-1, -1), hidden_channels),\n",
    "                # ('aps', 'rev_zone_ap', 'zones'): GraphConv((-1, -1), hidden_channels),\n",
    "            }, aggr='mean')\n",
    "            self.hidden_convs.append(conv)\n",
    "        \n",
    "        self.out_convs = torch.nn.ModuleList()\n",
    "        conv = HeteroConv({\n",
    "            ('aps', 'ap_ap', 'aps'): GraphConv((-1, -1), out_channels),\n",
    "            ('zones', 'zone_ap', 'aps'): GraphConv((-1, -1), out_channels),\n",
    "            ('aps', 'rev_zone_ap', 'zones'): GraphConv((-1, -1), out_channels),\n",
    "        }, aggr='mean')\n",
    "        self.out_convs.append(conv)        \n",
    "\n",
    "        # self.lin = Linear(hidden_channels, out_channels)\n",
    "\n",
    "    def forward(self, x_dict, edge_index_dict, edge_attr_dict):\n",
    "        initial_zones_signal = x_dict[\"zones\"]\n",
    "        for conv in self.hidden_convs:\n",
    "            x_dict = conv(x_dict, edge_index_dict, edge_attr_dict)\n",
    "            x_dict = {key: x.relu() for key, x in x_dict.items()}\n",
    "        x_dict[\"zones\"] = initial_zones_signal\n",
    "        for conv in self.out_convs:\n",
    "            x_dict = conv(x_dict, edge_index_dict, edge_attr_dict)\n",
    "           # x_dict = {key: x.relu() for key, x in x_dict.items()}\n",
    "\n",
    "        # out = self.lin(x_dict['zones'])\n",
    "        \n",
    "        return x_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YHsZyzkAZKlK"
   },
   "source": [
    "## Entrenamiento"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 356,
     "status": "ok",
     "timestamp": 1635462144424,
     "user": {
      "displayName": "Facundo Lezama",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjvX7z6H8HKQ9Q4HSRF20NaPjvwtJ3pQh0TGed2XQ=s64",
      "userId": "08104556030349101598"
     },
     "user_tz": 180
    },
    "id": "0MsSCtZkUDbj",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Split de los datos y armado del objeto Data con el grafo\n",
    "\n",
    "X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3, dataset_percentage=0.3)\n",
    "ap_edge_index, ap_edge_attr = ap_graph_creator(X_train[:,:-1], th=10, prune_th=20)\n",
    "zone_ap_edge_index, zone_ap_edge_attr = zone_ap_graph_creator(X_train[:,:-1], y_train, prune_th=10)\n",
    "heterodata = build_heterodata(ap_edge_index, ap_edge_attr, zone_ap_edge_index, zone_ap_edge_attr, num_classes, num_aps)\n",
    "T.ToUndirected()(heterodata)\n",
    "print(f\"Undirected: {heterodata.is_undirected()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "heterodata.is_undirected()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Armado del dataset\n",
    "\n",
    "x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_training_data = y_train\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "mean = x_training_data.mean(axis=0)\n",
    "std = x_training_data.std(axis=0)\n",
    "\n",
    "x_training_data = x_training_data - mean\n",
    "x_training_data /= std\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = build_dataset(x_training_data, y_training_data, heterodata, num_classes)\n",
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "# device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:1')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "learning_rate = 0.003"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### HeteroGNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeteroGNN(10, 1, 2).to(device)\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss = []\n",
    "train_accuracy = []\n",
    "test_loss = []\n",
    "test_accuracy = []\n",
    "best_test_accuracy = 0\n",
    "\n",
    "m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "for epoch in range(100):\n",
    "    print(f\"Epoch: {epoch+1}\")\n",
    "    \n",
    "    # TRAIN\n",
    "    model.train()\n",
    "    train_accuracy_epoch = []\n",
    "    train_loss_epoch = []\n",
    "    for d in train_loader:\n",
    "        \n",
    "        d = d.to(device)\n",
    "        out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict) \n",
    "        out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "        loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))\n",
    "        loss_result.backward()\n",
    "        train_loss_epoch.append(loss_result.detach().cpu())\n",
    "        \n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        output = m(out_zones)\n",
    "        train_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "    # if scheduler.get_last_lr()[0] > 0.0005:\n",
    "    if (epoch+1)%10 == 0:\n",
    "        scheduler.step()\n",
    "\n",
    "    train_accuracy.append(np.mean(train_accuracy_epoch))\n",
    "    train_loss.append(np.mean(train_loss_epoch))\n",
    "    \n",
    "    \n",
    "    \n",
    "    # VALIDATION\n",
    "    model.eval()\n",
    "    test_accuracy_epoch = []\n",
    "    test_loss_epoch = []\n",
    "    for d in test_loader:\n",
    "        \n",
    "        d = d.to(device)\n",
    "        out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "        out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "        loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))        \n",
    "        test_loss_epoch.append(loss_result.detach().cpu())\n",
    "        \n",
    "        output = m(out_zones)\n",
    "        test_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "    test_accuracy.append(np.mean(test_accuracy_epoch))\n",
    "    if test_accuracy[-1] > best_test_accuracy:\n",
    "        best_test_accuracy = test_accuracy[-1]\n",
    "        torch.save(model.state_dict(), \"UJI_HeteroGNN_best_model.pth\")\n",
    "        \n",
    "    test_loss.append(np.mean(test_loss_epoch))\n",
    "    \n",
    "    print(f\"    Train Loss {np.mean(train_loss_epoch)}, Val Loss {np.mean(test_loss_epoch)}\")\n",
    "    \n",
    "\n",
    "print(f\"Last LR: {scheduler.get_last_lr()}\")\n",
    "print(f\"Best Accuracy: Train {np.max(train_accuracy)}, Val {np.max(test_accuracy)}\")\n",
    "plt.figure()\n",
    "plt.plot(train_loss, label=\"Train loss\")\n",
    "plt.plot(test_loss, label=\"Validation loss\")\n",
    "plt.legend()\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(train_accuracy, label=\"Train accuracy\")\n",
    "plt.plot(test_accuracy, label=\"Validation accuracy\")\n",
    "plt.legend()\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = torch.nn.Softmax(dim=1)\n",
    "output = m(out_zones)\n",
    "accuracy = accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1)))\n",
    "\n",
    "print(accuracy)\n",
    "print(classification_report(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "\n",
    "df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "df_X_test = df_test[filtered_aps]\n",
    "df_y_test = df_test['CLASS']\n",
    "print(df_X_test.shape)\n",
    "\n",
    "df_X_test.values[df_X_test.values==100] = -105\n",
    "df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "df_X_test.describe()\n",
    "\n",
    "\n",
    "y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HeteroGNN(10, 1, 2)\n",
    "model.load_state_dict(torch.load(\"UJI_HeteroGNN_best_model.pth\"))\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loss = []\n",
    "test_accuracy = []\n",
    "\n",
    "m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "# TEST\n",
    "model.eval()\n",
    "for d in test_loader:\n",
    "\n",
    "    d = d.to(device)\n",
    "    out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "    out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "    output = m(out_zones)\n",
    "    test_accuracy.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "total_test_accuracy = np.mean(test_accuracy)\n",
    "print(f\"TEST ACCURACY: {total_test_accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = torch.nn.Softmax(dim=1)\n",
    "output = m(out_zones)\n",
    "accuracy = accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1)))\n",
    "\n",
    "print(accuracy)\n",
    "print(classification_report(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "neigh = KNeighborsClassifier(n_neighbors=5)\n",
    "neigh.fit(x_training_data[:,:,0], y_training_data.ravel())\n",
    "y_pred_knn = neigh.predict(x_test_data[:,:,0])\n",
    "\n",
    "print(accuracy_score(y_test_data, y_pred_knn))\n",
    "print(classification_report(y_test_data, y_pred_knn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "clf = MLPClassifier().fit(x_training_data[:,:,0], y_training_data.ravel())\n",
    "y_pred_fcnn = clf.predict(x_test_data[:,:,0])\n",
    "print(accuracy_score(y_test_data, y_pred_fcnn))\n",
    "print(classification_report(y_test_data, y_pred_fcnn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Bracco et al"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import auxiliary functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##\n",
    "# Data manipulation imports\n",
    "##\n",
    "import csv\n",
    "import time\n",
    "import numpy\n",
    "import copy\n",
    "import json\n",
    "import numpy\n",
    "import math\n",
    "###\n",
    "# Machine Learning imports\n",
    "##\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from fastprogress import master_bar, progress_bar\n",
    "from sklearn.metrics import precision_score\n",
    "\n",
    "##\n",
    "# Data viz libraries\n",
    "##\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def get_data(file, filter_macs=False, allowed_macs=[], to_zero=False, non_zero_macs=[]):\n",
    "    naming = {'from': {}, 'to': {}}\n",
    "    rows = []\n",
    "    rows_aux = []\n",
    "    naming_num = 0\n",
    "    header = []\n",
    "    row_length = len(allowed_macs) + 1\n",
    "    target_names_max = []\n",
    "    with open(file, 'r') as csvfile:\n",
    "        reader = csv.reader(csvfile, delimiter=',')\n",
    "\n",
    "        for i, row in enumerate(reader):\n",
    "            new_row = numpy.zeros(row_length)\n",
    "            used_index = 0\n",
    "            if i == 0:\n",
    "                header = row\n",
    "                index_acum = 1\n",
    "                index_acum_2 = 1\n",
    "                new_indexes = {}\n",
    "                zero_index = {}\n",
    "                for j, val in enumerate(row):\n",
    "                    if j == 0:\n",
    "                        continue\n",
    "                    if header[j].split('wifi-')[1] in allowed_macs:\n",
    "                        new_indexes[j] = index_acum\n",
    "                        index_acum += 1\n",
    "                    else:\n",
    "                        new_indexes[j] = None\n",
    "                    if header[j].split('wifi-')[1] in non_zero_macs:\n",
    "                        zero_index[j] = index_acum\n",
    "                        index_acum_2 += 1\n",
    "                    else:\n",
    "                        zero_index[j] = None\n",
    "\n",
    "            else:\n",
    "                for j, val in enumerate(row):\n",
    "                    if j == 0:\n",
    "                        # this is a name of the location\n",
    "                        lab = int(val.split('_')[1])\n",
    "                        if val not in naming['from']:\n",
    "                            naming['from'][val] = lab\n",
    "                            naming['to'][naming_num] = val\n",
    "                            target_names_max.append(val)\n",
    "                        new_row[0] = naming['from'][val]\n",
    "                        row[0] = naming['from'][val]\n",
    "                        continue\n",
    "                    index_to_use = new_indexes[j]\n",
    "                    idx = zero_index[j]\n",
    "                    if index_to_use is None and filter_macs:\n",
    "                        continue\n",
    "                    if val == '':\n",
    "                        new_row[index_to_use] = 0\n",
    "                        row[j] = 0\n",
    "                        continue\n",
    "                    try:\n",
    "                        if idx is None and to_zero:\n",
    "                            row[j] = 0\n",
    "                        else:\n",
    "                            float_value = float(val)\n",
    "                            new_row[index_to_use] = float_value\n",
    "                            row[j] = float_value\n",
    "                    except:\n",
    "                        print(\"problem parsing value \" + str(val))\n",
    "                if filter_macs:\n",
    "                    rows.append(new_row)\n",
    "                else:\n",
    "                    rows.append(row)\n",
    "    y = numpy.zeros(len(rows))\n",
    "    X = numpy.zeros((len(rows), len(rows[0])-1))\n",
    "\n",
    "    record_range = list(range(len(rows)))\n",
    "    for i in record_range:\n",
    "        y[i] = int(rows[i][0])\n",
    "        X[i, :] = numpy.array(rows[i][1:])\n",
    "    return X, y-1\n",
    "\n",
    "\n",
    "#####\n",
    "# Funcion Auxiliar para poder hacer un print lindo de los reportes de clasificacion\n",
    "#\n",
    "###\n",
    "\n",
    "def show_values(pc, fmt=\"%.2f\", **kw):\n",
    "    '''\n",
    "    Heatmap with text in each cell with matplotlib's pyplot\n",
    "    Source: https://stackoverflow.com/a/25074150/395857 \n",
    "    By HYRY\n",
    "    '''\n",
    "    pc.update_scalarmappable()\n",
    "    ax = pc.axes\n",
    "    for p, color, value in zip(pc.get_paths(), pc.get_facecolors(), pc.get_array()):\n",
    "        x, y = p.vertices[:-2, :].mean(0)\n",
    "        if numpy.all(color[:3] > 0.5):\n",
    "            color = (0.0, 0.0, 0.0)\n",
    "        else:\n",
    "            color = (1.0, 1.0, 1.0)\n",
    "        ax.text(x, y, fmt % value, ha=\"center\", va=\"center\", color=color, **kw)\n",
    "\n",
    "\n",
    "def cm2inch(*tupl):\n",
    "    '''\n",
    "    Specify figure size in centimeter in matplotlib\n",
    "    Source: https://stackoverflow.com/a/22787457/395857\n",
    "    By gns-ank\n",
    "    '''\n",
    "    inch = 2.54\n",
    "    if type(tupl[0]) == tuple:\n",
    "        return tuple(i/inch for i in tupl[0])\n",
    "    else:\n",
    "        return tuple(i/inch for i in tupl)\n",
    "\n",
    "\n",
    "def heatmap(AUC, title, xlabel, ylabel, xticklabels, yticklabels, figure_width=40, figure_height=20, correct_orientation=False, cmap='RdBu'):\n",
    "    '''\n",
    "    Inspired by:\n",
    "    - https://stackoverflow.com/a/16124677/395857 \n",
    "    - https://stackoverflow.com/a/25074150/395857\n",
    "    '''\n",
    "\n",
    "    # Plot it out\n",
    "    fig, ax = plt.subplots()    \n",
    "    #c = ax.pcolor(AUC, edgecolors='k', linestyle= 'dashed', linewidths=0.2, cmap='RdBu', vmin=0.0, vmax=1.0)\n",
    "    c = ax.pcolor(AUC, edgecolors='k', linestyle= 'dashed', linewidths=0.2, cmap=cmap)\n",
    "\n",
    "    # put the major ticks at the middle of each cell\n",
    "    ax.set_yticks(numpy.arange(AUC.shape[0]) + 0.5, minor=False)\n",
    "    ax.set_xticks(numpy.arange(AUC.shape[1]) + 0.5, minor=False)\n",
    "\n",
    "    # set tick labels\n",
    "    #ax.set_xticklabels(np.arange(1,AUC.shape[1]+1), minor=False)\n",
    "    ax.set_xticklabels(xticklabels, minor=False)\n",
    "    ax.set_yticklabels(yticklabels, minor=False)\n",
    "\n",
    "    # set title and x/y labels\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(ylabel)      \n",
    "\n",
    "    # Remove last blank column\n",
    "    plt.xlim( (0, AUC.shape[1]) )\n",
    "\n",
    "    # Turn off all the ticks\n",
    "    ax = plt.gca()    \n",
    "    for t in ax.xaxis.get_major_ticks():\n",
    "        t.tick1On = False\n",
    "        t.tick2On = False\n",
    "    for t in ax.yaxis.get_major_ticks():\n",
    "        t.tick1On = False\n",
    "        t.tick2On = False\n",
    "\n",
    "    # Add color bar\n",
    "    plt.colorbar(c)\n",
    "    plt.rcParams.update({'font.size': 14})\n",
    "\n",
    "\n",
    "    # Add text in each cell \n",
    "    show_values(c)\n",
    "\n",
    "    # Proper orientation (origin at the top left instead of bottom left)\n",
    "    if correct_orientation:\n",
    "        ax.invert_yaxis()\n",
    "        ax.xaxis.tick_top()       \n",
    "\n",
    "    # resize \n",
    "    fig = plt.gcf()\n",
    "    fig.set_size_inches(cm2inch(figure_width, figure_height))\n",
    "    fig.savefig(f'{title}.svg', format=\"svg\")\n",
    "\n",
    "\n",
    "\n",
    "def plot_classification_report(classification_report, title, cmap='RdBu'):\n",
    "    '''\n",
    "    Plot scikit-learn classification report.\n",
    "    Extension based on https://stackoverflow.com/a/31689645/395857 \n",
    "    '''\n",
    "    lines = classification_report.split('\\n')\n",
    "\n",
    "    classes = []\n",
    "    plotMat = []\n",
    "    support = []\n",
    "    class_names = []\n",
    "    for line in lines[2 : (len(lines) - 4)]:\n",
    "        t = line.strip().split()\n",
    "        if len(t) < 2: continue\n",
    "        classes.append(t[0])\n",
    "        v = []\n",
    "        for x in t[1: len(t) - 2]:\n",
    "            if t[1] != 'avg':\n",
    "                v.append(float(x))\n",
    "        support.append(int(t[-1]))\n",
    "        class_names.append(t[0])\n",
    "        plotMat.append(v)\n",
    "\n",
    "    xlabel = 'Métricas'\n",
    "    ylabel = 'Zonas'\n",
    "    xticklabels = ['Precisión', 'Exhaustividad']\n",
    "    yticklabels = ['{0} ({1})'.format(class_names[idx], sup) for idx, sup  in enumerate(support)]\n",
    "    figure_width = 25\n",
    "    figure_height = len(class_names) + 7\n",
    "    correct_orientation = False\n",
    "    heatmap(numpy.array(plotMat), title, xlabel, ylabel, xticklabels, yticklabels, figure_width, figure_height, correct_orientation, cmap=cmap)\n",
    "\n",
    "## Funcion Auxiliar\n",
    "def youden_statistic(y_actual, y_hat):\n",
    "    TP = 0\n",
    "    FP = 0\n",
    "    TN = 0\n",
    "    FN = 0\n",
    "    for i in range(len(y_hat)): \n",
    "        if y_actual[i]==y_hat[i]==1:\n",
    "           TP += 1\n",
    "        if y_hat[i]==1 and y_actual[i]!=y_hat[i]:\n",
    "           FP += 1\n",
    "        if y_actual[i]==y_hat[i]==0:\n",
    "           TN += 1\n",
    "        if y_hat[i]==0 and y_actual[i]!=y_hat[i]:\n",
    "           FN += 1\n",
    "    sensitivity = 0\n",
    "    if TP + FN != 0:\n",
    "        sensitivity = TP / (TP + FN)\n",
    "    specificity = 0\n",
    "    if TN + FP != 0:\n",
    "        specificity = TN / (TN + FP)\n",
    "    return specificity + sensitivity - 1\n",
    "\n",
    "\n",
    "\n",
    "def split_train_test_val(X,y):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2,  random_state=1)\n",
    "    \n",
    "    return X_train, X_test, X_val, y_train, y_test, y_val\n",
    "\n",
    "def train_algorithms(X_train, y_train, X_test, y_test, names, classifiers):\n",
    "    algorithms = {}\n",
    "\n",
    "    for name, clf in zip(names, classifiers):\n",
    "        try:\n",
    "            algorithms[name] =  clf.fit(X_train,y_train)\n",
    "        except Exception as e:\n",
    "            print(str(e))\n",
    "\n",
    "    results = {}\n",
    "\n",
    "    for alg in algorithms.keys():\n",
    "        results[alg] = algorithms[alg].predict(X_test)\n",
    "\n",
    "    youden = {}\n",
    "\n",
    "    for key in results.keys():\n",
    "        youden[key] = youden_statistic(y_test,results[key])\n",
    "    \n",
    "    return algorithms, youden\n",
    "\n",
    "def classify(algorithms, youden, X_val):\n",
    "    probabilities = {}\n",
    "\n",
    "    for alg in algorithms.keys():\n",
    "        probabilities[alg] = algorithms[alg].predict_proba(X_val)\n",
    "\n",
    "\n",
    "    shape = numpy.shape(probabilities['Nearest Neighbors'])\n",
    "\n",
    "    probs = numpy.zeros(shape)\n",
    "\n",
    "    for key in algorithms.keys():\n",
    "        probs = probs + numpy.matrix(probabilities[key])*youden[key]\n",
    "\n",
    "    y_final = numpy.argmax(probs, axis=1)\n",
    "\n",
    "    y_final = numpy.asarray(y_final).reshape(-1)\n",
    "    \n",
    "    return y_final\n",
    "\n",
    "\n",
    "def save_cm(y_val, y_final, matrix_img_name, target_names):\n",
    "    cm = confusion_matrix(y_val, y_final)\n",
    "    fig, ax = plt.subplots(figsize=(15,15))         # Sample figsize\n",
    "    heat_map = sns.heatmap(cm, annot=True, annot_kws={\"size\": 12}, fmt=\"d\", linewidths=.2,ax=ax,xticklabels=target_names, yticklabels=target_names)\n",
    "    fig = heat_map.get_figure()\n",
    "    plt.rcParams.update({'font.size': 18})\n",
    "    fig.savefig(f'{matrix_img_name}.svg', format=\"svg\")\n",
    "\n",
    "def report_classify(y_val, y_final, matrix_img_name, target_names):\n",
    "    prec = precision_score(y_val, y_final, average='macro')\n",
    "    class_rep = classification_report(y_val, y_final, target_names=target_names)\n",
    "    plot_classification_report(class_rep, matrix_img_name + '_class')\n",
    "    return prec\n",
    "\n",
    "\n",
    "def subsample(X, y, percentage):\n",
    "    X_subsample = numpy.zeros((0, 188))\n",
    "    y_subsample = []\n",
    "    unique, counts = numpy.unique(y, return_counts=True)\n",
    "    count_dict = dict(zip(unique, counts))\n",
    "\n",
    "    for i in range(0,16):\n",
    "        count = math.floor(count_dict[i]* percentage)\n",
    "        X_=X[y==i]\n",
    "        y_=y[y==i]\n",
    "        X_subsample = numpy.vstack([X_subsample,X_[:count]])\n",
    "        y_subsample = numpy.concatenate((y_subsample,y_[:count]), axis=None)\n",
    "    return X_subsample, y_subsample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names= [\"Nearest Neighbors\",\"Decision Tree\",\"Linear SVM\",\"Random Forest\",\"Neural Net\",\"AdaBoost\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3)\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_training_data = y_train\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "mean = x_training_data.mean(axis=0)\n",
    "std = x_training_data.std(axis=0)\n",
    "\n",
    "x_training_data = x_training_data - mean\n",
    "x_training_data /= std\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std\n",
    "\n",
    "\n",
    "classifiers = [KNeighborsClassifier(3),DecisionTreeClassifier(max_depth=5),SVC(kernel=\"linear\", C=0.025, probability=True), DecisionTreeClassifier(max_depth=5),RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),MLPClassifier(alpha=1, max_iter=1000),\n",
    "    AdaBoostClassifier()]\n",
    "\n",
    "algorithms, youden = train_algorithms(x_training_data[:,:,0], y_training_data.ravel(), x_test_data[:,:,0], y_test_data.ravel(), names, classifiers)\n",
    "\n",
    "y_final = classify(algorithms, youden, x_test_data[:,:,0])\n",
    "\n",
    "ACC_test = accuracy_score(y_test_data.ravel(), y_final)        \n",
    "print(ACC_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "df.head()\n",
    "\n",
    "df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "df_X_test = df_test[filtered_aps]\n",
    "df_y_test = df_test['CLASS']\n",
    "print(df_X_test.shape)\n",
    "\n",
    "df_X_test.values[df_X_test.values==100] = -105\n",
    "df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "df_X_test.describe()\n",
    "\n",
    "\n",
    "y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std\n",
    "\n",
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "### TEST BRACCO et al ####\n",
    "y_final = classify(algorithms, youden, X_test)\n",
    "ACC_test = accuracy_score(y_test, y_final)        \n",
    "print(ACC_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Análisis variando cantidad de muestras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### HeteroGNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "batch_size = 32\n",
    "learning_rate = 0.003\n",
    "print_every = 5\n",
    "\n",
    "porcentajes = [0.9, 1] # [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    print('Porcentaje de datos: ', porc)\n",
    "    \n",
    "    for i in range(5):    \n",
    "\n",
    "        # Split de los datos y armado del objeto Data con el grafo\n",
    "\n",
    "        X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3, dataset_percentage=porc)\n",
    "        ap_edge_index, ap_edge_attr = ap_graph_creator(X_train[:,:-1], th=10)\n",
    "        zone_ap_edge_index, zone_ap_edge_attr = zone_ap_graph_creator(X_train[:,:-1], y_train)\n",
    "        heterodata = build_heterodata(ap_edge_index, ap_edge_attr, zone_ap_edge_index, zone_ap_edge_attr, num_classes, num_aps)\n",
    "        T.ToUndirected()(heterodata)\n",
    "\n",
    "        # Armado del dataset\n",
    "\n",
    "        x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "        x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "        y_training_data = y_train\n",
    "        y_test_data = y_test\n",
    "\n",
    "        #normalize (x-mean)/std\n",
    "        mean = x_training_data.mean(axis=0)\n",
    "        std = x_training_data.std(axis=0)\n",
    "\n",
    "        x_training_data = x_training_data - mean\n",
    "        x_training_data /= std\n",
    "        x_test_data = x_test_data - mean\n",
    "        x_test_data /= std\n",
    "\n",
    "        train_dataset = build_dataset(x_training_data, y_training_data, heterodata, num_classes)\n",
    "        test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "\n",
    "        model = HeteroGNN(10, 1, 2).to(device)\n",
    "\n",
    "        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\n",
    "        loss = torch.nn.CrossEntropyLoss()\n",
    "        scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n",
    "\n",
    "        train_loss = []\n",
    "        train_accuracy = []\n",
    "        test_loss = []\n",
    "        test_accuracy = []\n",
    "        best_test_accuracy = 0\n",
    "\n",
    "        m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "        for epoch in range(100):\n",
    "            # print(f\"Epoch: {epoch+1}\")\n",
    "\n",
    "            # TRAIN\n",
    "            model.train()\n",
    "            train_accuracy_epoch = []\n",
    "            train_loss_epoch = []\n",
    "            for d in train_loader:\n",
    "\n",
    "                d = d.to(device)\n",
    "                out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict) \n",
    "                out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "                loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))\n",
    "                loss_result.backward()\n",
    "                train_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                output = m(out_zones)\n",
    "                train_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "            # if scheduler.get_last_lr()[0] > 0.0005:\n",
    "            if (epoch+1)%10 == 0:\n",
    "                scheduler.step()\n",
    "\n",
    "            train_accuracy.append(np.mean(train_accuracy_epoch))\n",
    "            train_loss.append(np.mean(train_loss_epoch))\n",
    "\n",
    "\n",
    "\n",
    "            # VALIDATION\n",
    "            model.eval()\n",
    "            test_accuracy_epoch = []\n",
    "            test_loss_epoch = []\n",
    "            for d in test_loader:\n",
    "\n",
    "                d = d.to(device)\n",
    "                out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "                out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "                loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))        \n",
    "                test_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "                output = m(out_zones)\n",
    "                test_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "            test_accuracy.append(np.mean(test_accuracy_epoch))\n",
    "            if test_accuracy[-1] > best_test_accuracy:\n",
    "                best_test_accuracy = test_accuracy[-1]\n",
    "                torch.save(model.state_dict(), f\"UJI_HeteroGNN_porc{porc}_{i}_best_model.pth\")\n",
    "\n",
    "            test_loss.append(np.mean(test_loss_epoch))\n",
    "\n",
    "        print(f\"Best Accuracy: Train {np.max(train_accuracy)}, Val {np.max(test_accuracy)}\")\n",
    "        accuracy[str(porc)].append(np.max(test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for porc in porcentajes:\n",
    "    print(f\"{porc}: {np.mean(accuracy[str(porc)])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "df.head()\n",
    "\n",
    "df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "df_X_test = df_test[filtered_aps]\n",
    "df_y_test = df_test['CLASS']\n",
    "print(df_X_test.shape)\n",
    "\n",
    "df_X_test.values[df_X_test.values==100] = -105\n",
    "df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "df_X_test.describe()\n",
    "\n",
    "\n",
    "y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std\n",
    "\n",
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "porcentajes = [0.9, 1] # [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "# accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    for i in range(5):\n",
    "        model = HeteroGNN(10, 1, 2)\n",
    "        model.load_state_dict(torch.load(f\"UJI_HeteroGNN_porc{porc}_{i}_best_model.pth\"))\n",
    "        model.to(device)\n",
    "        \n",
    "        test_loss = []\n",
    "        test_accuracy = []\n",
    "\n",
    "        m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "        # TEST\n",
    "        model.eval()\n",
    "        for d in test_loader:\n",
    "\n",
    "            d = d.to(device)\n",
    "            out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "            out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "            output = m(out_zones)\n",
    "            test_accuracy.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "        total_test_accuracy = np.mean(test_accuracy)\n",
    "        accuracy[str(porc)].append(total_test_accuracy) \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = []\n",
    "aux.append(accuracy['0.3'])\n",
    "aux.append(accuracy['0.4'])\n",
    "aux.append(accuracy['0.5'])\n",
    "aux.append(accuracy['0.6'])\n",
    "aux.append(accuracy['0.7'])\n",
    "aux.append(accuracy['0.8'])\n",
    "aux.append(accuracy['0.9'])\n",
    "aux.append(accuracy['1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[9,7])\n",
    "plt.boxplot(aux, showfliers=False) #, meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5, 6, 7, 8], [30, 40, 50, 60, 70, 80, 90, 100])\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Fingerprints sample size (%)')\n",
    "plt.grid()\n",
    "plt.savefig('UJIINDOORLOC_hetero_cant_muestras.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### HeteroGNNSimplified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "batch_size = 32\n",
    "learning_rate = 0.003\n",
    "print_every = 5\n",
    "\n",
    "porcentajes = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    print('Porcentaje de datos: ', porc)\n",
    "        \n",
    "    for i in range(5):    \n",
    "\n",
    "        # Split de los datos y armado del objeto Data con el grafo\n",
    "\n",
    "        X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3, dataset_percentage=porc)\n",
    "        ap_edge_index, ap_edge_attr = ap_graph_creator(X_train[:,:-1], th=10)\n",
    "        zone_ap_edge_index, zone_ap_edge_attr = zone_ap_graph_creator(X_train[:,:-1], y_train)\n",
    "        heterodata = build_heterodata(ap_edge_index, ap_edge_attr, zone_ap_edge_index, zone_ap_edge_attr, num_classes, num_aps)\n",
    "        T.ToUndirected()(heterodata)\n",
    "\n",
    "        # Armado del dataset\n",
    "\n",
    "        x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "        x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "        y_training_data = y_train\n",
    "        y_test_data = y_test\n",
    "\n",
    "        #normalize (x-mean)/std\n",
    "        mean = x_training_data.mean(axis=0)\n",
    "        std = x_training_data.std(axis=0)\n",
    "\n",
    "        x_training_data = x_training_data - mean\n",
    "        x_training_data /= std\n",
    "        x_test_data = x_test_data - mean\n",
    "        x_test_data /= std\n",
    "\n",
    "        train_dataset = build_dataset(x_training_data, y_training_data, heterodata, num_classes)\n",
    "        test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)        \n",
    "        \n",
    "        model = HeteroGNN_simplified(10, 1, 2).to(device)\n",
    "\n",
    "        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\n",
    "        loss = torch.nn.CrossEntropyLoss()\n",
    "        scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n",
    "\n",
    "        train_loss = []\n",
    "        train_accuracy = []\n",
    "        test_loss = []\n",
    "        test_accuracy = []\n",
    "        best_test_accuracy = 0\n",
    "\n",
    "        m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "        for epoch in range(100):\n",
    "            # print(f\"Epoch: {epoch+1}\")\n",
    "\n",
    "            # TRAIN\n",
    "            model.train()\n",
    "            train_accuracy_epoch = []\n",
    "            train_loss_epoch = []\n",
    "            for d in train_loader:\n",
    "\n",
    "                d = d.to(device)\n",
    "                out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict) \n",
    "                out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "                loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))\n",
    "                loss_result.backward()\n",
    "                train_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                output = m(out_zones)\n",
    "                train_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "            # if scheduler.get_last_lr()[0] > 0.0005:\n",
    "            if (epoch+1)%10 == 0:\n",
    "                scheduler.step()\n",
    "\n",
    "            train_accuracy.append(np.mean(train_accuracy_epoch))\n",
    "            train_loss.append(np.mean(train_loss_epoch))\n",
    "\n",
    "\n",
    "\n",
    "            # VALIDATION\n",
    "            model.eval()\n",
    "            test_accuracy_epoch = []\n",
    "            test_loss_epoch = []\n",
    "            for d in test_loader:\n",
    "\n",
    "                d = d.to(device)\n",
    "                out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "                out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "                loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))        \n",
    "                test_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "                output = m(out_zones)\n",
    "                test_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "            test_accuracy.append(np.mean(test_accuracy_epoch))\n",
    "            if test_accuracy[-1] > best_test_accuracy:\n",
    "                best_test_accuracy = test_accuracy[-1]\n",
    "                torch.save(model.state_dict(), f\"UJI_HeteroGNNSimplified_porc{porc}_{i}_best_model.pth\")\n",
    "\n",
    "            test_loss.append(np.mean(test_loss_epoch))\n",
    "\n",
    "        print(f\"Best Accuracy: Train {np.max(train_accuracy)}, Val {np.max(test_accuracy)}\")\n",
    "        accuracy[str(porc)].append(np.max(test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "df.head()\n",
    "\n",
    "df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "df_X_test = df_test[filtered_aps]\n",
    "df_y_test = df_test['CLASS']\n",
    "print(df_X_test.shape)\n",
    "\n",
    "df_X_test.values[df_X_test.values==100] = -105\n",
    "df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "df_X_test.describe()\n",
    "\n",
    "\n",
    "y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std\n",
    "\n",
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "porcentajes = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "# accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    for i in range(5):\n",
    "        model = HeteroGNN_simplified(10, 1, 2)\n",
    "        model.load_state_dict(torch.load(f\"UJI_HeteroGNNSimplified_porc{porc}_{i}_best_model.pth\"))\n",
    "        model.to(device)\n",
    "        \n",
    "        test_loss = []\n",
    "        test_accuracy = []\n",
    "\n",
    "        m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "        # TEST\n",
    "        model.eval()\n",
    "        for d in test_loader:\n",
    "\n",
    "            d = d.to(device)\n",
    "            out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "            out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "            output = m(out_zones)\n",
    "            test_accuracy.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "        total_test_accuracy = np.mean(test_accuracy)\n",
    "        accuracy[str(porc)].append(total_test_accuracy)\n",
    "        \n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = []\n",
    "aux.append(accuracy['0.3'])\n",
    "aux.append(accuracy['0.4'])\n",
    "aux.append(accuracy['0.5'])\n",
    "aux.append(accuracy['0.6'])\n",
    "aux.append(accuracy['0.7'])\n",
    "aux.append(accuracy['0.8'])\n",
    "aux.append(accuracy['0.9'])\n",
    "aux.append(accuracy['1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[9,7])\n",
    "plt.boxplot(aux, showfliers=False) #, meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5, 6, 7, 8], [30, 40, 50, 60, 70, 80, 90, 100])\n",
    "plt.yticks([0.95, 0.90, 0.85, 0.80, 0.75])\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Fingerprints sample size (%)')\n",
    "plt.grid()\n",
    "# plt.savefig('UJIINDOORLOC_hetero_cant_muestras.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device = torch.device('cuda:1')\n",
    "\n",
    "batch_size = 32\n",
    "learning_rate = 0.003\n",
    "print_every = 5\n",
    "\n",
    "porcentajes = [0.8, 0.9, 1] #[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "# accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    print('Porcentaje de datos: ', porc)\n",
    "    \n",
    "    for i in range(10):    \n",
    "\n",
    "        # Split de los datos y armado del objeto Data con el grafo\n",
    "\n",
    "        X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3, dataset_percentage=porc)\n",
    "\n",
    "        # Armado del dataset\n",
    "\n",
    "        x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "        x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "        y_training_data = y_train\n",
    "        y_test_data = y_test\n",
    "\n",
    "        #normalize (x-mean)/std\n",
    "        mean = x_training_data.mean(axis=0)\n",
    "        std = x_training_data.std(axis=0)\n",
    "\n",
    "        x_training_data = x_training_data - mean\n",
    "        x_training_data /= std\n",
    "        x_test_data = x_test_data - mean\n",
    "        x_test_data /= std\n",
    "\n",
    "        train_dataset = build_dataset(x_training_data, y_training_data, heterodata, num_classes)\n",
    "        test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)        \n",
    "\n",
    "        \n",
    "        #### FIT KNN ####\n",
    "        neigh = KNeighborsClassifier(n_neighbors=5)\n",
    "        neigh.fit(x_training_data[:,:,0], y_training_data.ravel())\n",
    "        \n",
    "        #################\n",
    "        \n",
    "        dataset = 'UjiIndoorLoc.zip'\n",
    "        zip_file = ZipFile(dataset)\n",
    "        df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "        df.head()\n",
    "\n",
    "        df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "        df_X_test = df_test[filtered_aps]\n",
    "        df_y_test = df_test['CLASS']\n",
    "        print(df_X_test.shape)\n",
    "\n",
    "        df_X_test.values[df_X_test.values==100] = -105\n",
    "        df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "        df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "        df_X_test.describe()\n",
    "\n",
    "\n",
    "        y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "        X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "        # Armado del dataset\n",
    "\n",
    "        x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "        y_test_data = y_test\n",
    "\n",
    "        #normalize (x-mean)/std\n",
    "\n",
    "        x_test_data = x_test_data - mean\n",
    "        x_test_data /= std\n",
    "\n",
    "        test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "        #### KNN PREDICT ####\n",
    "        y_pred_knn = neigh.predict(x_test_data[:,:,0])\n",
    "        acc = accuracy_score(y_test_data, y_pred_knn)\n",
    "        accuracy[str(porc)].append(acc) \n",
    "        print(acc)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Prunning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "batch_size = 32\n",
    "learning_rate = 0.003\n",
    "print_every = 5\n",
    "\n",
    "porcentajes = [0.3] # [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    print('Porcentaje de datos: ', porc)\n",
    "        \n",
    "    for i in range(5):    \n",
    "        # Split de los datos y armado del objeto Data con el grafo\n",
    "\n",
    "        X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3, dataset_percentage=porc)\n",
    "        ap_edge_index, ap_edge_attr = ap_graph_creator(X_train[:,:-1], th=10, prune_th=10)\n",
    "        zone_ap_edge_index, zone_ap_edge_attr = zone_ap_graph_creator(X_train[:,:-1], y_train, prune_th=10)\n",
    "        heterodata = build_heterodata(ap_edge_index, ap_edge_attr, zone_ap_edge_index, zone_ap_edge_attr, num_classes, num_aps)\n",
    "        T.ToUndirected()(heterodata)\n",
    "\n",
    "        # Armado del dataset\n",
    "\n",
    "        x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "        x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "        y_training_data = y_train\n",
    "        y_test_data = y_test\n",
    "\n",
    "        #normalize (x-mean)/std\n",
    "        mean = x_training_data.mean(axis=0)\n",
    "        std = x_training_data.std(axis=0)\n",
    "\n",
    "        x_training_data = x_training_data - mean\n",
    "        x_training_data /= std\n",
    "        x_test_data = x_test_data - mean\n",
    "        x_test_data /= std\n",
    "\n",
    "        train_dataset = build_dataset(x_training_data, y_training_data, heterodata, num_classes)\n",
    "        test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "    \n",
    "        model = HeteroGNN(10, 1, 2).to(device)\n",
    "\n",
    "        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "        optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\n",
    "        loss = torch.nn.CrossEntropyLoss()\n",
    "        scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n",
    "\n",
    "        train_loss = []\n",
    "        train_accuracy = []\n",
    "        test_loss = []\n",
    "        test_accuracy = []\n",
    "        best_test_accuracy = 0\n",
    "\n",
    "        m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "        for epoch in range(100):\n",
    "            # print(f\"Epoch: {epoch+1}\")\n",
    "\n",
    "            # TRAIN\n",
    "            model.train()\n",
    "            train_accuracy_epoch = []\n",
    "            train_loss_epoch = []\n",
    "            for d in train_loader:\n",
    "\n",
    "                d = d.to(device)\n",
    "                out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict) \n",
    "                out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "                loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))\n",
    "                loss_result.backward()\n",
    "                train_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                output = m(out_zones)\n",
    "                train_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "            # if scheduler.get_last_lr()[0] > 0.0005:\n",
    "            if (epoch+1)%10 == 0:\n",
    "                scheduler.step()\n",
    "\n",
    "            train_accuracy.append(np.mean(train_accuracy_epoch))\n",
    "            train_loss.append(np.mean(train_loss_epoch))\n",
    "\n",
    "\n",
    "\n",
    "            # VALIDATION\n",
    "            model.eval()\n",
    "            test_accuracy_epoch = []\n",
    "            test_loss_epoch = []\n",
    "            for d in test_loader:\n",
    "\n",
    "                d = d.to(device)\n",
    "                out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "                out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "                loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))        \n",
    "                test_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "                output = m(out_zones)\n",
    "                test_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "            test_accuracy.append(np.mean(test_accuracy_epoch))\n",
    "            if test_accuracy[-1] > best_test_accuracy:\n",
    "                best_test_accuracy = test_accuracy[-1]\n",
    "                torch.save(model.state_dict(), f\"UJI_HeteroGNN_porc{porc}_{i}_best_model_prunning.pth\")\n",
    "\n",
    "            test_loss.append(np.mean(test_loss_epoch))\n",
    "\n",
    "        print(f\"Best Accuracy: Train {np.max(train_accuracy)}, Val {np.max(test_accuracy)}\")\n",
    "        accuracy[str(porc)].append(np.max(test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "df.head()\n",
    "\n",
    "df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "df_X_test = df_test[filtered_aps]\n",
    "df_y_test = df_test['CLASS']\n",
    "print(df_X_test.shape)\n",
    "\n",
    "df_X_test.values[df_X_test.values==100] = -105\n",
    "df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "df_X_test.describe()\n",
    "\n",
    "\n",
    "y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std\n",
    "\n",
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "porcentajes = [0.3] # [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]\n",
    "# accuracy = {\"0.3\":[], \"0.4\":[], \"0.5\":[], \"0.6\":[], \"0.7\":[], \"0.8\":[], \"0.9\":[], \"1\":[]}\n",
    "\n",
    "for porc in porcentajes:\n",
    "    for i in range(5):\n",
    "        model = HeteroGNN(10, 1, 2)\n",
    "        model.load_state_dict(torch.load(f\"UJI_HeteroGNN_porc{porc}_{i}_best_model_prunning.pth\"))\n",
    "        model.to(device)\n",
    "        \n",
    "        test_loss = []\n",
    "        test_accuracy = []\n",
    "\n",
    "        m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "        # TEST\n",
    "        model.eval()\n",
    "        for d in test_loader:\n",
    "\n",
    "            d = d.to(device)\n",
    "            out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "            out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "            output = m(out_zones)\n",
    "            test_accuracy.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "        total_test_accuracy = np.mean(test_accuracy)\n",
    "        accuracy[str(porc)].append(total_test_accuracy) \n",
    "print(accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = {'0.3': [0.820786943319838, 0.8063638663967612, 0.8150303643724697, 0.817877024291498, 0.8158527327935223, 0.7353649068322982, 0.6814440993788821, 0.7112577639751553, 0.6951863354037267, 0.7273291925465838], '0.4': [], '0.5': [], '0.6': [], '0.7': [], '0.8': [], '0.9': [], '1': []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aux = []\n",
    "# aux.append(accuracy_cant_muestras_mnav_hetero['0.1'])\n",
    "aux.append(accuracy['0.3'])\n",
    "aux.append(accuracy['0.4'])\n",
    "aux.append(accuracy['0.5'])\n",
    "aux.append(accuracy['0.6'])\n",
    "aux.append(accuracy['0.7'])\n",
    "aux.append(accuracy['0.8'])\n",
    "aux.append(accuracy['0.9'])\n",
    "aux.append(accuracy['1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=[9,7])\n",
    "plt.boxplot(aux, showfliers=False) #, meanline=True, showmeans=True)\n",
    "plt.xticks([1], [30])\n",
    "plt.yticks([0.95, 0.90, 0.85, 0.80, 0.75])\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Fingerprints sample size (%)')\n",
    "plt.title('HeteroGNN prune_th=10')\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agregado de zona"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "batch_size = 32\n",
    "learning_rate = 0.003\n",
    "print_every = 50\n",
    "\n",
    "zones_to_remove = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] #, 13, 14, 15]\n",
    "accuracy = {\"0\":[],\"1\":[],\"2\":[],\"3\":[],\"4\":[],\"5\":[],\"6\":[],\"7\":[],\"8\":[],\"9\":[],\"10\":[],\"11\":[],\"12\":[]} #,\"13\":[],\"14\":[],\"15\":[],}\n",
    "\n",
    "for zone_to_remove in zones_to_remove:\n",
    "    print('Zone removed: ', zone_to_remove)\n",
    "    \n",
    "    # Split de los datos y armado del objeto Data con el grafo\n",
    "    X_train, X_test, y_train, y_test, num_aps, num_classes, filtered_aps, enc_train = preprocess_dataset(dataset, filter_std=3)\n",
    "    ap_edge_index, ap_edge_attr = ap_graph_creator(X_train[:,:-1], th=10)\n",
    "    zone_ap_edge_index, zone_ap_edge_attr = zone_ap_graph_creator(X_train[:,:-1], y_train, zone_to_remove=zone_to_remove)\n",
    "    heterodata = build_heterodata(ap_edge_index, ap_edge_attr, zone_ap_edge_index, zone_ap_edge_attr, num_classes, num_aps)\n",
    "    T.ToUndirected()(heterodata)\n",
    "    \n",
    "    # Armado del dataset\n",
    "\n",
    "    x_training_data = np.reshape(X_train,(X_train.shape[0],num_aps,1))\n",
    "    x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "    y_training_data = y_train\n",
    "    y_test_data = y_test\n",
    "\n",
    "    #normalize (x-mean)/std\n",
    "    mean = x_training_data.mean(axis=0)\n",
    "    std = x_training_data.std(axis=0)\n",
    "\n",
    "    x_training_data = x_training_data - mean\n",
    "    x_training_data /= std\n",
    "    x_test_data = x_test_data - mean\n",
    "    x_test_data /= std    \n",
    "\n",
    "    train_dataset = build_dataset(x_training_data, y_training_data, heterodata, num_classes, zone_to_remove=zone_to_remove)\n",
    "    test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes, zone_to_remove=zone_to_remove)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    model = HeteroGNN_simplified(10, 1, 2).to(device)\n",
    "\n",
    "    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=5e-4)\n",
    "    loss = torch.nn.CrossEntropyLoss()\n",
    "    scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.8)\n",
    "\n",
    "    train_loss = []\n",
    "    train_accuracy = []\n",
    "    test_loss = []\n",
    "    test_accuracy = []\n",
    "    best_test_accuracy = 0\n",
    "\n",
    "    m = torch.nn.Softmax(dim=1)\n",
    "\n",
    "    for epoch in range(100):\n",
    "        # print(f\"Epoch: {epoch+1}\")\n",
    "\n",
    "        # TRAIN\n",
    "        model.train()\n",
    "        train_accuracy_epoch = []\n",
    "        train_loss_epoch = []\n",
    "        for d in train_loader:\n",
    "\n",
    "            d = d.to(device)\n",
    "            out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict) \n",
    "            out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "            loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))\n",
    "            loss_result.backward()\n",
    "            train_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            output = m(out_zones)\n",
    "            train_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "        # if scheduler.get_last_lr()[0] > 0.0005:\n",
    "        if (epoch+1)%10 == 0:\n",
    "            scheduler.step()\n",
    "\n",
    "        train_accuracy.append(np.mean(train_accuracy_epoch))\n",
    "        train_loss.append(np.mean(train_loss_epoch))\n",
    "\n",
    "\n",
    "\n",
    "        # VALIDATION\n",
    "        model.eval()\n",
    "        test_accuracy_epoch = []\n",
    "        test_loss_epoch = []\n",
    "        for d in test_loader:\n",
    "\n",
    "            d = d.to(device)\n",
    "            out = model(d.x_dict, d.edge_index_dict, d.edge_attr_dict)\n",
    "            out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "            loss_result = loss(out_zones.cpu(), d[\"zones\"].y.cpu().type(torch.long))        \n",
    "            test_loss_epoch.append(loss_result.detach().cpu())\n",
    "\n",
    "            output = m(out_zones)\n",
    "            test_accuracy_epoch.append(accuracy_score(d[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "\n",
    "        test_accuracy.append(np.mean(test_accuracy_epoch))\n",
    "        if test_accuracy[-1] > best_test_accuracy:\n",
    "            best_test_accuracy = test_accuracy[-1]\n",
    "            torch.save(model.state_dict(), f\"UJI_HeteroGNNSimplified_without_{zone_to_remove}_best_model.pth\")\n",
    "\n",
    "        test_loss.append(np.mean(test_loss_epoch))\n",
    "\n",
    "    print(f\"Best Accuracy: Train {np.max(train_accuracy)}, Val {np.max(test_accuracy)}\")\n",
    "    accuracy[str(zone_to_remove)].append(np.max(test_accuracy))    \n",
    "print(accuracy)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'UjiIndoorLoc.zip'\n",
    "zip_file = ZipFile(dataset)\n",
    "df_test = pd.read_csv(zip_file.open('ValidationData.csv'))\n",
    "\n",
    "df_test['CLASS'] = df_test['BUILDINGID'].astype(str) + df_test['FLOOR'].astype(str)\n",
    "\n",
    "df_X_test = df_test[filtered_aps]\n",
    "df_y_test = df_test['CLASS']\n",
    "print(df_X_test.shape)\n",
    "\n",
    "df_X_test.values[df_X_test.values==100] = -105\n",
    "df_X_test.iloc[:,:] = 105 + df_X_test.values\n",
    "df_X_test['CLASS'] = df_y_test.values \n",
    "\n",
    "df_X_test.describe()\n",
    "\n",
    "\n",
    "y_test = enc_train.transform(df_X_test['CLASS'].values.reshape(-1,1))\n",
    "X_test = df_X_test.iloc[:,:-1].values\n",
    "\n",
    "\n",
    "# Armado del dataset\n",
    "\n",
    "x_test_data = np.reshape(X_test,(X_test.shape[0],num_aps,1))\n",
    "y_test_data = y_test\n",
    "\n",
    "#normalize (x-mean)/std\n",
    "\n",
    "x_test_data = x_test_data - mean\n",
    "x_test_data /= std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = build_dataset(x_test_data, y_test_data, heterodata, num_classes)\n",
    "# test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for zone_to_remove in zones_to_remove:\n",
    "    print('Zone removed: ', zone_to_remove)\n",
    "    \n",
    "    model = HeteroGNN_simplified(10, 1, 2)\n",
    "    model.load_state_dict(torch.load(f\"UJI_HeteroGNNSimplified_without_{zone_to_remove}_best_model.pth\"))\n",
    "    model.to(device)\n",
    "\n",
    "    test_loader = DataLoader(test_dataset, batch_size=len(test_dataset), shuffle=False)\n",
    "    \n",
    "    # VALIDATION\n",
    "    model.eval()\n",
    "\n",
    "    for data in test_loader:\n",
    "        data = data.to(device)\n",
    "        out = model(data.x_dict, data.edge_index_dict, data.edge_attr_dict) \n",
    "        out_zones = out[\"zones\"].cpu().reshape(out[\"zones\"].cpu().shape[0]//num_classes,num_classes)\n",
    "\n",
    "        output = m(out_zones)\n",
    "        print(classification_report(data[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1))))\n",
    "        plt.figure(figsize=[9,7])\n",
    "        cf_matrix = confusion_matrix(data[\"zones\"].y.cpu().reshape(-1).type(torch.long), np.array(torch.argmax(output.cpu(), axis=1)), normalize=\"true\")\n",
    "        sns.heatmap(cf_matrix, annot=True, fmt=\".0%\", cmap=\"YlGnBu\", vmin=0, vmax=0.2, cbar=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "n4qV8TIQYl62",
    "gfqVFAK32Th4",
    "fmC2Dsazdx7i",
    "YHsZyzkAZKlK",
    "gU46OaNwNj7Z",
    "P3QFKooiTRQF",
    "DdJpdEQmWRuO",
    "9Y6Zs61LXiik",
    "MTSumaDohl3K",
    "d2MElifG5G5q",
    "2EjKrCuHim66",
    "iSVeCEtNim7C",
    "1uwrgRzJim7D",
    "kpCBGsGe6J3Z",
    "Tb5Kdlky-4DN",
    "2cJTvncIOZ6I",
    "SwovNzD_THq_",
    "V5fnHhVyXMC4"
   ],
   "name": "tesis_mnav_simple_gnn.ipynb",
   "provenance": [
    {
     "file_id": "1AOWB0JuB1YtTUv8waFAYjdVb27Oi_zVw",
     "timestamp": 1617051239005
    },
    {
     "file_id": "1-veytfLbAuWzFrDTNxXhvsEfn8Th0D2K",
     "timestamp": 1616972473551
    },
    {
     "file_id": "1iQI9YVQL7SxQAvy2A4VAE8jJCv7nBfbI",
     "timestamp": 1615845377065
    }
   ]
  },
  "interpreter": {
   "hash": "feb5c0b0a29f8cb9ce82bbdb934224dc5d3b107b617c193c25692c16f2b91aa2"
  },
  "kernelspec": {
   "display_name": "flezama-tesis",
   "language": "python",
   "name": "flezama-tesis"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}