embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 76,
     "status": "ok",
     "timestamp": 1679785587250,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 360
    },
    "id": "C_ugxp2TCJb0",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/home/liying/Documents/MS thesis/master-thesis', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python39.zip', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/lib-dynload', '', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/site-packages']\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n",
    "\n",
    "import sys\n",
    "print(sys.path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 3035,
     "status": "ok",
     "timestamp": 1679785728401,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 360
    },
    "id": "1-_z-Z-qaRQY",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "import shutil\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from os.path import join\n",
    "from torch.utils.data import Dataset\n",
    "import torch\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from torchinfo import summary\n",
    "import configparser\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import datetime\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 7975,
     "status": "ok",
     "timestamp": 1679785737874,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 360
    },
    "id": "GtUY9RXDbOcC",
    "outputId": "62a9ec69-d7db-4773-e1e8-6cb484c60eb9"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "from multimodal import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 237,
     "status": "ok",
     "timestamp": 1679785738098,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 360
    },
    "id": "Yl_3HmEkjQ5O",
    "outputId": "a8865ebe-20a8-48ce-bc4f-9d1fef892285"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 31117,
     "status": "ok",
     "timestamp": 1679291917551,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "id": "K_XfF_k6bkok",
    "outputId": "1a11045b-dae1-46c6-8109-0873189d4f5a",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./Audio/ESC50/meta/esc50.csv\n",
      "./Audio/kitchen20/kitchen20.csv\n",
      "./Audio/silent_sound/silent_sound.csv\n",
      "loading  fold1\n",
      "loading  fold2\n",
      "loading  fold3\n",
      "loading  fold4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liying/Documents/MS thesis/master-thesis/esc.py:121: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  self.df = self.df[0].append(self.df[1:])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading  fold1\n",
      "loading  fold2\n",
      "loading  fold3\n",
      "loading  fold4\n",
      "Loading...\n",
      "Audio data: (480, 1, 16000)\n",
      "IMU data: (60613, 6, 20)\n",
      "Audio data: (7200, 1, 16000)\n",
      "IMU data: (7200, 6, 20)\n",
      "{'brush_teeth': 0, 'click_mouse': 1, 'drink': 2, 'eat': 3, 'flush_toilet': 4, 'sit': 5, 'type': 6, 'walk': 7, 'wash': 8}\n"
     ]
    }
   ],
   "source": [
    "time_window = 1\n",
    "audio_rate = 16000\n",
    "audio_input_length = int(audio_rate * time_window)\n",
    "n_train_multi_data_per_class = 800\n",
    "n_test_multi_data_per_class = 200\n",
    "\n",
    "audio_train_set = ESC70Select(\n",
    "    time_window=time_window,\n",
    "    folds=[1, 2, 3, 4],\n",
    "    transforms=lambda x: nn.functional.pad(x,((audio_input_length-x.shape[1])//2, (audio_input_length-x.shape[1])//2))\\\n",
    "        if (x.shape[1] % 2) == 0 \\\n",
    "        else nn.functional.pad(x,((audio_input_length-x.shape[1])//2, (audio_input_length-x.shape[1])//2 + 1)), \n",
    "    overwrite=False,\n",
    "    use_bc_learning=False,\n",
    "    audio_rate=audio_rate)\n",
    "\n",
    "imu_train_set = WISDMSelect(\n",
    "    folds=[1, 2, 3, 4],\n",
    "    time_window=time_window,\n",
    "    overwrite=False,\n",
    "    normalize_acc=True)\n",
    "\n",
    "audio_train_loader = torch.utils.data.DataLoader(audio_train_set, \n",
    "                                            batch_size=32, \n",
    "                                            shuffle=True)\n",
    "\n",
    "imu_train_loader = DataLoader(imu_train_set, batch_size=128, \n",
    "                            shuffle=True, num_workers=4)\n",
    "\n",
    "multimodal_train_set = MultimodalDataset(audio_train_set, \n",
    "                                         imu_train_set, \n",
    "                                         num_data_per_class=n_train_multi_data_per_class, \n",
    "                                         time_window=time_window,\n",
    "                                         overwrite=False\n",
    "                                         )\n",
    "\n",
    "print('Audio data: ({}, {}, {})'.format(len(audio_train_set.sounds), \n",
    "                          audio_train_set.sounds[0].shape[0], \n",
    "                          audio_train_set.sounds[0].shape[1]))\n",
    "print('IMU data: ({}, {}, {})'.format(len(imu_train_set.imus), \n",
    "                          imu_train_set.imus[0].shape[0], \n",
    "                          imu_train_set.imus[0].shape[1]))\n",
    "print('Audio data: ({}, {}, {})'.format(len(multimodal_train_set.sounds), \n",
    "                          multimodal_train_set.sounds[0].shape[0], \n",
    "                          multimodal_train_set.sounds[0].shape[1]))\n",
    "print('IMU data: ({}, {}, {})'.format(len(multimodal_train_set.imus), \n",
    "                          multimodal_train_set.imus[0].shape[0], \n",
    "                          multimodal_train_set.imus[0].shape[1]))\n",
    "\n",
    "print(multimodal_train_set.get_label_mapping())\n",
    "\n",
    "multimodal_train_loader = DataLoader(multimodal_train_set, batch_size=8, \n",
    "                            shuffle=True, num_workers=4)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 19423,
     "status": "ok",
     "timestamp": 1679291936970,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "id": "EABuF4hjajw4",
    "outputId": "17ca3e9f-7958-40c8-da42-0d95f617f6d9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./Audio/ESC50/meta/esc50.csv\n",
      "./Audio/kitchen20/kitchen20.csv\n",
      "./Audio/silent_sound/silent_sound.csv\n",
      "loading  fold5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liying/Documents/MS thesis/master-thesis/esc.py:121: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  self.df = self.df[0].append(self.df[1:])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading  fold5\n",
      "Loading...\n",
      "Audio data: (1800, 1, 16000)\n",
      "IMU data: (1800, 6, 20)\n"
     ]
    }
   ],
   "source": [
    "audio_test_set = ESC70Select(\n",
    "    time_window=time_window,\n",
    "    folds=[5],\n",
    "    transforms=lambda x: nn.functional.pad(x,((audio_input_length-x.shape[1])//2, (audio_input_length-x.shape[1])//2))\\\n",
    "        if (x.shape[1] % 2) == 0 \\\n",
    "        else nn.functional.pad(x,((audio_input_length-x.shape[1])//2, (audio_input_length-x.shape[1])//2 + 1)), \n",
    "    overwrite=False,\n",
    "    use_bc_learning=False,\n",
    "    audio_rate=audio_rate)\n",
    "\n",
    "audio_test_loader = DataLoader(audio_test_set, batch_size=32, \n",
    "                            shuffle=False, num_workers=2)\n",
    "\n",
    "imu_test_set = WISDMSelect(\n",
    "    folds=[5],\n",
    "    time_window=time_window,\n",
    "    overwrite=False,\n",
    "    normalize_acc=True)\n",
    "\n",
    "imu_test_loader = DataLoader(imu_test_set, batch_size=128, \n",
    "                            shuffle=False, num_workers=2)\n",
    "\n",
    "\n",
    "multimodal_test_set = MultimodalDataset(audio_test_set, \n",
    "                                        imu_test_set, \n",
    "                                        num_data_per_class=n_test_multi_data_per_class,\n",
    "                                        time_window=time_window,\n",
    "                                        overwrite=False\n",
    "                                        )\n",
    "\n",
    "\n",
    "print('Audio data: ({}, {}, {})'.format(len(multimodal_test_set.sounds), \n",
    "                          multimodal_test_set.sounds[0].shape[0], \n",
    "                          multimodal_test_set.sounds[0].shape[1]))\n",
    "print('IMU data: ({}, {}, {})'.format(len(multimodal_test_set.imus), \n",
    "                          multimodal_test_set.imus[0].shape[0], \n",
    "                          multimodal_test_set.imus[0].shape[1]))\n",
    "\n",
    "multimodal_test_loader = DataLoader(multimodal_test_set, batch_size=8, \n",
    "                            shuffle=True, num_workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "OEXJlsrt3iUT"
   },
   "source": [
    "# Audio Module: BEATs\n",
    "Use a pre-trained model to extract sound features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1679291936970,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "hidden": true,
    "id": "R0Bozh973hwn",
    "outputId": "9d6a07a9-c78a-4e0a-fd0c-2060053bc2e3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/home/liying/Documents/MS thesis/master-thesis', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python39.zip', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/lib-dynload', '', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/site-packages', '/home/liying/Documents/MS thesis/master-thesis/BEATs']\n"
     ]
    }
   ],
   "source": [
    "sys.path.append('/home/liying/Documents/MS thesis/master-thesis/BEATs')\n",
    "print(sys.path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1679291936971,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "hidden": true,
    "id": "irlYUkFEoAhu",
    "outputId": "c8176dd6-41e7-4031-ae4e-ebc9b53f4ecf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'blender': 0,\n",
       " 'no_sound': 1,\n",
       " 'stove-burner': 2,\n",
       " 'water-flowing': 3,\n",
       " 'drawer': 4,\n",
       " 'clean-dishes': 5,\n",
       " 'chopping': 6,\n",
       " 'eat': 7,\n",
       " 'peel': 8,\n",
       " 'toilet_flush': 9,\n",
       " 'footsteps': 10,\n",
       " 'brushing_teeth': 11,\n",
       " 'drinking_sipping': 12,\n",
       " 'mouse_click': 13,\n",
       " 'keyboard_typing': 14}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audio_train_set.get_label_mapping()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true,
    "id": "-DrGqLnD3_SL"
   },
   "outputs": [],
   "source": [
    "from BEATs import BEATs, BEATsConfig\n",
    "\n",
    "# load the pre-trained checkpoints\n",
    "checkpoint = torch.load('./BEATs/BEATs_iter3_plus_AS2M.pt')\n",
    "\n",
    "cfg = BEATsConfig(checkpoint['cfg'])\n",
    "BEATs_model = BEATs(cfg)\n",
    "BEATs_model.load_state_dict(checkpoint['model'])\n",
    "BEATs_model.eval()\n",
    "\n",
    "# extract the the audio representation\n",
    "audio_input_16khz = audio_train_set.sounds[0]\n",
    "padding_mask = torch.zeros_like(audio_train_set.sounds[0]).bool()\n",
    "\n",
    "representation = BEATs_model.extract_features(audio_input_16khz, padding_mask=padding_mask)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Layer (type:depth-idx)                                  Param #\n",
      "================================================================================\n",
      "BEATs                                                   --\n",
      "├─Linear: 1-1                                           393,984\n",
      "├─Conv2d: 1-2                                           131,072\n",
      "├─Dropout: 1-3                                          --\n",
      "├─TransformerEncoder: 1-4                               --\n",
      "│    └─Sequential: 2-1                                  --\n",
      "│    │    └─Conv1d: 3-1                                 4,719,488\n",
      "│    │    └─SamePad: 3-2                                --\n",
      "│    │    └─GELU: 3-3                                   --\n",
      "│    └─ModuleList: 2-2                                  --\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-4        7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-5        7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-6        7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-7        7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-8        7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-9        7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-10       7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-11       7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-12       7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-13       7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-14       7,092,244\n",
      "│    │    └─TransformerSentenceEncoderLayer: 3-15       7,092,244\n",
      "│    └─LayerNorm: 2-3                                   1,536\n",
      "├─LayerNorm: 1-5                                        1,024\n",
      "================================================================================\n",
      "Total params: 90,354,032\n",
      "Trainable params: 90,354,032\n",
      "Non-trainable params: 0\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "print(summary(BEATs_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true,
    "id": "mX3JEcPjWX2E"
   },
   "source": [
    "## Experiment: finetune model on ESC70Select dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true,
    "id": "kGpELYJXSVfc"
   },
   "outputs": [],
   "source": [
    "class BEATsFinetuned(nn.Module):\n",
    "    def __init__(self, BEATs_pretrained_model, n_class, predictor_dropout=0.0):\n",
    "        super().__init__()\n",
    "        self.BEATs = BEATs_pretrained_model\n",
    "        self.predictor_dropout = nn.Dropout(predictor_dropout)\n",
    "        self.predictor = nn.Linear(768, n_class)\n",
    " \n",
    "    def forward(self, x):\n",
    "        x = self.BEATs.extract_features(x, padding_mask=torch.zeros_like(x).bool())[0]\n",
    "        x = self.predictor_dropout(x)\n",
    "        logits = self.predictor(x).mean(dim=1)\n",
    "        lprobs = torch.sigmoid(logits)\n",
    "        return lprobs\n",
    "\n",
    "    def extract_features(self, x):\n",
    "        x = self.BEATs.extract_features(x, padding_mask=torch.zeros_like(x).bool())[0]\n",
    "        x = self.predictor_dropout(x)\n",
    "        logits = self.predictor(x).mean(dim=1)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1276,
     "status": "ok",
     "timestamp": 1679289624256,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "hidden": true,
    "id": "BMBYRintWcj-",
    "outputId": "b43d05de-e570-4979-f273-440e7ee1ddc0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 15])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Freeze the pretrained model\n",
    "for param in BEATs_model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "BEATs_finetuned_model = BEATsFinetuned(BEATs_model, audio_train_set.nClasses)\n",
    "output = BEATs_finetuned_model(audio_train_set.sounds[0])\n",
    "output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true,
    "id": "Yy0hnqXTW4Ag"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 440
    },
    "executionInfo": {
     "elapsed": 655829,
     "status": "error",
     "timestamp": 1679290282413,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "hidden": true,
    "id": "bt2XvFxyh9-e",
    "outputId": "a4237585-f609-48a6-fda3-f05cd66348fb",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'no_sound': 0, 'blender': 1, 'stove-burner': 2, 'water-flowing': 3, 'drawer': 4, 'clean-dishes': 5, 'chopping': 6, 'eat': 7, 'peel': 8, 'toilet_flush': 9, 'footsteps': 10, 'brushing_teeth': 11, 'drinking_sipping': 12, 'mouse_click': 13, 'keyboard_typing': 14}\n",
      "Loss: 2.703262436389923, Accuracy: 0.08125\n",
      "Loss: 2.6863094727198282, Accuracy: 0.18125\n",
      "Loss: 2.668513762950897, Accuracy: 0.3\n",
      "Loss: 2.6522530714670816, Accuracy: 0.3645833333333333\n",
      "Loss: 2.635932270685832, Accuracy: 0.4666666666666667\n",
      "Loss: 2.620411479473114, Accuracy: 0.5125\n",
      "Loss: 2.606025139490763, Accuracy: 0.5645833333333333\n",
      "Loss: 2.5914759993553163, Accuracy: 0.6208333333333333\n",
      "Loss: 2.577638653914134, Accuracy: 0.6666666666666666\n",
      "Loss: 2.564376974105835, Accuracy: 0.7125\n",
      "Loss: 2.551648751894633, Accuracy: 0.71875\n",
      "Loss: 2.5392409880956013, Accuracy: 0.7375\n",
      "Loss: 2.5272345622380574, Accuracy: 0.7625\n",
      "Loss: 2.5158973693847657, Accuracy: 0.7604166666666666\n",
      "Loss: 2.5044994155565896, Accuracy: 0.7666666666666667\n",
      "Loss: 2.4939009388287863, Accuracy: 0.78125\n",
      "Loss: 2.4835455616315207, Accuracy: 0.7833333333333333\n",
      "Loss: 2.473237518469493, Accuracy: 0.79375\n",
      "Loss: 2.4634318113327027, Accuracy: 0.8\n",
      "Loss: 2.454006330172221, Accuracy: 0.8\n",
      "Loss: 2.4446454922358196, Accuracy: 0.8\n",
      "Loss: 2.4359884301821393, Accuracy: 0.81875\n",
      "Loss: 2.427233417828878, Accuracy: 0.8229166666666666\n",
      "Loss: 2.419087553024292, Accuracy: 0.8333333333333334\n",
      "Loss: 2.4109250664711, Accuracy: 0.8270833333333333\n",
      "Loss: 2.403146433830261, Accuracy: 0.8458333333333333\n",
      "Loss: 2.3957582116127014, Accuracy: 0.83125\n",
      "Loss: 2.388483472665151, Accuracy: 0.8416666666666667\n",
      "Loss: 2.381380848089854, Accuracy: 0.8458333333333333\n",
      "Loss: 2.37441771030426, Accuracy: 0.8520833333333333\n",
      "Loss: 2.3678036371866864, Accuracy: 0.8520833333333333\n",
      "Loss: 2.36129199663798, Accuracy: 0.8479166666666667\n",
      "Loss: 2.35509649515152, Accuracy: 0.85625\n",
      "Loss: 2.3488556504249574, Accuracy: 0.8520833333333333\n",
      "Loss: 2.342900987466176, Accuracy: 0.8583333333333333\n",
      "Loss: 2.337339731057485, Accuracy: 0.85625\n",
      "Loss: 2.331634799639384, Accuracy: 0.8541666666666666\n",
      "Loss: 2.326226170857747, Accuracy: 0.8541666666666666\n",
      "Loss: 2.321034590403239, Accuracy: 0.8520833333333333\n",
      "Loss: 2.3159555435180663, Accuracy: 0.8583333333333333\n",
      "Loss: 2.311206746101379, Accuracy: 0.85625\n",
      "Loss: 2.30595684448878, Accuracy: 0.8625\n",
      "Loss: 2.3012861728668215, Accuracy: 0.86875\n",
      "Loss: 2.2966639002164206, Accuracy: 0.8583333333333333\n",
      "Loss: 2.292017952601115, Accuracy: 0.8604166666666667\n",
      "Loss: 2.2878124396006267, Accuracy: 0.8666666666666667\n",
      "Loss: 2.2833536942799886, Accuracy: 0.8729166666666667\n",
      "Loss: 2.2792644103368125, Accuracy: 0.8666666666666667\n",
      "Loss: 2.2752171754837036, Accuracy: 0.8666666666666667\n",
      "Loss: 2.271195634206136, Accuracy: 0.86875\n",
      "Loss: 2.267342491944631, Accuracy: 0.8729166666666667\n",
      "Loss: 2.263618044058482, Accuracy: 0.8729166666666667\n",
      "Loss: 2.259994467099508, Accuracy: 0.8770833333333333\n",
      "Loss: 2.25635902484258, Accuracy: 0.88125\n",
      "Loss: 2.252742612361908, Accuracy: 0.8791666666666667\n",
      "Loss: 2.249236011505127, Accuracy: 0.88125\n",
      "Loss: 2.24599107503891, Accuracy: 0.88125\n",
      "Loss: 2.242753318945567, Accuracy: 0.8833333333333333\n",
      "Loss: 2.239340341091156, Accuracy: 0.88125\n",
      "Loss: 2.2361655950546266, Accuracy: 0.8833333333333333\n",
      "Loss: 2.2329665939013164, Accuracy: 0.8916666666666667\n",
      "Loss: 2.2300698081652324, Accuracy: 0.8895833333333333\n",
      "Loss: 2.226985716819763, Accuracy: 0.8916666666666667\n",
      "Loss: 2.224168264865875, Accuracy: 0.8895833333333333\n",
      "Loss: 2.221243139108022, Accuracy: 0.8895833333333333\n",
      "Loss: 2.2182862043380736, Accuracy: 0.8979166666666667\n",
      "Loss: 2.2156511743863425, Accuracy: 0.8916666666666667\n",
      "Loss: 2.2128869851430255, Accuracy: 0.8979166666666667\n",
      "Loss: 2.2103466987609863, Accuracy: 0.8958333333333334\n",
      "Loss: 2.2076623876889547, Accuracy: 0.8979166666666667\n",
      "Loss: 2.205228328704834, Accuracy: 0.9\n",
      "Loss: 2.202668273448944, Accuracy: 0.8979166666666667\n",
      "Loss: 2.200034050146739, Accuracy: 0.9020833333333333\n",
      "Loss: 2.19766708612442, Accuracy: 0.9020833333333333\n",
      "Loss: 2.195325195789337, Accuracy: 0.9\n",
      "Loss: 2.193045699596405, Accuracy: 0.9041666666666667\n",
      "Loss: 2.1907169938087465, Accuracy: 0.9020833333333333\n",
      "Loss: 2.188422969977061, Accuracy: 0.90625\n",
      "Loss: 2.186174726486206, Accuracy: 0.90625\n",
      "Loss: 2.184038992722829, Accuracy: 0.90625\n",
      "Loss: 2.181912573178609, Accuracy: 0.90625\n",
      "Loss: 2.1799172043800352, Accuracy: 0.9083333333333333\n",
      "Loss: 2.1776636362075807, Accuracy: 0.90625\n",
      "Loss: 2.1757574240366617, Accuracy: 0.9104166666666667\n",
      "Loss: 2.1737619558970134, Accuracy: 0.9083333333333333\n",
      "Loss: 2.1718177795410156, Accuracy: 0.9125\n",
      "Loss: 2.1698403199513754, Accuracy: 0.90625\n",
      "Loss: 2.167923895517985, Accuracy: 0.9125\n",
      "Loss: 2.166032612323761, Accuracy: 0.9145833333333333\n",
      "Loss: 2.1641634424527485, Accuracy: 0.9083333333333333\n",
      "Loss: 2.162369970480601, Accuracy: 0.9083333333333333\n",
      "Loss: 2.160548237959544, Accuracy: 0.9104166666666667\n",
      "Loss: 2.158755358060201, Accuracy: 0.9104166666666667\n",
      "Loss: 2.1570175369580586, Accuracy: 0.9104166666666667\n",
      "Loss: 2.155260705947876, Accuracy: 0.9125\n",
      "Loss: 2.1535446763038637, Accuracy: 0.9104166666666667\n",
      "Loss: 2.1518831928571065, Accuracy: 0.9125\n",
      "Loss: 2.1503375649452208, Accuracy: 0.9125\n",
      "Loss: 2.148618197441101, Accuracy: 0.9125\n",
      "Loss: 2.147114896774292, Accuracy: 0.9104166666666667\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(BEATs_finetuned_model.parameters(), lr=0.001, momentum=0.9)\n",
    "BEATs_finetuned_model.to(device)\n",
    "\n",
    "print(audio_train_set.get_label_mapping())\n",
    "# Training loop\n",
    "n_epochs = 100\n",
    "summary = {'loss': [[] for _ in range(n_epochs)], 'acc': [[] for _ in range(n_epochs)]}\n",
    "for e in range(n_epochs):\n",
    "    for i, (sounds, labels) in enumerate(audio_train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        sounds = sounds.squeeze(dim=1).to(device)\n",
    "        labels = labels.to(device)\n",
    "        # Run the Net\n",
    "        x = BEATs_finetuned_model(sounds)\n",
    "        # print(x.shape)\n",
    "        # print(labels.shape)\n",
    "        # x = x.view(x.size()[:-1])\n",
    "\n",
    "        # Optimize net\n",
    "        loss = criterion(x, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        summary['loss'][e].append(loss.item())\n",
    "\n",
    "            # Calculat accuracy\n",
    "        _, pred = x.data.topk(1, dim=1)\n",
    "        pred = pred.view(pred.shape[:-1])\n",
    "        acc = torch.sum(pred == labels)/x.shape[0]\n",
    "        summary['acc'][e].append(acc.item())\n",
    "        \n",
    "    print('Loss: {}, Accuracy: {}'.format(np.mean(summary['loss'][e]), np.mean(summary['acc'][e])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true,
    "id": "2eyD8-5uWmvw"
   },
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 440
    },
    "executionInfo": {
     "elapsed": 2131,
     "status": "error",
     "timestamp": 1679291571264,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "hidden": true,
    "id": "dR-3eDHjo9l4",
    "outputId": "d895fb7b-19f1-48c6-ffdf-11f4c0426093"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.89\n"
     ]
    }
   ],
   "source": [
    "test_accuracy = []\n",
    "for i, (sounds, labels) in enumerate(audio_test_loader):\n",
    "        # Run the Net\n",
    "        sounds = sounds.squeeze(dim=1).to(device)\n",
    "        labels = labels.to(device)\n",
    "        x = BEATs_finetuned_model(sounds)\n",
    "\n",
    "        # loss = criterion(x, labels.long())\n",
    "        # summary['loss'][e].append(loss.item())\n",
    "        # Calculat accuracy\n",
    "        _, pred = x.data.topk(1, dim=1)\n",
    "        pred = pred.view(pred.shape[:-1])\n",
    "        acc = torch.sum(pred == labels)/x.shape[0]\n",
    "        summary['acc'][e].append(acc.item())\n",
    "print(np.mean(summary['acc'][e]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "JpM4j-4-XFL5"
   },
   "source": [
    "# IMU Module: LIMU-BERT\n",
    "A pre-trained autoencoder model using LIMU-BERT architecture to extract acceleration features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/home/liying/Documents/MS thesis/master-thesis', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python39.zip', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/lib-dynload', '', '/home/liying/miniconda3/envs/pytorch-gpu/lib/python3.9/site-packages', '/home/liying/Documents/MS thesis/master-thesis/BEATs', '/home/liying/Documents/MS thesis/master-thesis/LIMUBert']\n"
     ]
    }
   ],
   "source": [
    "sys.path.append('/home/liying/Documents/MS thesis/master-thesis/LIMUBert')\n",
    "print(sys.path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true,
    "id": "Xe9EURZzAymE"
   },
   "outputs": [],
   "source": [
    "from LIMUBert.utils import load_model_config, Preprocess4Normalization, IMUDataset\n",
    "from LIMUBert.models import LIMUBertModel4Pretrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Load LIMU-BERT model\n",
    "\n",
    "model_cfg = load_model_config('pretrain_base', 'base', 'v1', path_bert='LIMUBert/config/limu_bert.json')\n",
    "if model_cfg is None:\n",
    "    print(\"Unable to find corresponding model config!\")\n",
    "\n",
    "pipeline = [Preprocess4Normalization(model_cfg.feature_num)]\n",
    "LIMUBert_model = LIMUBertModel4Pretrain(model_cfg, output_embed=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================================================\n",
      "Layer (type:depth-idx)                   Param #\n",
      "=================================================================\n",
      "LIMUBertModel4Pretrain                   --\n",
      "├─Transformer: 1-1                       --\n",
      "│    └─Embeddings: 2-1                   --\n",
      "│    │    └─Linear: 3-1                  504\n",
      "│    │    └─Embedding: 3-2               8,640\n",
      "│    │    └─LayerNorm: 3-3               144\n",
      "│    └─MultiHeadedSelfAttention: 2-2     --\n",
      "│    │    └─Linear: 3-4                  5,256\n",
      "│    │    └─Linear: 3-5                  5,256\n",
      "│    │    └─Linear: 3-6                  5,256\n",
      "│    └─Linear: 2-3                       5,256\n",
      "│    └─LayerNorm: 2-4                    144\n",
      "│    └─PositionWiseFeedForward: 2-5      --\n",
      "│    │    └─Linear: 3-7                  10,512\n",
      "│    │    └─Linear: 3-8                  10,440\n",
      "│    └─LayerNorm: 2-6                    144\n",
      "├─Linear: 1-2                            5,256\n",
      "├─Linear: 1-3                            5,256\n",
      "├─LayerNorm: 1-4                         144\n",
      "├─Linear: 1-5                            438\n",
      "=================================================================\n",
      "Total params: 62,646\n",
      "Trainable params: 62,646\n",
      "Non-trainable params: 0\n",
      "=================================================================\n"
     ]
    }
   ],
   "source": [
    "# load the pre-trained checkpoints\n",
    "checkpoint = torch.load('./LIMUBert/saved/pretrain_base_wisdm_20_100/wisdm.pt')\n",
    "\n",
    "# cfg = BEATsConfig(checkpoint['cfg'])\n",
    "# BEATs_model = BEATs(cfg)\n",
    "LIMUBert_model.load_state_dict(checkpoint)\n",
    "# LIMUBert_model.eval()\n",
    "print(summary(LIMUBert_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14398\n"
     ]
    }
   ],
   "source": [
    "LIMUBert_model.to('cpu')\n",
    "print(len(imu_test_loader.dataset.imus))\n",
    "# for i, (imus, labels) in enumerate(imu_test_loader):\n",
    "#     # Run the Net\n",
    "# #     print(imus.shape)\n",
    "#     imus = imus.transpose(-1, 1).to('cpu')\n",
    "#     labels = labels.to('cpu')\n",
    "#     x = LIMUBert_model(imus)\n",
    "#     if i % 50 == 0:\n",
    "#         print(x.shape)\n",
    "# print(np.mean(summary['acc'][e]))\n",
    "\n",
    "# test_output=LIMUBert_model(torch.rand(1, 100, 6))\n",
    "# test_output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Experiment: finetune model on WISDMSelect dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class LIMUBertFinetuned(nn.Module):\n",
    "    def __init__(self, LIMUBert_pretrained_model, n_class, predictor_dropout=0.0):\n",
    "        super().__init__()\n",
    "        self.LIMUBert = LIMUBert_pretrained_model\n",
    "        self.predictor_dropout = nn.Dropout(predictor_dropout)\n",
    "#         self.predictor = nn.Linear(72, n_class)\n",
    "        self.predictor = GRU(72, n_class)\n",
    " \n",
    "    def forward(self, x):\n",
    "        x = self.LIMUBert(x.transpose(-1,1)) # Input to LIMUBert model is N * L * C\n",
    "        x = self.predictor_dropout(x)\n",
    "#         logits = self.predictor(x).mean(dim=1)\n",
    "        logits = self.predictor(x)\n",
    "        lprobs = torch.sigmoid(logits)\n",
    "        return lprobs\n",
    "\n",
    "    def extract_features(self, x):\n",
    "        x = self.LIMUBert(x.transpose(-1,1))\n",
    "        x = self.predictor_dropout(x)\n",
    "#         logits = self.predictor(x).mean(dim=1)\n",
    "        logits = self.predictor(x)\n",
    "        return logits\n",
    "    \n",
    "class GRU(nn.Module):\n",
    "    def __init__(self, input_feature_dim, output_feature_dim, training=False):\n",
    "        super().__init__()\n",
    "        self.dropout = True\n",
    "        self.num_rnn = 2\n",
    "        self.num_linear = 1\n",
    "        self.rnn_io = [[input_feature_dim, 20], [20, output_feature_dim]]\n",
    "        self.num_layers = [2, 1]\n",
    "        for i in range(self.num_rnn):\n",
    "            self.__setattr__('gru' + str(i), nn.GRU(self.rnn_io[i][0], self.rnn_io[i][1], num_layers=self.num_layers[i],\n",
    "                                         batch_first=True))\n",
    "        \n",
    "\n",
    "    def forward(self, input_seqs, training=False):\n",
    "        h = input_seqs\n",
    "        for i in range(self.num_rnn):\n",
    "            rnn = self.__getattr__('gru' + str(i))\n",
    "            h, _ = rnn(h)\n",
    "            h = nn.functional.relu(h)\n",
    "#         print(h.shape)  \n",
    "        h = h[:, -1, :]\n",
    "#         print(h.shape)\n",
    "        if self.dropout:\n",
    "            h = nn.functional.dropout(h, training=training)\n",
    "        return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================================================\n",
      "Layer (type:depth-idx)                        Param #\n",
      "======================================================================\n",
      "LIMUBertFinetuned                             --\n",
      "├─LIMUBertModel4Pretrain: 1-1                 --\n",
      "│    └─Transformer: 2-1                       --\n",
      "│    │    └─Embeddings: 3-1                   (9,288)\n",
      "│    │    └─MultiHeadedSelfAttention: 3-2     (15,768)\n",
      "│    │    └─Linear: 3-3                       (5,256)\n",
      "│    │    └─LayerNorm: 3-4                    (144)\n",
      "│    │    └─PositionWiseFeedForward: 3-5      (20,952)\n",
      "│    │    └─LayerNorm: 3-6                    (144)\n",
      "│    └─Linear: 2-2                            (5,256)\n",
      "│    └─Linear: 2-3                            (5,256)\n",
      "│    └─LayerNorm: 2-4                         (144)\n",
      "│    └─Linear: 2-5                            (438)\n",
      "├─Dropout: 1-2                                --\n",
      "├─GRU: 1-3                                    --\n",
      "│    └─GRU: 2-6                               8,160\n",
      "│    └─GRU: 2-7                               720\n",
      "======================================================================\n",
      "Total params: 71,526\n",
      "Trainable params: 8,880\n",
      "Non-trainable params: 62,646\n",
      "======================================================================\n"
     ]
    }
   ],
   "source": [
    "# Freeze the pretrained model\n",
    "for param in LIMUBert_model.parameters():\n",
    "    param.requires_grad = False\n",
    "    \n",
    "LIMUBert_finetuned_model = LIMUBertFinetuned(LIMUBert_model, imu_train_set.nClasses)\n",
    "# output = LIMUBert_finetuned_model(imu_train_set.imus[0].unsqueeze(0))\n",
    "# output.shape\n",
    "print(summary(LIMUBert_finetuned_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'walking': 0, 'jogging': 1, 'sitting': 2, 'standing': 3, 'typing': 4, 'teeth': 5, 'pasta': 6, 'drinking': 7}\n",
      "Loss: 2.078253234060187, Accuracy: 0.1540648496464679\n",
      "Loss: 2.0764355885355097, Accuracy: 0.1673331767320633\n",
      "Loss: 2.0747011887399776, Accuracy: 0.18660479329134289\n",
      "Loss: 2.073084316755596, Accuracy: 0.20871240606433467\n",
      "Loss: 2.0715565054040206, Accuracy: 0.22311795118607972\n",
      "Loss: 2.070141440943668, Accuracy: 0.23756109036897358\n",
      "Loss: 2.068834124113384, Accuracy: 0.24810385343275573\n",
      "Loss: 2.06751742613943, Accuracy: 0.2566118422307466\n",
      "Loss: 2.0662729765239516, Accuracy: 0.2641635339511068\n",
      "Loss: 2.065074012154027, Accuracy: 0.2707706768261759\n",
      "Loss: 2.063909068860506, Accuracy: 0.2780592105890575\n",
      "Loss: 2.062872856541684, Accuracy: 0.2832307330871883\n",
      "Loss: 2.0618502039658395, Accuracy: 0.28916353395110683\n",
      "Loss: 2.060886337882594, Accuracy: 0.2919736843360098\n",
      "Loss: 2.059885213249608, Accuracy: 0.2940836467240986\n",
      "Loss: 2.058993831433748, Accuracy: 0.2954816730398881\n",
      "Loss: 2.0580927095915142, Accuracy: 0.29712640988199335\n",
      "Loss: 2.0572304575066815, Accuracy: 0.297991071563018\n",
      "Loss: 2.0563747180135628, Accuracy: 0.29836231203455676\n",
      "Loss: 2.055465437236585, Accuracy: 0.3002678571563018\n",
      "Loss: 2.0545054686696904, Accuracy: 0.30284304524722855\n",
      "Loss: 2.053565861049451, Accuracy: 0.3029111843360098\n",
      "Loss: 2.052625977365594, Accuracy: 0.30385573318130094\n",
      "Loss: 2.0516975026381643, Accuracy: 0.3046240602668963\n",
      "Loss: 2.050820975554617, Accuracy: 0.30507518805955586\n",
      "Loss: 2.049933564035516, Accuracy: 0.3051174813195279\n",
      "Loss: 2.0490581863804866, Accuracy: 0.3054863722700822\n",
      "Loss: 2.0482594138697574, Accuracy: 0.30610432342479105\n",
      "Loss: 2.047368438620316, Accuracy: 0.3068280075725756\n",
      "Loss: 2.046677629571212, Accuracy: 0.3065695489707746\n",
      "Loss: 2.0459378041719134, Accuracy: 0.3065695489707746\n",
      "Loss: 2.0452525590595445, Accuracy: 0.3067199249016611\n",
      "Loss: 2.044622220491108, Accuracy: 0.30591400391177126\n",
      "Loss: 2.04401932766563, Accuracy: 0.30611842117811505\n",
      "Loss: 2.0434081880669845, Accuracy: 0.30589990615844725\n",
      "Loss: 2.042876188378585, Accuracy: 0.3052843045247228\n",
      "Loss: 2.0422119190818386, Accuracy: 0.3055145677767302\n",
      "Loss: 2.04170957615501, Accuracy: 0.3056931392142647\n",
      "Loss: 2.0411738646657844, Accuracy: 0.3051456768261759\n",
      "Loss: 2.0406748570893942, Accuracy: 0.30480263170443084\n",
      "Loss: 2.040219319494147, Accuracy: 0.30496710538864136\n",
      "Loss: 2.039746101279008, Accuracy: 0.30491306405318414\n",
      "Loss: 2.0393053606936804, Accuracy: 0.3046381580202203\n",
      "Loss: 2.038817400681345, Accuracy: 0.3043350564806085\n",
      "Loss: 2.038392880088405, Accuracy: 0.3045535715002763\n",
      "Loss: 2.0380976526360763, Accuracy: 0.3040906956321315\n",
      "Loss: 2.0377369554419267, Accuracy: 0.30453007519245145\n",
      "Loss: 2.0373198383732847, Accuracy: 0.30437969926156494\n",
      "Loss: 2.0368572536267733, Accuracy: 0.30519736848379436\n",
      "Loss: 2.0367139364543716, Accuracy: 0.3040930451531159\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(LIMUBert_finetuned_model.parameters(), lr=0.001, momentum=0.9)\n",
    "# optimizer = optim.Adam(params=LIMUBert_finetuned_model.parameters(), lr=1e-3)\n",
    "LIMUBert_finetuned_model.to(device)\n",
    "\n",
    "imu_train_loader = DataLoader(imu_train_set, batch_size=128, \n",
    "                            shuffle=True, num_workers=4)\n",
    "print(imu_train_set.get_label_mapping())\n",
    "# Training loop\n",
    "n_epochs = 50\n",
    "summary = {'loss': [[] for _ in range(n_epochs)], 'acc': [[] for _ in range(n_epochs)]}\n",
    "for e in range(n_epochs):\n",
    "    for i, (imus, labels) in enumerate(imu_train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        imus = imus.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # Run the Net\n",
    "        x = LIMUBert_finetuned_model(imus)\n",
    "        # print(x.shape)\n",
    "        # print(labels.shape)\n",
    "        # x = x.view(x.size()[:-1])\n",
    "\n",
    "        # Optimize net\n",
    "        loss = criterion(x, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        summary['loss'][e].append(loss.item())\n",
    "\n",
    "            # Calculat accuracy\n",
    "        _, pred = x.data.topk(1, dim=1)\n",
    "        pred = pred.view(pred.shape[:-1])\n",
    "        acc = torch.sum(pred == labels)/x.shape[0]\n",
    "        summary['acc'][e].append(acc.item())\n",
    "        \n",
    "    print('Loss: {}, Accuracy: {}'.format(np.mean(summary['loss'][e]), np.mean(summary['acc'][e])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 1.8995697397934763, Accuracy: 1.0\n",
      "Loss: 1.8805682696794208, Accuracy: 1.0\n",
      "Loss: 1.880459771658245, Accuracy: 1.0\n",
      "Loss: 1.8804107791499087, Accuracy: 1.0\n",
      "Loss: 1.8804002197165237, Accuracy: 1.0\n",
      "Loss: 1.8803832317653455, Accuracy: 1.0\n",
      "Loss: 1.8803760930111533, Accuracy: 1.0\n",
      "Loss: 1.8803772851040488, Accuracy: 1.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[67], line 33\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;66;03m# print(x.shape)\u001b[39;00m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;66;03m# print(labels.shape)\u001b[39;00m\n\u001b[1;32m     29\u001b[0m \u001b[38;5;66;03m# x = x.view(x.size()[:-1])\u001b[39;00m\n\u001b[1;32m     30\u001b[0m \n\u001b[1;32m     31\u001b[0m \u001b[38;5;66;03m# Optimize net\u001b[39;00m\n\u001b[1;32m     32\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(x, labels\u001b[38;5;241m.\u001b[39mlong())\n\u001b[0;32m---> 33\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     34\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m     35\u001b[0m summary[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mloss\u001b[39m\u001b[38;5;124m'\u001b[39m][e]\u001b[38;5;241m.\u001b[39mappend(loss\u001b[38;5;241m.\u001b[39mitem())\n",
      "File \u001b[0;32m~/miniconda3/envs/pytorch-gpu/lib/python3.9/site-packages/torch/_tensor.py:396\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    387\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    388\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    389\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    390\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    394\u001b[0m         create_graph\u001b[38;5;241m=\u001b[39mcreate_graph,\n\u001b[1;32m    395\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs)\n\u001b[0;32m--> 396\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/pytorch-gpu/lib/python3.9/site-packages/torch/autograd/__init__.py:173\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    168\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    170\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    171\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    172\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 173\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    174\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    175\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import copy\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "# optimizer = optim.SGD(LIMUBert_finetuned_model.parameters(), lr=0.001, momentum=0.9)\n",
    "optimizer = optim.Adam(params=LIMUBert_finetuned_model.parameters(), lr=1e-3)\n",
    "LIMUBert_finetuned_model.to(device)\n",
    "\n",
    "imu_tmp_train_set = copy.deepcopy(imu_train_set)\n",
    "# imu_tmp_train_set.imus[0][0:3] /= 9.8\n",
    "imu_tmp_train_set.imus = [imu_tmp_train_set.imus[0]]*len(imu_tmp_train_set)\n",
    "imu_tmp_train_set.labels = [imu_tmp_train_set.labels[0]]*len(imu_tmp_train_set)\n",
    "\n",
    "imu_tmp_train_loader = DataLoader(imu_tmp_train_set, batch_size=128, \n",
    "                            shuffle=True, num_workers=4)\n",
    "\n",
    "\n",
    "\n",
    "# Training loop\n",
    "n_epochs = 100\n",
    "summary = {'loss': [[] for _ in range(n_epochs)], 'acc': [[] for _ in range(n_epochs)]}\n",
    "for e in range(n_epochs):\n",
    "    for i, (imus, labels) in enumerate(imu_tmp_train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        imus = imus.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # Run the Net\n",
    "        x = LIMUBert_finetuned_model(imus)\n",
    "        # print(x.shape)\n",
    "        # print(labels.shape)\n",
    "        # x = x.view(x.size()[:-1])\n",
    "\n",
    "        # Optimize net\n",
    "        loss = criterion(x, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        summary['loss'][e].append(loss.item())\n",
    "\n",
    "            # Calculat accuracy\n",
    "        _, pred = x.data.topk(1, dim=1)\n",
    "        pred = pred.view(pred.shape[:-1])\n",
    "        acc = torch.sum(pred == labels)/x.shape[0]\n",
    "        summary['acc'][e].append(acc.item())\n",
    "        \n",
    "    print('Loss: {}, Accuracy: {}'.format(np.mean(summary['loss'][e]), np.mean(summary['acc'][e])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3212503270679341\n"
     ]
    }
   ],
   "source": [
    "test_accuracy = []\n",
    "for i, (imus, labels) in enumerate(imu_test_loader):\n",
    "        # Run the Net\n",
    "        imus = imus.to(device)\n",
    "        labels = labels.to(device)\n",
    "        x = LIMUBert_finetuned_model(imus)\n",
    "\n",
    "        # loss = criterion(x, labels.long())\n",
    "        # summary['loss'][e].append(loss.item())\n",
    "        # Calculat accuracy\n",
    "        _, pred = x.data.topk(1, dim=1)\n",
    "        pred = pred.view(pred.shape[:-1])\n",
    "        acc = torch.sum(pred == labels)/x.shape[0]\n",
    "        summary['acc'][e].append(acc.item())\n",
    "print(np.mean(summary['acc'][e]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n9Ap0dLIqbM_"
   },
   "source": [
    "# Multimodal Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Audio and IMU Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AudioModule(nn.Module):\n",
    "    def __init__(self, BEATs_pretrained_model, dropout_p=0.0):\n",
    "        super().__init__()\n",
    "        self.BEATs = BEATs_pretrained_model # need to freeze params\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    " \n",
    "    def forward(self, x):\n",
    "        x = self.BEATs.extract_features(x, padding_mask=torch.zeros_like(x).bool())[0]\n",
    "        embeddings = self.dropout(x)\n",
    "#         embeddings = embeddings.mean(dim=1)\n",
    "        return embeddings\n",
    "\n",
    "\n",
    "class IMUModule(nn.Module):\n",
    "    def __init__(self, LIMUBert_pretrained_model, dropout_p=0.0):\n",
    "        super().__init__()\n",
    "        self.imu_model = LIMUBert_pretrained_model # need to freeze params\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    " \n",
    "    def forward(self, x):\n",
    "        x = self.imu_model(x)\n",
    "        embeddings = self.dropout(x)\n",
    "        return embeddings\n",
    "    \n",
    "\n",
    "class GRU(nn.Module):\n",
    "    def __init__(self, input_feature_dim, output_feature_dim, training=False):\n",
    "        super().__init__()\n",
    "        self.dropout = True\n",
    "        self.num_rnn = 2\n",
    "        self.num_linear = 1\n",
    "        self.rnn_io = [[input_feature_dim, 256], [256, output_feature_dim]]\n",
    "        self.num_layers = [2, 1]\n",
    "        for i in range(self.num_rnn):\n",
    "            self.__setattr__('gru' + str(i), nn.GRU(self.rnn_io[i][0], self.rnn_io[i][1], num_layers=self.num_layers[i],\n",
    "                                         batch_first=True))\n",
    "        \n",
    "\n",
    "    def forward(self, input_seqs, training=False):\n",
    "        h = input_seqs\n",
    "        for i in range(self.num_rnn):\n",
    "            rnn = self.__getattr__('gru' + str(i))\n",
    "            h, _ = rnn(h)\n",
    "            h = nn.functional.relu(h)\n",
    "#         print(h.shape)  \n",
    "        h = h[:, -1, :]\n",
    "#         print(h.shape)\n",
    "        if self.dropout:\n",
    "            h = nn.functional.dropout(h, training=training)\n",
    "        return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_multimodal_embed_dataset(multimodal_loader, audio_module, imu_module, device, overwrite=False):\n",
    "    \n",
    "    save_path = multimodal_loader.dataset.db_path.split('.npz')[0] + '_embeddings.npz'\n",
    "    config_file = './Multimodal/dataset_config.json'\n",
    "    \n",
    "    if not os.path.isfile(save_path) or overwrite:\n",
    "        dataset = {}\n",
    "        dataset['audio_embeddings'] = []\n",
    "        dataset['imu_embeddings'] = []\n",
    "        dataset['labels'] = []\n",
    "        \n",
    "        audio_module.eval()\n",
    "        imu_module.eval()\n",
    "\n",
    "        for i, (sounds, imus, labels) in enumerate(multimodal_loader):\n",
    "            sounds = sounds.squeeze(dim=1).to(device)\n",
    "            imus = imus.permute(0, 2, 1).to(device)\n",
    "            \n",
    "            with torch.no_grad():\n",
    "                audio_embeddings = audio_module(sounds).cpu().numpy()\n",
    "                imu_embeddings = imu_module(imus).cpu().numpy()\n",
    "            \n",
    "            dataset['audio_embeddings'].append(audio_embeddings)\n",
    "            dataset['imu_embeddings'].append(imu_embeddings)\n",
    "            dataset['labels'].append(labels.numpy())\n",
    "            \n",
    "        dataset['audio_embeddings'] = np.concatenate(dataset['audio_embeddings'], axis=0)\n",
    "        dataset['imu_embeddings'] = np.concatenate(dataset['imu_embeddings'], axis=0)\n",
    "        dataset['labels'] = np.concatenate(dataset['labels'], axis=0)\n",
    "        \n",
    "        np.savez(save_path, **dataset)\n",
    "        \n",
    "    else:\n",
    "        dataset = np.load(save_path, allow_pickle=True)\n",
    "    \n",
    "    if not os.path.isfile(config_file) or overwrite:\n",
    "        dataset_config = {}\n",
    "        dataset_config['db_path'] = save_path\n",
    "        dataset_config['classes'] = multimodal_loader.dataset.classes\n",
    "        dataset_config['nClasses'] = multimodal_loader.dataset.nClasses\n",
    "        dataset_config['time_window'] = multimodal_loader.dataset.time_window\n",
    "        dataset_config['num_data_per_class'] = multimodal_loader.dataset.num_data_per_class\n",
    "        dataset_config['label_mapping'] = multimodal_loader.dataset.get_label_mapping()\n",
    "        \n",
    "        with open(config_file, 'w') as f:\n",
    "            json.dump(dataset_config, f)\n",
    "        \n",
    "    else:\n",
    "        with open(config_file, 'r') as f:\n",
    "            dataset_config = json.load(f)\n",
    "        \n",
    "    return dataset, dataset_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get Multimodal Embedding Dataset for training and testing\n",
    "audio_module = AudioModule(BEATs_model).to(device)\n",
    "imu_module = IMUModule(LIMUBert_model).to(device)\n",
    "\n",
    "embed_dataset, embed_dataset_config = get_multimodal_embed_dataset(DataLoader(multimodal_train_set, batch_size=32, \n",
    "                            shuffle=False, num_workers=4), audio_module, imu_module, device=device, overwrite=False)\n",
    "multimodal_embed_train_set = MultimodalEmbed(embed_dataset, embed_dataset_config)\n",
    "\n",
    "embed_dataset, embed_dataset_config = get_multimodal_embed_dataset(DataLoader(multimodal_test_set, batch_size=32, \n",
    "                            shuffle=False, num_workers=4), audio_module, imu_module, device=device, overwrite=False)\n",
    "multimodal_embed_test_set = MultimodalEmbed(embed_dataset, embed_dataset_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1800, 20, 72)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multimodal_embed_test_set.imu_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get DataLoader for training and testing\n",
    "multimodal_embed_train_loader = DataLoader(multimodal_embed_train_set, batch_size=128, \n",
    "                            shuffle=True, num_workers=4)\n",
    "multimodal_embed_test_loader = DataLoader(multimodal_embed_test_set, batch_size=128, \n",
    "                            shuffle=False, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "JgriyD7YF-gg"
   },
   "outputs": [],
   "source": [
    "class AudioAndIMUFusion(nn.Module):\n",
    "    def __init__(\n",
    "        self,\n",
    "        audio_feature_dim1,\n",
    "        imu_feature_dim1,\n",
    "        audio_feature_dim2,\n",
    "        imu_feature_dim2,\n",
    "        fusion_output_dim,\n",
    "        output_dim,\n",
    "        dropout_p=0.0,\n",
    "        ):\n",
    "        \"\"\"\n",
    "        \n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.audio_gru = GRU(audio_feature_dim1, audio_feature_dim2)\n",
    "        self.imu_gru = GRU(imu_feature_dim1, imu_feature_dim2)\n",
    "        self.fusion = nn.Linear(\n",
    "            in_features=(audio_feature_dim2 + imu_feature_dim2), \n",
    "            out_features=fusion_output_dim\n",
    "            )\n",
    "        self.fc = nn.Linear(\n",
    "            in_features=fusion_output_dim, \n",
    "            out_features=output_dim\n",
    "            )\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    "    \n",
    "    def forward(self, audio_embeddings, imu_embeddings):\n",
    "        audio_features = self.audio_gru(audio_embeddings)\n",
    "#         audio_features = audio_embeddings\n",
    "#         print(audio_features.shape)\n",
    "        imu_features = self.imu_gru(imu_embeddings)\n",
    "        combined = torch.cat([audio_features, imu_features], dim=1)\n",
    "        fused = self.dropout(\n",
    "            nn.functional.relu(\n",
    "                self.fusion(combined)\n",
    "                )\n",
    "            )\n",
    "        logits = self.fc(fused)\n",
    "        pred = torch.sigmoid(logits)\n",
    "        return pred\n",
    "    \n",
    "    def extract_features(self, audio_embeddings, imu_embeddings):\n",
    "        audio_features = self.audio_gru(audio_embeddings)\n",
    "        imu_features = self.imu_gru(imu_embeddings)\n",
    "        combined = torch.cat([audio_features, imu_features], dim=1)\n",
    "        fused = self.dropout(\n",
    "            nn.functional.relu(\n",
    "                self.fusion(combined)\n",
    "                )\n",
    "            )\n",
    "        return fused"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "NM4mCGvLRAXh"
   },
   "outputs": [],
   "source": [
    "# Freeze the two feature models\n",
    "# for param in BEATs_model.parameters():\n",
    "#     param.requires_grad = False\n",
    "# for param in LIMUBert_model.parameters():\n",
    "#     param.requires_grad = False\n",
    "\n",
    "audio_feature_dim1 = 768\n",
    "imu_feature_dim1 = 72\n",
    "audio_feature_dim2 = 128\n",
    "imu_feature_dim2 = 128\n",
    "fusion_output_dim = 128\n",
    "n_class = multimodal_embed_train_set.nClasses\n",
    "\n",
    "multimodal_model = AudioAndIMUFusion(audio_feature_dim1, \n",
    "                                     imu_feature_dim1, \n",
    "                                     audio_feature_dim2, \n",
    "                                     imu_feature_dim2, \n",
    "                                     fusion_output_dim,\n",
    "                                     n_class\n",
    "                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "pdrrgJs9pEja"
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'dict' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[41], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43msummary\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmultimodal_model\u001b[49m\u001b[43m)\u001b[49m)\n",
      "\u001b[0;31mTypeError\u001b[0m: 'dict' object is not callable"
     ]
    }
   ],
   "source": [
    "print(summary(multimodal_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OYmABjGAIKmS"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qt6cjGDciHeY",
    "outputId": "9ed52ccc-3e41-43cb-dd23-a2bca5545949"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Loss: 2.194183115373578, Accuracy: 0.22436951754385964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Loss: 2.193968421534488, Accuracy: 0.23081140350877194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2, Loss: 2.1938008509184184, Accuracy: 0.23848684210526316\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3, Loss: 2.1936462009162234, Accuracy: 0.24300986842105263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 27.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4, Loss: 2.193410216716298, Accuracy: 0.24917763157894737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5, Loss: 2.1932818805962278, Accuracy: 0.25219298245614036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6, Loss: 2.1930450598398843, Accuracy: 0.2583607456140351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7, Loss: 2.192833938096699, Accuracy: 0.2649396929824561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8, Loss: 2.1925896636226723, Accuracy: 0.2759046052631579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9, Loss: 2.192352102513899, Accuracy: 0.28495065789473684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10, Loss: 2.192117745416206, Accuracy: 0.296875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11, Loss: 2.19190948469597, Accuracy: 0.2975603070175439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12, Loss: 2.1915958680604635, Accuracy: 0.30633223684210525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13, Loss: 2.1913199006465445, Accuracy: 0.3159265350877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14, Loss: 2.1910416996269895, Accuracy: 0.3204495614035088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15, Loss: 2.190722754127101, Accuracy: 0.32497258771929827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16, Loss: 2.1904305115080716, Accuracy: 0.33292214912280704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 31.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17, Loss: 2.190071373655085, Accuracy: 0.3397752192982456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18, Loss: 2.189749600594504, Accuracy: 0.3540296052631579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19, Loss: 2.1893854852308308, Accuracy: 0.36828399122807015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20, Loss: 2.188965550640173, Accuracy: 0.37924890350877194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21, Loss: 2.188574657105563, Accuracy: 0.3959703947368421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22, Loss: 2.188086141619766, Accuracy: 0.40995065789473684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 31.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23, Loss: 2.187676245706123, Accuracy: 0.4095394736842105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24, Loss: 2.1871997389877054, Accuracy: 0.4243421052631579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25, Loss: 2.18669719445078, Accuracy: 0.42598684210526316\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 26.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26, Loss: 2.186139658877724, Accuracy: 0.44353070175438597\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27, Loss: 2.1855369325269733, Accuracy: 0.4557291666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28, Loss: 2.1849404259731897, Accuracy: 0.46710526315789475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29, Loss: 2.184230457272446, Accuracy: 0.4739583333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30, Loss: 2.1835963600560238, Accuracy: 0.49300986842105265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31, Loss: 2.182827167343675, Accuracy: 0.5145285087719298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32, Loss: 2.1820050749862405, Accuracy: 0.5272752192982456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33, Loss: 2.181138833363851, Accuracy: 0.5327576754385965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34, Loss: 2.1801978496083043, Accuracy: 0.5461896929824561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35, Loss: 2.179154027972305, Accuracy: 0.5648300438596491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36, Loss: 2.1780270191661097, Accuracy: 0.5689418859649122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37, Loss: 2.1768298358247993, Accuracy: 0.5681195175438597\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38, Loss: 2.1755009534066185, Accuracy: 0.5753837719298246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39, Loss: 2.1740432580312095, Accuracy: 0.5971765350877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 27.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40, Loss: 2.172477253696375, Accuracy: 0.6066337719298246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41, Loss: 2.17074587888885, Accuracy: 0.6160910087719298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42, Loss: 2.1689223992197135, Accuracy: 0.6278782894736842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43, Loss: 2.1667879882611727, Accuracy: 0.6380208333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44, Loss: 2.1645492629001013, Accuracy: 0.6403508771929824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 26.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45, Loss: 2.1619228480155006, Accuracy: 0.6355537280701754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 27.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46, Loss: 2.1589486557140685, Accuracy: 0.6225328947368421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47, Loss: 2.155693819648341, Accuracy: 0.6177357456140351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48, Loss: 2.1519038593559934, Accuracy: 0.620202850877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49, Loss: 2.147798358348378, Accuracy: 0.6152686403508771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50, Loss: 2.143156884009378, Accuracy: 0.5974506578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51, Loss: 2.137959028545179, Accuracy: 0.5726425438596491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52, Loss: 2.1318868461408114, Accuracy: 0.5531798245614035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53, Loss: 2.125082626677396, Accuracy: 0.5416666666666666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54, Loss: 2.117576101370025, Accuracy: 0.5078125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55, Loss: 2.1091084187490896, Accuracy: 0.4765625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56, Loss: 2.09979978778906, Accuracy: 0.4605263157894737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57, Loss: 2.08948584606773, Accuracy: 0.44243421052631576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58, Loss: 2.0778987407684326, Accuracy: 0.4231085526315789\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59, Loss: 2.066295360264025, Accuracy: 0.3998081140350877\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60, Loss: 2.0536481497580543, Accuracy: 0.38308662280701755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61, Loss: 2.040314222636976, Accuracy: 0.3788377192982456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 26.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62, Loss: 2.027423885830662, Accuracy: 0.36499451754385964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63, Loss: 2.0140277461001745, Accuracy: 0.36499451754385964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64, Loss: 2.000983215214913, Accuracy: 0.3578673245614035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65, Loss: 1.988324941250316, Accuracy: 0.35183662280701755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66, Loss: 1.9768498801348502, Accuracy: 0.3432017543859649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67, Loss: 1.9663994458683751, Accuracy: 0.34196820175438597\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68, Loss: 1.955798862273233, Accuracy: 0.35074013157894735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69, Loss: 1.9463492339117485, Accuracy: 0.3691063596491228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70, Loss: 1.9386420981925832, Accuracy: 0.4055646929824561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71, Loss: 1.9316917218660052, Accuracy: 0.40858004385964913\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72, Loss: 1.9253009704121373, Accuracy: 0.41968201754385964\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73, Loss: 1.9187896795440138, Accuracy: 0.3832236842105263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74, Loss: 1.9132465207785891, Accuracy: 0.3658168859649123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75, Loss: 1.9088310839837057, Accuracy: 0.3522478070175439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76, Loss: 1.9049855345173885, Accuracy: 0.3471765350877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77, Loss: 1.9010453893427264, Accuracy: 0.3541666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78, Loss: 1.8984284400939941, Accuracy: 0.3560855263157895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 27.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79, Loss: 1.8948384607047366, Accuracy: 0.35238486842105265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 27.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80, Loss: 1.8921494128411276, Accuracy: 0.3574561403508772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81, Loss: 1.8887460566403573, Accuracy: 0.37225877192982454\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 28.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82, Loss: 1.8868696752347445, Accuracy: 0.37979714912280704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 28.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83, Loss: 1.884484512764111, Accuracy: 0.38308662280701755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84, Loss: 1.882176633466754, Accuracy: 0.39555921052631576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85, Loss: 1.8803682724634807, Accuracy: 0.4088541666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 29.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86, Loss: 1.8795105114317776, Accuracy: 0.41488486842105265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:01<00:00, 30.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87, Loss: 1.8768789454510337, Accuracy: 0.4206414473684211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 25.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88, Loss: 1.875058270337289, Accuracy: 0.4457236842105263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89, Loss: 1.8737963856312267, Accuracy: 0.46066337719298245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90, Loss: 1.8715836311641492, Accuracy: 0.4809484649122807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91, Loss: 1.8702415792565596, Accuracy: 0.5072642543859649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92, Loss: 1.8689846344161452, Accuracy: 0.5182291666666666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93, Loss: 1.8675867611901802, Accuracy: 0.5313870614035088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94, Loss: 1.8654062852524875, Accuracy: 0.5323464912280702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95, Loss: 1.8634892848500035, Accuracy: 0.5271381578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 24.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96, Loss: 1.861466315754673, Accuracy: 0.5045230263157895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97, Loss: 1.8590222597122192, Accuracy: 0.5037006578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 24.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98, Loss: 1.856354322349816, Accuracy: 0.4897203947368421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 24.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99, Loss: 1.8537598952912449, Accuracy: 0.48739035087719296\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 100, Loss: 1.8509997129440308, Accuracy: 0.4683388157894737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 101, Loss: 1.8479704020316141, Accuracy: 0.46450109649122806\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 102, Loss: 1.8452358036710506, Accuracy: 0.4613486842105263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 103, Loss: 1.841891190461945, Accuracy: 0.4557291666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 104, Loss: 1.8385020954567088, Accuracy: 0.45490679824561403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 105, Loss: 1.8352773189544678, Accuracy: 0.4496984649122807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 106, Loss: 1.831532827594824, Accuracy: 0.4682017543859649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 107, Loss: 1.827790289594416, Accuracy: 0.48519736842105265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 108, Loss: 1.8232437079412895, Accuracy: 0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 109, Loss: 1.8193160170002987, Accuracy: 0.5098684210526315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 110, Loss: 1.8144570379926448, Accuracy: 0.5400219298245614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 111, Loss: 1.8093062430097346, Accuracy: 0.5219298245614035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 112, Loss: 1.804750118339271, Accuracy: 0.4765625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 113, Loss: 1.7988313373766447, Accuracy: 0.4575109649122807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 114, Loss: 1.7940554848888464, Accuracy: 0.4570997807017544\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 23.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 115, Loss: 1.7884508852373089, Accuracy: 0.4466831140350877\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:02<00:00, 22.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 116, Loss: 1.7830021736914652, Accuracy: 0.4439418859649123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 16.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 117, Loss: 1.7772473473297923, Accuracy: 0.43146929824561403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 118, Loss: 1.7715444857614082, Accuracy: 0.4361293859649123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 16.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 119, Loss: 1.7669897267692967, Accuracy: 0.4331140350877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 120, Loss: 1.7612137480785972, Accuracy: 0.43516995614035087\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 16.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 121, Loss: 1.7555984697843854, Accuracy: 0.4465460526315789\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 122, Loss: 1.7509016300502576, Accuracy: 0.4420230263157895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 123, Loss: 1.7459457736266286, Accuracy: 0.4465460526315789\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 124, Loss: 1.7410001461965996, Accuracy: 0.45285087719298245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 125, Loss: 1.7365215493921649, Accuracy: 0.4557291666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 126, Loss: 1.731098449021055, Accuracy: 0.45860745614035087\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 127, Loss: 1.7261441255870618, Accuracy: 0.45860745614035087\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 128, Loss: 1.722020057209751, Accuracy: 0.4603892543859649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 129, Loss: 1.717756047583463, Accuracy: 0.4595668859649123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 130, Loss: 1.713565232460959, Accuracy: 0.46011513157894735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 131, Loss: 1.7099061305062813, Accuracy: 0.4616228070175439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 132, Loss: 1.7055753992314924, Accuracy: 0.4777960526315789\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 133, Loss: 1.701400503777621, Accuracy: 0.4836896929824561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 134, Loss: 1.697499850340057, Accuracy: 0.48862390350877194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 135, Loss: 1.6937561181553624, Accuracy: 0.48807565789473684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 136, Loss: 1.6904932678791516, Accuracy: 0.4876644736842105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 137, Loss: 1.686907216122276, Accuracy: 0.49890350877192985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 138, Loss: 1.6832357080359208, Accuracy: 0.4960252192982456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 139, Loss: 1.6798090913839507, Accuracy: 0.49588815789473684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 140, Loss: 1.6767173273521556, Accuracy: 0.49136513157894735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 141, Loss: 1.6729628516916644, Accuracy: 0.49575109649122806\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 16.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 142, Loss: 1.6697233894415069, Accuracy: 0.4987664473684211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 16.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 143, Loss: 1.6668849995261745, Accuracy: 0.5023300438596491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 144, Loss: 1.6639115831308198, Accuracy: 0.5050712719298246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 16.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 145, Loss: 1.6608276032564933, Accuracy: 0.5017817982456141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 146, Loss: 1.6577759834758021, Accuracy: 0.5015076754385965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 147, Loss: 1.6547612892953973, Accuracy: 0.5010964912280702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 148, Loss: 1.6519467391465839, Accuracy: 0.501233552631579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 149, Loss: 1.6496032183630425, Accuracy: 0.5037006578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 150, Loss: 1.6471934423112033, Accuracy: 0.5052083333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 151, Loss: 1.6447709777898956, Accuracy: 0.5069901315789473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 152, Loss: 1.641581418221457, Accuracy: 0.5128837719298246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 153, Loss: 1.6394850768541034, Accuracy: 0.5086348684210527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 154, Loss: 1.6372258203071461, Accuracy: 0.5142543859649122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 155, Loss: 1.6351094162255002, Accuracy: 0.5138432017543859\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 156, Loss: 1.6327980443050987, Accuracy: 0.5106907894736842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 157, Loss: 1.6310204351157473, Accuracy: 0.5100054824561403\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 158, Loss: 1.6292054318545157, Accuracy: 0.5120614035087719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 159, Loss: 1.6268990374448007, Accuracy: 0.5138432017543859\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 160, Loss: 1.62504429984511, Accuracy: 0.5119243421052632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 161, Loss: 1.6232090330960458, Accuracy: 0.5121984649122807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 162, Loss: 1.6213527604153282, Accuracy: 0.5127467105263158\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 163, Loss: 1.620081665223105, Accuracy: 0.510827850877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 164, Loss: 1.6187152067820232, Accuracy: 0.5087719298245614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 165, Loss: 1.6168782376406485, Accuracy: 0.5102796052631579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 166, Loss: 1.6158853288282429, Accuracy: 0.5104166666666666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 167, Loss: 1.6142952755877846, Accuracy: 0.5112390350877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 168, Loss: 1.6131928008899354, Accuracy: 0.5067160087719298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 169, Loss: 1.6118136100601732, Accuracy: 0.5053453947368421\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 170, Loss: 1.6102737418392248, Accuracy: 0.5098684210526315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 171, Loss: 1.6087743357608193, Accuracy: 0.5112390350877193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 172, Loss: 1.607601013099938, Accuracy: 0.5137061403508771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 173, Loss: 1.6069268992072658, Accuracy: 0.5091831140350878\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 57/57 [00:03<00:00, 17.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 174, Loss: 1.6057585331431605, Accuracy: 0.5115131578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|██████████████████████▋                    | 30/57 [00:02<00:01, 14.38it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[43], line 28\u001b[0m\n\u001b[1;32m     26\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m     27\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[0;32m---> 28\u001b[0m summary[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mloss\u001b[39m\u001b[38;5;124m'\u001b[39m][e]\u001b[38;5;241m.\u001b[39mappend(\u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m     30\u001b[0m     \u001b[38;5;66;03m# Calculat accuracy\u001b[39;00m\n\u001b[1;32m     31\u001b[0m _, pred \u001b[38;5;241m=\u001b[39m x\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mtopk(\u001b[38;5;241m1\u001b[39m, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(multimodal_model.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "# Place on GPU\n",
    "multimodal_model.to(device)\n",
    "\n",
    "# Training loop\n",
    "n_epochs = 200\n",
    "summary = {'loss': [[] for _ in range(n_epochs)], 'acc': [[] for _ in range(n_epochs)]}\n",
    "for e in range(n_epochs):\n",
    "    for i, (audio_embeds, imu_embeds, labels) in enumerate(tqdm(multimodal_embed_train_loader)):\n",
    "        # Zero the grads\n",
    "        optimizer.zero_grad()\n",
    "        audio_embeds = audio_embeds.to(device)\n",
    "        imu_embeds = imu_embeds.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        # Run the Net\n",
    "        x = multimodal_model(audio_embeds, imu_embeds)\n",
    "#         print(x.shape)\n",
    "        # print(labels.shape)\n",
    "        # x = x.view(x.size()[:-1])\n",
    "\n",
    "        # Optimize net\n",
    "        loss = criterion(x, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        summary['loss'][e].append(loss.item())\n",
    "\n",
    "            # Calculat accuracy\n",
    "        _, pred = x.data.topk(1, dim=1)\n",
    "        pred = pred.view(pred.shape[:-1])\n",
    "        acc = torch.sum(pred == labels)/x.shape[0]\n",
    "        summary['acc'][e].append(acc.item())\n",
    " \n",
    "    print('Epoch: {}, Loss: {}, Accuracy: {}'.format(e, np.mean(summary['loss'][e]), np.mean(summary['acc'][e])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "id": "QWFHQNjmoJTT"
   },
   "outputs": [],
   "source": [
    "torch.save({\n",
    "    'model_config': {\n",
    "        'n_class': n_class,\n",
    "        'audio_feature_dim': audio_feature_dim,\n",
    "        'imu_feature_dim': imu_feature_dim,\n",
    "        'fusion_output_size': fusion_output_size,\n",
    "        },\n",
    "    'model_state_dict': multimodal_model.state_dict(),\n",
    "    'optimizer_state_dict': optimizer.state_dict(),\n",
    "    }, \n",
    "    './saved_models/multimodal_embed_model_{}-{}.pt'.format(datetime.datetime.now().date().month, datetime.datetime.now().date().day))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BYNOeExyIMvY"
   },
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 677,
     "status": "ok",
     "timestamp": 1679293569691,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "id": "TZw7YUk_nwLB",
    "outputId": "9339da6e-f6df-459f-cf22-a9ad87fd551f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint = torch.load('./saved_models/multimodal_embed_model_4-17.pt', map_location=device)\n",
    "\n",
    "multimodal_model = AudioAndIMUFusion(\n",
    "                               checkpoint['model_config']['n_class'],\n",
    "                               checkpoint['model_config']['audio_feature_dim'],\n",
    "                               checkpoint['model_config']['imu_feature_dim'],\n",
    "                               checkpoint['model_config']['fusion_output_size']\n",
    "                               )\n",
    "                                     \n",
    "multimodal_model.load_state_dict(checkpoint['model_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1191653,
     "status": "ok",
     "timestamp": 1679293569029,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "id": "tCaU73mbHMo1",
    "outputId": "88b02551-0afe-4c91-8496-463e0fc7608b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 1.495086904229789, Accuracy: 0.9202586206896551\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "multimodal_model.to(device)\n",
    "\n",
    "test_loss = []\n",
    "test_acc = []\n",
    "\n",
    "for i, (audio_embeds, imu_embeds, labels) in enumerate(multimodal_embed_test_loader):\n",
    "    # Zero the grads\n",
    "    optimizer.zero_grad()\n",
    "    audio_embeds = audio_embeds.to(device)\n",
    "    imu_embeds = imu_embeds.to(device)\n",
    "    labels = labels.to(device)\n",
    "\n",
    "    # Run the Net\n",
    "    x = multimodal_model(audio_embeds, imu_embeds)\n",
    "    # Optimize net\n",
    "    loss = criterion(x, labels.long())\n",
    "    test_loss.append(loss.item())\n",
    "\n",
    "        # Calculat accuracy\n",
    "    _, pred = x.data.topk(1, dim=1)\n",
    "    pred = pred.view(pred.shape[:-1])\n",
    "    acc = torch.sum(pred == labels)/x.shape[0]\n",
    "    test_acc.append(acc.item())\n",
    "    \n",
    "print('Loss: {}, Accuracy: {}'.format(np.mean(test_loss), np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3E9e3PvhH_pW"
   },
   "source": [
    "# Generate and save fusion embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "0tPeD1vNnbiF"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8, 128)\n"
     ]
    }
   ],
   "source": [
    "embeddings1 = 0\n",
    "for i, (audio_embeds, imu_embeds, labels) in enumerate(multimodal_embed_train_loader):\n",
    "    audio_embeds = audio_embeds.to(device)\n",
    "    imu_embeds = imu_embeds.to(device)\n",
    "    labels = labels.to(device)\n",
    "\n",
    "    embeddings1 = multimodal_model.extract_features(audio_embeds, imu_embeds).detach().cpu().numpy()\n",
    "print(embeddings1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fusion_embed_dataset(multimodal_embed_loader, multimodal_model, device, overwrite=False):\n",
    "    \n",
    "    save_path = multimodal_embed_loader.dataset.db_path.replace('MultimodalDataset', 'fusion')\n",
    "    config_file = './Multimodal/dataset_config.json'\n",
    "    multimodal_model.eval()\n",
    "    \n",
    "    if not os.path.isfile(save_path) or overwrite:\n",
    "        dataset = {}\n",
    "        dataset['embeddings'] = []\n",
    "        dataset['labels'] = []\n",
    "\n",
    "        for i, (audio_embeds, imu_embeds, labels) in enumerate(multimodal_embed_loader):\n",
    "            audio_embeds = audio_embeds.to(device)\n",
    "            imu_embeds = imu_embeds.to(device)\n",
    "\n",
    "            embeddings = multimodal_model.extract_features(audio_embeds, imu_embeds).detach().cpu().numpy()\n",
    "            \n",
    "            dataset['embeddings'].append(embeddings)\n",
    "            dataset['labels'].append(labels.numpy())\n",
    "            \n",
    "        dataset['embeddings'] = np.concatenate(dataset['embeddings'], axis=0)\n",
    "        dataset['labels'] = np.concatenate(dataset['labels'], axis=0)\n",
    "        np.savez(save_path, **dataset)\n",
    "        \n",
    "    else:\n",
    "        dataset = np.load(save_path, allow_pickle=True)\n",
    "    \n",
    "    with open(config_file, 'r') as f:\n",
    "        dataset_config = json.load(f)\n",
    "        \n",
    "    return dataset, dataset_config\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4287700,
     "status": "ok",
     "timestamp": 1679297859152,
     "user": {
      "displayName": "LIYING HAN",
      "userId": "12884455188191918408"
     },
     "user_tz": 420
    },
    "id": "3WXyF002H_SE",
    "outputId": "0686cb71-89d8-4485-8e79-98de04fa9586"
   },
   "outputs": [],
   "source": [
    "fusion_dataset, fusion_dataset_config = get_fusion_embed_dataset(multimodal_embed_train_loader, multimodal_model, device)\n",
    "fusion_embed_train_set = FusionEmbed(fusion_dataset, fusion_dataset_config)\n",
    "\n",
    "fusion_dataset, fusion_dataset_config = get_fusion_embed_dataset(multimodal_embed_test_loader, multimodal_model, device)\n",
    "fusion_embed_test_set = FusionEmbed(fusion_dataset, fusion_dataset_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "7qQqe1Tt-a0o",
    "outputId": "9e57a8d8-382a-4572-ee6b-5af762bb8f25"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "480\n",
      "12102\n",
      "(900, 128)\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "print(len(audio_train_set.sounds))\n",
    "print(len(imu_train_set.imus))\n",
    "print(fusion_embed_train_set.embeddings.shape)\n",
    "print(len(fusion_embed_test_set.label_mapping))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Qw_mFIdsNhDC"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyPp7qHkEhEWMOTHswUVmPFw",
   "collapsed_sections": [
    "mX3JEcPjWX2E",
    "IaoQsFL_agq1",
    "dgVasASbPomf"
   ],
   "machine_shape": "hm",
   "provenance": [],
   "toc_visible": true
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "291.2px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}