scripts/Segformer_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Original model (No - LoRA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import SegformerForSemanticSegmentation, SegformerImageProcessor\n",
    "\n",
    "# Load the base model\n",
    "model_directory = 'guimCC/segformer-v0-gta'\n",
    "original_model = SegformerForSemanticSegmentation.from_pretrained(model_directory)\n",
    "\n",
    "processor = SegformerImageProcessor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LoRA Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from peft import PeftConfig, PeftModel\n",
    "from transformers import SegformerForSemanticSegmentation, SegformerImageProcessor\n",
    "\n",
    "\n",
    "base_model_id = \"guimCC/segformer-v0-gta\"\n",
    "lora_model_id = \"guimCC/segformer-v0-gta-cityscapes\"\n",
    "\n",
    "processor = SegformerImageProcessor()\n",
    "\n",
    "model = SegformerForSemanticSegmentation.from_pretrained(base_model_id)\n",
    "\n",
    "config = PeftConfig.from_pretrained(lora_model_id)\n",
    "\n",
    "# Load the Lora model\n",
    "lora_model = PeftModel.from_pretrained(model, lora_model_id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### GTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "265c236fea7b4d28a0e9fd8ef295e7fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/53 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6deb7106f00481d9fba1e80a3113130",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc4467aa0410469bb3b1e92bfa0d30ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/26 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb4e5d0c01034ace8a54b4543c8c621c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/53 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fb8f3112d4f4bccbb3b7beb4d8e7acc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfd5f9c106fb410e80d1e832f0cf100c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Resolving data files:   0%|          | 0/26 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c06c64d5fccf41348c08f265789b6a44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading dataset shards:   0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "# Load dataset from hugginface\n",
    "hf_datasets = load_dataset(\"guimCC/gta5-cityscapes-labeling\")\n",
    "\n",
    "gta_train_ds = hf_datasets[\"train\"]\n",
    "gta_test_ds = hf_datasets[\"test\"].train_test_split(test_size=0.1)['test']\n",
    "gta_val_ds = hf_datasets[\"validation\"].train_test_split(test_size=0.1)['test']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cityscapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_from_disk\n",
    "\n",
    "# Load the dataset from disk\n",
    "path_to_cityscapes_dataset = \"./../cityscapes_train_1000_dataset_v3\"\n",
    "loaded_dataset = load_from_disk(path_to_cityscapes_dataset)\n",
    "\n",
    "# Prepare train and test splits\n",
    "loaded_dataset = loaded_dataset.train_test_split(test_size=0.1)\n",
    "cty_test_ds = loaded_dataset[\"test\"]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### GTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/huggingface_hub/file_download.py:671: FutureWarning: 'cached_download' is the legacy way to download files from the HF hub, please consider upgrading to 'hf_hub_download'\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "import numpy as np\n",
    "import json\n",
    "from huggingface_hub import cached_download, hf_hub_url\n",
    "import torch\n",
    "from torch import nn\n",
    "import evaluate\n",
    "\n",
    "# ID handling\n",
    "\n",
    "repo_id = \"huggingface/label-files\"\n",
    "filename = \"cityscapes-id2label.json\"\n",
    "id2label = json.load(open(cached_download(hf_hub_url(repo_id, filename, repo_type=\"dataset\")), \"r\"))\n",
    "id2label = {int(k): v for k, v in id2label.items()}\n",
    "label2id = {v: k for k, v in id2label.items()}\n",
    "id2label[19] = 'ignore'\n",
    "label2id['ignore'] = 19\n",
    "num_labels = len(id2label)\n",
    "\n",
    "# Transformations\n",
    "\n",
    "def val_transforms(example_batch):\n",
    "    images = [Image.fromarray(np.array(x, dtype=np.uint8)) for x in example_batch['image']]\n",
    "    labels = [Image.fromarray(np.array(x, dtype=np.uint8), mode='L') for x in example_batch['mask']]\n",
    "    \n",
    "    # Ensure labels are within the expected range\n",
    "    labels = [Image.fromarray(np.minimum(np.array(label), num_labels - 1), mode='L') for label in labels]\n",
    "    \n",
    "    inputs = processor(images=images, segmentation_maps=labels, return_tensors=\"pt\")\n",
    "    return inputs\n",
    "\n",
    "gta_test_ds.set_transform(val_transforms)\n",
    "\n",
    "# Metrics\n",
    "\n",
    "metric = evaluate.load(\"mean_iou\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "  with torch.no_grad():\n",
    "    logits, labels = eval_pred\n",
    "    logits_tensor = torch.from_numpy(logits)\n",
    "    # scale the logits to the size of the label\n",
    "    logits_tensor = nn.functional.interpolate(\n",
    "        logits_tensor,\n",
    "        size=labels.shape[-2:],\n",
    "        mode=\"bilinear\",\n",
    "        align_corners=False,\n",
    "    ).argmax(dim=1)\n",
    "\n",
    "    pred_labels = logits_tensor.detach().cpu().numpy()\n",
    "    metrics = metric.compute(\n",
    "        predictions=pred_labels,\n",
    "        references=labels,\n",
    "        num_labels=len(id2label),\n",
    "        ignore_index=19,\n",
    "        reduce_labels=processor.do_reduce_labels,\n",
    "    )\n",
    "    \n",
    "    # add per category metrics as individual key-value pairs\n",
    "    per_category_accuracy = metrics.pop(\"per_category_accuracy\").tolist()\n",
    "    per_category_iou = metrics.pop(\"per_category_iou\").tolist()\n",
    "\n",
    "    metrics.update({f\"accuracy_{id2label[i]}\": v for i, v in enumerate(per_category_accuracy)})\n",
    "    metrics.update({f\"iou_{id2label[i]}\": v for i, v in enumerate(per_category_iou)})\n",
    "    \n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cityscapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import numpy as np\n",
    "import json\n",
    "from huggingface_hub import cached_download, hf_hub_url\n",
    "from torchvision.transforms import ColorJitter\n",
    "from torch import nn\n",
    "import evaluate\n",
    "\n",
    "# ID handling\n",
    "\n",
    "repo_id = \"huggingface/label-files\"\n",
    "filename = \"cityscapes-id2label.json\"\n",
    "id2label = json.load(open(cached_download(hf_hub_url(repo_id, filename, repo_type=\"dataset\")), \"r\"))\n",
    "id2label = {int(k): v for k, v in id2label.items()}\n",
    "label2id = {v: k for k, v in id2label.items()}\n",
    "\n",
    "\n",
    "id2label[19] = 'ignore'\n",
    "label2id['ignore'] = 19\n",
    "num_labels = len(id2label)\n",
    "\n",
    "\n",
    "# Transformations\n",
    "jitter = ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.1)\n",
    "\n",
    "def handle_grayscale_image(image):\n",
    "    np_image = np.array(image)\n",
    "    if np_image.ndim == 2:\n",
    "        tiled_image = np.tile(np.expand_dims(np_image, -1), 3)\n",
    "        return Image.fromarray(tiled_image)\n",
    "    else:\n",
    "        return Image.fromarray(np_image)\n",
    "\n",
    "\n",
    "def val_transforms(example_batch):\n",
    "    images = [handle_grayscale_image(x) for x in example_batch[\"image\"]]\n",
    "    labels = [x for x in example_batch[\"annotation\"]]\n",
    "    inputs = processor(images, labels)\n",
    "    return inputs\n",
    "\n",
    "cty_test_ds.set_transform(val_transforms)\n",
    "\n",
    "# Metrics\n",
    "\n",
    "metric = evaluate.load(\"mean_iou\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    \n",
    "    with torch.no_grad(): # Don't want to store the gradients while computing this metric since it's validation\n",
    "        logits, labels = eval_pred\n",
    "        logits_tensor = torch.from_numpy(logits)\n",
    "        # scale the logits to the size of the label\n",
    "        logits_tensor = nn.functional.interpolate(\n",
    "            logits_tensor,\n",
    "            size=labels.shape[-2:],\n",
    "            mode=\"bilinear\",\n",
    "            align_corners=False,\n",
    "        ).argmax(dim=1)\n",
    "\n",
    "        pred_labels = logits_tensor.detach().cpu().numpy()\n",
    "        # currently using _compute instead of compute\n",
    "        # see this issue for more info: https://github.com/huggingface/evaluate/pull/328#issuecomment-1286866576\n",
    "        metrics = metric._compute(\n",
    "            predictions=pred_labels,\n",
    "            references=labels,\n",
    "            num_labels=len(id2label),\n",
    "            ignore_index=0,\n",
    "            reduce_labels=processor.do_reduce_labels,\n",
    "        )\n",
    "\n",
    "        # add per category metrics as individual key-value pairs\n",
    "        per_category_accuracy = metrics.pop(\"per_category_accuracy\").tolist()\n",
    "        per_category_iou = metrics.pop(\"per_category_iou\").tolist()\n",
    "\n",
    "        metrics.update({f\"accuracy_{id2label[i]}\": v for i, v in enumerate(per_category_accuracy)})\n",
    "        metrics.update({f\"iou_{id2label[i]}\": v for i, v in enumerate(per_category_iou)})\n",
    "\n",
    "        return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ORIGINAL model (SegFormer) and GTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# Assuming evaluation doesn't require gradient updates\n",
    "original_gta_eval_args = TrainingArguments(\n",
    "    output_dir='./segformer_evaluation/sgf-v0-cty',  # Directory to store evaluation results\n",
    "    do_train=False,\n",
    "    do_eval=True,\n",
    "    eval_steps=10,\n",
    "    logging_steps=5,\n",
    "    per_device_eval_batch_size=10,\n",
    "    dataloader_num_workers=0,\n",
    "    remove_unused_columns=False,\n",
    "\n",
    ")\n",
    "\n",
    "original_gta_trainer = Trainer(\n",
    "    model=original_model,\n",
    "    args=original_gta_eval_args,\n",
    "    eval_dataset=gta_test_ds,\n",
    "    compute_metrics=compute_metrics  # Your metrics function as defined earlier\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/torch/nn/modules/conv.py:456: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at /opt/conda/conda-bld/pytorch_1712608839953/work/aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)\n",
      "  return F.conv2d(input, weight, bias, self.stride,\n",
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='8' max='8' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [8/8 08:18]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------+---------------------+\n",
      "|           Metric            |        Value        |\n",
      "+-----------------------------+---------------------+\n",
      "|          eval_loss          | 0.5294152498245239  |\n",
      "|        eval_mean_iou        | 0.3657142979584772  |\n",
      "|     eval_mean_accuracy      | 0.4490071911589912  |\n",
      "|    eval_overall_accuracy    | 0.8048245155313702  |\n",
      "|     eval_accuracy_road      |         nan         |\n",
      "|   eval_accuracy_sidewalk    | 0.8854939033079142  |\n",
      "|   eval_accuracy_building    | 0.9221117802639245  |\n",
      "|     eval_accuracy_wall      | 0.4531118964645282  |\n",
      "|     eval_accuracy_fence     | 0.18434359327693234 |\n",
      "|     eval_accuracy_pole      | 0.19585926209927154 |\n",
      "| eval_accuracy_traffic light |         0.0         |\n",
      "| eval_accuracy_traffic sign  |         0.0         |\n",
      "|  eval_accuracy_vegetation   | 0.8764858918100011  |\n",
      "|    eval_accuracy_terrain    | 0.5068282016669517  |\n",
      "|      eval_accuracy_sky      | 0.9757929543468974  |\n",
      "|    eval_accuracy_person     | 0.44411699445284925 |\n",
      "|     eval_accuracy_rider     |         0.0         |\n",
      "|      eval_accuracy_car      | 0.9087279597769711  |\n",
      "|     eval_accuracy_truck     | 0.7945874327535539  |\n",
      "|      eval_accuracy_bus      | 0.7488686605981795  |\n",
      "|     eval_accuracy_train     |         0.0         |\n",
      "|  eval_accuracy_motorcycle   |         0.0         |\n",
      "|    eval_accuracy_bicycle    |         0.0         |\n",
      "|    eval_accuracy_ignore     | 0.6348081012028566  |\n",
      "|        eval_iou_road        |         0.0         |\n",
      "|      eval_iou_sidewalk      | 0.8009334491598179  |\n",
      "|      eval_iou_building      | 0.7266711719517865  |\n",
      "|        eval_iou_wall        | 0.3215004708425464  |\n",
      "|       eval_iou_fence        | 0.15264261537870766 |\n",
      "|        eval_iou_pole        | 0.1628374567834637  |\n",
      "|   eval_iou_traffic light    |         0.0         |\n",
      "|    eval_iou_traffic sign    |         0.0         |\n",
      "|     eval_iou_vegetation     | 0.7074546379991478  |\n",
      "|      eval_iou_terrain       | 0.4421839475528206  |\n",
      "|        eval_iou_sky         | 0.9217683284767816  |\n",
      "|       eval_iou_person       | 0.3254464021758411  |\n",
      "|       eval_iou_rider        |         0.0         |\n",
      "|        eval_iou_car         | 0.7893534205158645  |\n",
      "|       eval_iou_truck        | 0.6923464075549707  |\n",
      "|        eval_iou_bus         | 0.7166837933842275  |\n",
      "|       eval_iou_train        |         0.0         |\n",
      "|     eval_iou_motorcycle     |         0.0         |\n",
      "|      eval_iou_bicycle       |         0.0         |\n",
      "|       eval_iou_ignore       | 0.5544638573935684  |\n",
      "|        eval_runtime         |      691.0586       |\n",
      "|   eval_samples_per_second   |        0.891        |\n",
      "|    eval_steps_per_second    |        0.012        |\n",
      "+-----------------------------+---------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--mean_iou/9e450724f21f05592bfb0255fe2fa576df8171fa060d11121d8aecfff0db80d0/mean_iou.py:260: RuntimeWarning: invalid value encountered in divide\n",
      "  acc = total_area_intersect / total_area_label\n"
     ]
    }
   ],
   "source": [
    "# NOTE: Since the dataset is BIG, it takes several minutes to perform the evaluation\n",
    "\n",
    "from tabulate import tabulate\n",
    "\n",
    "original_gta_eval_results = original_gta_trainer.evaluate()\n",
    "\n",
    "# Convert the dictionary to a list of lists for tabulate\n",
    "data = [[key, value] for key, value in original_gta_eval_results.items()]\n",
    "\n",
    "# Create the table\n",
    "table = tabulate(data, headers=[\"Metric\", \"Value\"], tablefmt=\"pretty\")\n",
    "\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ORIGINAL model (SegFormer) and CityScapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# Assuming evaluation doesn't require gradient updates\n",
    "original_cty_eval_args = TrainingArguments(\n",
    "    output_dir='./segformer_evaluation/sgf-v0-cty',  # Directory to store evaluation results\n",
    "    do_train=False,\n",
    "    do_eval=True,\n",
    "    eval_steps=10,\n",
    "    logging_steps=5,\n",
    "    per_device_eval_batch_size=10,\n",
    "    dataloader_num_workers=0,\n",
    "    remove_unused_columns=False,\n",
    "\n",
    ")\n",
    "\n",
    "original_cty_trainer = Trainer(\n",
    "    model=original_model,\n",
    "    args=original_cty_eval_args,\n",
    "    eval_dataset=cty_test_ds,\n",
    "    compute_metrics=compute_metrics  # Your metrics function as defined earlier\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/torch/nn/modules/conv.py:456: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at /opt/conda/conda-bld/pytorch_1712608839953/work/aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)\n",
      "  return F.conv2d(input, weight, bias, self.stride,\n",
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2' max='2' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2/2 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------+----------------------+\n",
      "|           Metric            |        Value         |\n",
      "+-----------------------------+----------------------+\n",
      "|          eval_loss          |  1.3254294395446777  |\n",
      "|        eval_mean_iou        | 0.08238448517048705  |\n",
      "|     eval_mean_accuracy      |  0.3562056136454559  |\n",
      "|    eval_overall_accuracy    |  0.5914905134656971  |\n",
      "|     eval_accuracy_road      |         nan          |\n",
      "|   eval_accuracy_sidewalk    |  0.2697358350534592  |\n",
      "|   eval_accuracy_building    |  0.6456586171711773  |\n",
      "|     eval_accuracy_wall      | 0.34087879595744164  |\n",
      "|     eval_accuracy_fence     |  0.1841422404263489  |\n",
      "|     eval_accuracy_pole      |  0.1184297041191259  |\n",
      "| eval_accuracy_traffic light |         0.0          |\n",
      "| eval_accuracy_traffic sign  |         0.0          |\n",
      "|  eval_accuracy_vegetation   |  0.8448956095643261  |\n",
      "|    eval_accuracy_terrain    |  0.8293257496975742  |\n",
      "|      eval_accuracy_sky      |  0.9877007309439114  |\n",
      "|    eval_accuracy_person     |  0.281972937581842   |\n",
      "|     eval_accuracy_rider     |         0.0          |\n",
      "|      eval_accuracy_car      |  0.7281122007400996  |\n",
      "|     eval_accuracy_truck     |  0.5497908783110934  |\n",
      "|      eval_accuracy_bus      | 0.27485213240635054  |\n",
      "|     eval_accuracy_train     |         nan          |\n",
      "|  eval_accuracy_motorcycle   |         0.0          |\n",
      "|    eval_accuracy_bicycle    |         0.0          |\n",
      "|    eval_accuracy_ignore     |         nan          |\n",
      "|        eval_iou_road        |         0.0          |\n",
      "|      eval_iou_sidewalk      | 0.017473393722426917 |\n",
      "|      eval_iou_building      | 0.02086228299625734  |\n",
      "|        eval_iou_wall        | 0.07631868817589382  |\n",
      "|       eval_iou_fence        |  0.1576799868475461  |\n",
      "|        eval_iou_pole        |  0.0836467277975578  |\n",
      "|   eval_iou_traffic light    |         0.0          |\n",
      "|    eval_iou_traffic sign    |         0.0          |\n",
      "|     eval_iou_vegetation     | 0.032802702450772235 |\n",
      "|      eval_iou_terrain       | 0.24182858227320642  |\n",
      "|        eval_iou_sky         |  0.368423331858066   |\n",
      "|       eval_iou_person       | 0.020007329826716703 |\n",
      "|       eval_iou_rider        |         0.0          |\n",
      "|        eval_iou_car         | 0.06797167152507307  |\n",
      "|       eval_iou_truck        | 0.20680484454988138  |\n",
      "|        eval_iou_bus         |  0.2714856762158561  |\n",
      "|       eval_iou_train        |         nan          |\n",
      "|     eval_iou_motorcycle     |         0.0          |\n",
      "|      eval_iou_bicycle       |         0.0          |\n",
      "|       eval_iou_ignore       |         0.0          |\n",
      "|        eval_runtime         |       15.6516        |\n",
      "|   eval_samples_per_second   |        6.389         |\n",
      "|    eval_steps_per_second    |        0.128         |\n",
      "+-----------------------------+----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--mean_iou/9e450724f21f05592bfb0255fe2fa576df8171fa060d11121d8aecfff0db80d0/mean_iou.py:259: RuntimeWarning: invalid value encountered in divide\n",
      "  iou = total_area_intersect / total_area_union\n",
      "/home/gcasadella/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--mean_iou/9e450724f21f05592bfb0255fe2fa576df8171fa060d11121d8aecfff0db80d0/mean_iou.py:260: RuntimeWarning: invalid value encountered in divide\n",
      "  acc = total_area_intersect / total_area_label\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "original_cty_eval_results = original_cty_trainer.evaluate()\n",
    "\n",
    "# Convert the dictionary to a list of lists for tabulate\n",
    "data = [[key, value] for key, value in original_cty_eval_results.items()]\n",
    "\n",
    "# Create the table\n",
    "table = tabulate(data, headers=[\"Metric\", \"Value\"], tablefmt=\"pretty\")\n",
    "\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ORIGINAL model (SegFormer) and CityScapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# Assuming evaluation doesn't require gradient updates\n",
    "lora_cty_eval_args = TrainingArguments(\n",
    "    output_dir='./segformer_evaluation/sgf-v0-lora-cty',  # Directory to store evaluation results\n",
    "    do_train=False,\n",
    "    do_eval=True,\n",
    "    eval_steps=10,\n",
    "    logging_steps=5,\n",
    "    per_device_eval_batch_size=10,\n",
    "    dataloader_num_workers=0,\n",
    "    remove_unused_columns=False,\n",
    "    label_names=[\"labels\"]\n",
    "\n",
    ")\n",
    "\n",
    "# Compute Metrics Issue: https://github.com/huggingface/transformers/issues/29186\n",
    "lora_cty_trainer = Trainer(\n",
    "    model=lora_model,\n",
    "    args=lora_cty_eval_args,\n",
    "    eval_dataset=cty_test_ds,\n",
    "    compute_metrics=compute_metrics  # Your metrics function as defined earlier\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/torch/nn/modules/conv.py:456: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at /opt/conda/conda-bld/pytorch_1712608839953/work/aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)\n",
      "  return F.conv2d(input, weight, bias, self.stride,\n",
      "/home/gcasadella/miniconda3/envs/hugginface/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2' max='2' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2/2 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------+----------------------+\n",
      "|           Metric            |        Value         |\n",
      "+-----------------------------+----------------------+\n",
      "|          eval_loss          | 0.17324452102184296  |\n",
      "|        eval_mean_iou        | 0.18430768490184848  |\n",
      "|     eval_mean_accuracy      |  0.8627898017208178  |\n",
      "|    eval_overall_accuracy    |  0.9071604023011562  |\n",
      "|     eval_accuracy_road      |         nan          |\n",
      "|   eval_accuracy_sidewalk    |  0.8530138924322193  |\n",
      "|   eval_accuracy_building    |  0.8949419367377679  |\n",
      "|     eval_accuracy_wall      |  0.9192215409224207  |\n",
      "|     eval_accuracy_fence     |  0.8682961170132303  |\n",
      "|     eval_accuracy_pole      |  0.8069361180945014  |\n",
      "| eval_accuracy_traffic light |  0.740641158221303   |\n",
      "| eval_accuracy_traffic sign  |  0.830272932142559   |\n",
      "|  eval_accuracy_vegetation   |  0.8866166828131362  |\n",
      "|    eval_accuracy_terrain    |  0.9078715592436172  |\n",
      "|      eval_accuracy_sky      |  0.994060500236837   |\n",
      "|    eval_accuracy_person     |  0.6593918230758039  |\n",
      "|     eval_accuracy_rider     |  0.7969170362829928  |\n",
      "|      eval_accuracy_car      |  0.9601327545199916  |\n",
      "|     eval_accuracy_truck     |  0.8822943636725752  |\n",
      "|      eval_accuracy_bus      |  0.8789301649891045  |\n",
      "|     eval_accuracy_train     |         nan          |\n",
      "|  eval_accuracy_motorcycle   |  0.9327532712801835  |\n",
      "|    eval_accuracy_bicycle    |  0.8551347775756574  |\n",
      "|    eval_accuracy_ignore     |         nan          |\n",
      "|        eval_iou_road        |         0.0          |\n",
      "|      eval_iou_sidewalk      |  0.1766750538248035  |\n",
      "|      eval_iou_building      | 0.03195902870149398  |\n",
      "|        eval_iou_wall        | 0.18423634791563445  |\n",
      "|       eval_iou_fence        |  0.360287958051986   |\n",
      "|        eval_iou_pole        | 0.10142635367478099  |\n",
      "|   eval_iou_traffic light    |  0.2084794853508376  |\n",
      "|    eval_iou_traffic sign    | 0.18915822450832648  |\n",
      "|     eval_iou_vegetation     | 0.040099290154139135 |\n",
      "|      eval_iou_terrain       | 0.28870436901175883  |\n",
      "|        eval_iou_sky         | 0.37355114320516264  |\n",
      "|       eval_iou_person       |  0.0308082404565493  |\n",
      "|       eval_iou_rider        |  0.1889948440275215  |\n",
      "|        eval_iou_car         | 0.08488876677596577  |\n",
      "|       eval_iou_truck        | 0.21007373920190947  |\n",
      "|        eval_iou_bus         | 0.38212803392583233  |\n",
      "|       eval_iou_train        |         0.0          |\n",
      "|     eval_iou_motorcycle     |  0.4795741434318213  |\n",
      "|      eval_iou_bicycle       | 0.17080099091659787  |\n",
      "|       eval_iou_ignore       |         nan          |\n",
      "|        eval_runtime         |       14.5684        |\n",
      "|   eval_samples_per_second   |        6.864         |\n",
      "|    eval_steps_per_second    |        0.137         |\n",
      "+-----------------------------+----------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gcasadella/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--mean_iou/9e450724f21f05592bfb0255fe2fa576df8171fa060d11121d8aecfff0db80d0/mean_iou.py:259: RuntimeWarning: invalid value encountered in divide\n",
      "  iou = total_area_intersect / total_area_union\n",
      "/home/gcasadella/.cache/huggingface/modules/evaluate_modules/metrics/evaluate-metric--mean_iou/9e450724f21f05592bfb0255fe2fa576df8171fa060d11121d8aecfff0db80d0/mean_iou.py:260: RuntimeWarning: invalid value encountered in divide\n",
      "  acc = total_area_intersect / total_area_label\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "lora_cty_eval_results = lora_cty_trainer.evaluate()\n",
    "\n",
    "# Convert the dictionary to a list of lists for tabulate\n",
    "data = [[key, value] for key, value in lora_cty_eval_results.items()]\n",
    "\n",
    "# Create the table\n",
    "table = tabulate(data, headers=[\"Metric\", \"Value\"], tablefmt=\"pretty\")\n",
    "\n",
    "print(table)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hugginface",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}