pfuji-size-norm-model-trainer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Model Training, Inference and Evaluation Module\n",
        "\n",
        "In this notebook we first train the segmentation model, plot logging plots and perform evaluation on the trained apple segmentation task.\n",
        "\n",
        "#### 1. Open3D's RanLA-Net Model Setup and Training\n",
        "\n",
        "In this section we declare a custom dataloader object for Open3D's segmentation model training and then plot the tensorboard logger plots for better model learning understanding.\n",
        "\n",
        "#### 2. Model Performance Inference and Evaluation\n",
        "\n",
        "In this section we evaluate performance of the trained segmentation model, and also plot the final segmentation outputs for better qualitative understanding."
      ],
      "metadata": {
        "id": "asbF3zF0b12r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# for loading the dataset into the runtime from google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "Pf3sLG6kwI47"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# installing open3d library for importing RandLa-Net model implementation\n",
        "!pip install open3d"
      ],
      "metadata": {
        "id": "Pd7Nrl0b2nH-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# cloning repository for building the creating custom dataloaders\n",
        "!git clone https://github.com/isl-org/Open3D-ML"
      ],
      "metadata": {
        "id": "aSk-ko8sqOPt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# installing compatible pytorch version\n",
        "!pip install -r Open3D-ML/requirements-torch-cuda.txt"
      ],
      "metadata": {
        "id": "DI4gg7zW5pQ2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# checking the torch installed version\n",
        "# latest version 1.13.0+cu116 is not compatible with the \n",
        "import torch\n",
        "print(torch.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lu8AWhzC5q8U",
        "outputId": "b02b8170-0824-427e-f6ac-9d57f236c31a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.13.1+cu116\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# setting up for the Open3D-ML package after all the environment restarts\n",
        "import sys\n",
        "sys.path.insert(0,'Open3D-ML')"
      ],
      "metadata": {
        "id": "9U5K7Eib5snN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# checking the path variables for the Open3D-ML package source code\n",
        "from ml3d.datasets.base_dataset import BaseDataset, BaseDatasetSplit\n",
        "from ml3d.utils import make_dir, DATASET"
      ],
      "metadata": {
        "id": "3J967SNc6Wim"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# extracting the fuji data dataset for model training into current runtime\n",
        "# TODO: update the corresponding path variables for \n",
        "!unzip drive/MyDrive/point-cloud-prototyping/datasets/fuji-norm-complete-dataset.zip -d ."
      ],
      "metadata": {
        "id": "dd8w68fQw8zs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# listing paths of all the splits of the norm pfuji size dataset\n",
        "!ls fuji-norm-dataset/train"
      ],
      "metadata": {
        "id": "b-dVAQRWmH4c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# cleaning the fuji-sfm dataset from the complete dataset to train only on pfuji size dataset\n",
        "import re, os, shutil\n",
        "folder_paths = ['fuji-norm-dataset/train',\n",
        "                'fuji-norm-dataset/test',\n",
        "                'fuji-norm-dataset/valid']\n",
        "\n",
        "for folder in folder_paths:\n",
        "    for filename in os.listdir(folder):\n",
        "        file_path = os.path.join(folder, filename)\n",
        "        nums = re.findall(r'\\d+', file_path)\n",
        "        if int(nums[0]) < 160:\n",
        "            try:\n",
        "                if os.path.isfile(file_path) or os.path.islink(file_path):\n",
        "                    os.unlink(file_path)\n",
        "                elif os.path.isdir(file_path):\n",
        "                    shutil.rmtree(file_path)\n",
        "            except Exception as e:\n",
        "                print('Failed to delete %s. Reason: %s' % (file_path, e))"
      ],
      "metadata": {
        "id": "aJf28P5SmBLh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# utilizing only the normals information for training and removing the rgb during training"
      ],
      "metadata": {
        "id": "GXFdfej5mBkC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1. Open3D's RanLA-Net Model Setup and Training"
      ],
      "metadata": {
        "id": "BF4hrIeiuqPD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0CyA1iEpKFMc"
      },
      "outputs": [],
      "source": [
        "# custom data set loader code, loads the '.npy' files from train, val & test directories\n",
        "import numpy as np\n",
        "import os, sys, glob, pickle\n",
        "from pathlib import Path\n",
        "from os.path import join, exists, dirname, abspath\n",
        "from sklearn.neighbors import KDTree\n",
        "from tqdm import tqdm\n",
        "import logging\n",
        "\n",
        "# from .base_dataset import BaseDataset, BaseDatasetSplit\n",
        "# from ..utils import make_dir, DATASET\n",
        "# updated paths for the ml3d github source code usage\n",
        "from ml3d.datasets.base_dataset import BaseDataset, BaseDatasetSplit\n",
        "from ml3d.utils import make_dir, DATASET\n",
        "\n",
        "log = logging.getLogger(__name__)\n",
        "# Expect point clouds to be in npy format with train, val and test files in separate folders\n",
        "# Expected format of npy files : ['x', 'y', 'z', 'feat_1', 'feat_2', ........,'feat_n', 'class']\n",
        "# For test files, format should be : ['x', 'y', 'z', 'feat_1', 'feat_2', ........,'feat_n', 'class']\n",
        "\n",
        "class Custom3DSplit(BaseDatasetSplit):\n",
        "    \"\"\"This class is used to create a custom dataset split.\n",
        "    Initialize the class.\n",
        "    Args:\n",
        "        dataset: The dataset to split.\n",
        "        split: A string identifying the dataset split that is usually one of\n",
        "        'training', 'test', 'validation', or 'all'.\n",
        "        **kwargs: The configuration of the model as keyword arguments.\n",
        "    Returns:\n",
        "        A dataset split object providing the requested subset of the data.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, dataset, split='training'):\n",
        "        super().__init__(dataset, split=split)\n",
        "        self.cfg = dataset.cfg\n",
        "        path_list = dataset.get_split_list(split)\n",
        "        log.info(\"Found {} pointclouds for {}\".format(len(path_list), split))\n",
        "        self.path_list = path_list\n",
        "        self.split = split\n",
        "        self.dataset = dataset\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.path_list)\n",
        "\n",
        "    def get_data(self, idx):\n",
        "        pc_path = self.path_list[idx]\n",
        "        data = np.load(pc_path)\n",
        "  \n",
        "        points = np.array(data[:, :3], dtype=np.float32)\n",
        "        feat = np.array(data[:, 6:9], dtype=np.float32)\n",
        "        labels = np.array(data[:, 9], dtype=np.int32).reshape((-1,))\n",
        "\n",
        "        data = {'point': points,  'feat': feat, 'label': labels}\n",
        "        return data\n",
        "\n",
        "    def get_attr(self, idx):\n",
        "        pc_path = Path(self.path_list[idx])\n",
        "        name = pc_path.name.replace('.npy', '')\n",
        "        attr = {'name': name, 'path': str(pc_path), 'split': self.split}\n",
        "        return attr\n",
        "\n",
        "\n",
        "class Custom3D(BaseDataset):\n",
        "    \"\"\"A template for customized dataset that you can use with a dataloader to\n",
        "    feed data when training a model. This inherits all functions from the base\n",
        "    dataset and can be modified by users. Initialize the function by passing the\n",
        "    dataset and other details.\n",
        "    Args:\n",
        "        dataset_path: The path to the dataset to use.\n",
        "        name: The name of the dataset.\n",
        "        cache_dir: The directory where the cache is stored.\n",
        "        use_cache: Indicates if the dataset should be cached.\n",
        "        num_points: The maximum number of points to use when splitting the dataset.\n",
        "        ignored_label_inds: A list of labels that should be ignored in the dataset.\n",
        "        test_result_folder: The folder where the test results should be stored.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 dataset_path,\n",
        "                 name='Custom3D',\n",
        "                 cache_dir='./logs/cache',\n",
        "                 use_cache=False,\n",
        "                 num_points=65536,\n",
        "                 ignored_label_inds=[],\n",
        "                 test_result_folder='./test',\n",
        "                 **kwargs):\n",
        "\n",
        "        super().__init__(dataset_path=dataset_path,\n",
        "                         name=name,\n",
        "                         cache_dir=cache_dir,\n",
        "                         use_cache=use_cache,\n",
        "                         num_points=num_points,\n",
        "                         ignored_label_inds=ignored_label_inds,\n",
        "                         test_result_folder=test_result_folder,\n",
        "                         **kwargs)\n",
        "\n",
        "        cfg = self.cfg\n",
        "\n",
        "        self.dataset_path = cfg.dataset_path\n",
        "\n",
        "        self.label_to_names = self.get_label_to_names()\n",
        "\n",
        "        self.num_classes = len(self.label_to_names)\n",
        "        self.label_values = np.sort([k for k, v in self.label_to_names.items()])\n",
        "        self.label_to_idx = {l: i for i, l in enumerate(self.label_values)}\n",
        "        self.ignored_labels = np.array(cfg.ignored_label_inds)\n",
        "\n",
        "        self.train_dir = str(Path(cfg.dataset_path) / cfg.train_dir)\n",
        "        self.val_dir = str(Path(cfg.dataset_path) / cfg.val_dir)\n",
        "        self.test_dir = str(Path(cfg.dataset_path) / cfg.test_dir)\n",
        "        # verifying the list of the directories for info\n",
        "        print(self.train_dir, self.val_dir, self.test_dir)\n",
        "        self.train_files = [f for f in glob.glob(self.train_dir + \"/*.npy\")]\n",
        "        self.val_files = [f for f in glob.glob(self.val_dir + \"/*.npy\")]\n",
        "        self.test_files = [f for f in glob.glob(self.test_dir + \"/*.npy\")]\n",
        "        # verifying the list of the loaded files for info\n",
        "        print(\"Training Data List: \" ,self.train_files, \"\\nValidation Data List: \", self.val_files, \"\\nTesting Data List: \", self.test_files)\n",
        "\n",
        "    @staticmethod\n",
        "    def get_label_to_names():\n",
        "        \"\"\"Returns a label to names dictionary object.\n",
        "        Returns:\n",
        "            A dict where keys are label numbers and\n",
        "            values are the corresponding names.\n",
        "        \"\"\"\n",
        "        label_to_names = {0: 'background', 1: 'apple'}\n",
        "        return label_to_names\n",
        "\n",
        "    def get_split(self, split):\n",
        "        \"\"\"Returns a dataset split.\n",
        "        Args:\n",
        "            split: A string identifying the dataset split that is usually one of\n",
        "            'training', 'test', 'validation', or 'all'.\n",
        "        Returns:\n",
        "            A dataset split object providing the requested subset of the data.\n",
        "        \"\"\"\n",
        "        return Custom3DSplit(self, split=split)\n",
        "\n",
        "    def get_split_list(self, split):\n",
        "        \"\"\"Returns a dataset split.\n",
        "        Args:\n",
        "            split: A string identifying the dataset split that is usually one of\n",
        "            'training', 'test', 'validation', or 'all'.\n",
        "        Returns:\n",
        "            A dataset split object providing the requested subset of the data.\n",
        "        Raises:\n",
        "             ValueError: Indicates that the split name passed is incorrect. The\n",
        "             split name should be one of 'training', 'test', 'validation', or\n",
        "             'all'.\n",
        "        \"\"\"\n",
        "        if split in ['test', 'testing']:\n",
        "            self.rng.shuffle(self.test_files)\n",
        "            return self.test_files\n",
        "        elif split in ['val', 'validation']:\n",
        "            self.rng.shuffle(self.val_files)\n",
        "            return self.val_files\n",
        "        elif split in ['train', 'training']:\n",
        "            self.rng.shuffle(self.train_files)\n",
        "            return self.train_files\n",
        "        elif split in ['all']:\n",
        "            files = self.val_files + self.train_files + self.test_files\n",
        "            \n",
        "            return files\n",
        "        else:\n",
        "            raise ValueError(\"Invalid split {}\".format(split))\n",
        "\n",
        "    def is_tested(self, attr):\n",
        "        \"\"\"Checks if a datum in the dataset has been tested.\n",
        "        Args:\n",
        "            dataset: The current dataset to which the datum belongs to.\n",
        "            attr: The attribute that needs to be checked.\n",
        "        Returns:\n",
        "            If the dataum attribute is tested, then return the path where the\n",
        "            attribute is stored; else, returns false.\n",
        "        \"\"\"\n",
        "        cfg = self.cfg\n",
        "        name = attr['name']\n",
        "        path = cfg.test_result_folder\n",
        "        store_path = join(path, self.name, name + '.npy')\n",
        "        if exists(store_path):\n",
        "            print(\"{} already exists.\".format(store_path))\n",
        "            return True\n",
        "        else:\n",
        "            return False\n",
        "\n",
        "    def save_test_result(self, results, attr):\n",
        "        \"\"\"Saves the output of a model.\n",
        "        Args:\n",
        "            results: The output of a model for the datum associated with the attribute passed.\n",
        "            attr: The attributes that correspond to the outputs passed in results.\n",
        "        \"\"\"\n",
        "        cfg = self.cfg\n",
        "        name = attr['name']\n",
        "        path = cfg.test_result_folder\n",
        "        make_dir(path)\n",
        "        pred = results['predict_labels']\n",
        "        pred = np.array(self.label_to_names[pred])\n",
        "        store_path = join(path, name + '.npy')\n",
        "        np.save(store_path, pred)\n",
        "\n",
        "DATASET._register_module(Custom3D)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch related import statements\n",
        "import os\n",
        "import random\n",
        "import open3d.ml as _ml3d\n",
        "import open3d.ml.torch as ml3d\n",
        "# general import statements\n",
        "import logging\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "from pathlib import Path\n",
        "import os, sys, glob, pickle\n",
        "from sklearn.neighbors import KDTree\n",
        "from os.path import join, exists, dirname, abspath"
      ],
      "metadata": {
        "id": "Jz8LrKRQ211W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# RandLA-Net model configuration file for S3DIS dataset\n",
        "!cat Open3D-ML/ml3d/configs/randlanet_s3dis.yml\n",
        "# modifying this file based on the pipe dataset requirements with %%writefile command\n",
        "# for custom dataset we are modifying configurations like, data subset paths, classes, class weights etc."
      ],
      "metadata": {
        "id": "i_GuLZ7W213U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# loading the previous checkpoint for further training after runtime disconnect\n",
        "!ls drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/RandLANet_fuji-apple-norm-segmentation_torch/checkpoint/ckpt_00010.pth"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O4F2eW0T8jEn",
        "outputId": "96583513-2a4a-4577-c80c-a8f83c9aa49b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/RandLANet_fuji-apple-norm-segmentation_torch/checkpoint/ckpt_00010.pth\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile Open3D-ML/ml3d/configs/randlanet_s3dis.yml\n",
        "dataset:\n",
        "  name: fuji-apple-norm-segmentation\n",
        "  dataset_path: fuji-norm-dataset\n",
        "  train_dir: train\n",
        "  val_dir: valid\n",
        "  test_dir: test\n",
        "  cache_dir: drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/cache\n",
        "  class_weights: []\n",
        "  ignored_label_inds: []\n",
        "  num_points: 40960\n",
        "  test_area_idx: 1\n",
        "  test_result_folder: ./test\n",
        "  use_cache: False\n",
        "model:\n",
        "  name: RandLANet\n",
        "  batcher: DefaultBatcher\n",
        "  ckpt_path: \n",
        "  num_neighbors: 16\n",
        "  num_layers: 5\n",
        "  num_points: 40960\n",
        "  num_classes: 2\n",
        "  ignored_label_inds: []\n",
        "  sub_sampling_ratio: [4, 4, 4, 4, 2]\n",
        "  in_channels: 6\n",
        "  dim_features: 8\n",
        "  dim_output: [16, 64, 128, 256, 512]\n",
        "  grid_size: 0.04\n",
        "  augment:\n",
        "    recenter:\n",
        "      dim: [0, 1]\n",
        "    rotate:\n",
        "      method: vertical\n",
        "    scale:\n",
        "      min_s: 0.9\n",
        "      max_s: 1.1\n",
        "    noise:\n",
        "      noise_std: 0.001\n",
        "pipeline:\n",
        "  name: SemanticSegmentation\n",
        "  optimizer:\n",
        "    lr: 0.001\n",
        "  batch_size: 1\n",
        "  main_log_dir: drive/MyDrive/point-cloud-prototyping/fuji-seg-logs\n",
        "  max_epoch: 50\n",
        "  save_ckpt_freq: 5\n",
        "  scheduler_gamma: 0.99\n",
        "  test_batch_size: 3\n",
        "  train_sum_dir: drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/train-logs\n",
        "  val_batch_size: 5\n",
        "  summary:\n",
        "    record_for: []\n",
        "    max_pts:\n",
        "    use_reference: false\n",
        "    max_outputs: 1"
      ],
      "metadata": {
        "id": "zWHcpk7f7b6H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ad51e87-b1f2-434c-a0bb-77922bf99c13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting Open3D-ML/ml3d/configs/randlanet_s3dis.yml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# removing 'class_weights' vector for model training, know logged bug\n",
        "# otherwise, [1,3] (or [1, num_classes]) incompatible shape error is produced\n",
        "cfg_file = \"Open3D-ML/ml3d/configs/randlanet_s3dis.yml\"\n",
        "cfg = _ml3d.utils.Config.load_from_file(cfg_file)\n",
        "model = ml3d.models.RandLANet(**cfg.model)"
      ],
      "metadata": {
        "id": "Mlr081947b8K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# verifying the updated config file for the custom data loader and RandLa-net model\n",
        "print(cfg.dataset)\n",
        "print(cfg.pipeline)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6V5ZrkoV7b-g",
        "outputId": "f275d5c5-9cf9-4b15-97f8-4cef65ece55f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'name': 'fuji-apple-norm-segmentation', 'dataset_path': 'fuji-norm-dataset', 'train_dir': 'train', 'val_dir': 'valid', 'test_dir': 'test', 'cache_dir': 'drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/cache', 'class_weights': [], 'ignored_label_inds': [], 'num_points': 40960, 'test_area_idx': 1, 'test_result_folder': './test', 'use_cache': False}\n",
            "{'name': 'SemanticSegmentation', 'optimizer': {'lr': 0.001}, 'batch_size': 1, 'main_log_dir': 'drive/MyDrive/point-cloud-prototyping/fuji-seg-logs', 'max_epoch': 50, 'save_ckpt_freq': 5, 'scheduler_gamma': 0.99, 'test_batch_size': 3, 'train_sum_dir': 'drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/train-logs', 'val_batch_size': 5, 'summary': {'record_for': [], 'max_pts': None, 'use_reference': False, 'max_outputs': 1}}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# loading the custom dataset object for model training\n",
        "dataset = Custom3D(cfg.dataset.pop('dataset_path', None), **cfg.dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sIwhUBSA9dCu",
        "outputId": "7c67ca24-ffc9-4c6d-9126-62bab9b560e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fuji-norm-dataset/train fuji-norm-dataset/valid fuji-norm-dataset/test\n",
            "Training Data List:  ['fuji-norm-dataset/train/data_patch_238.npy', 'fuji-norm-dataset/train/data_patch_263.npy', 'fuji-norm-dataset/train/data_patch_367.npy', 'fuji-norm-dataset/train/data_patch_281.npy', 'fuji-norm-dataset/train/data_patch_364.npy', 'fuji-norm-dataset/train/data_patch_396.npy', 'fuji-norm-dataset/train/data_patch_383.npy', 'fuji-norm-dataset/train/data_patch_262.npy', 'fuji-norm-dataset/train/data_patch_174.npy', 'fuji-norm-dataset/train/data_patch_197.npy', 'fuji-norm-dataset/train/data_patch_357.npy', 'fuji-norm-dataset/train/data_patch_402.npy', 'fuji-norm-dataset/train/data_patch_286.npy', 'fuji-norm-dataset/train/data_patch_178.npy', 'fuji-norm-dataset/train/data_patch_189.npy', 'fuji-norm-dataset/train/data_patch_328.npy', 'fuji-norm-dataset/train/data_patch_316.npy', 'fuji-norm-dataset/train/data_patch_209.npy', 'fuji-norm-dataset/train/data_patch_267.npy', 'fuji-norm-dataset/train/data_patch_166.npy', 'fuji-norm-dataset/train/data_patch_227.npy', 'fuji-norm-dataset/train/data_patch_348.npy', 'fuji-norm-dataset/train/data_patch_217.npy', 'fuji-norm-dataset/train/data_patch_381.npy', 'fuji-norm-dataset/train/data_patch_388.npy', 'fuji-norm-dataset/train/data_patch_208.npy', 'fuji-norm-dataset/train/data_patch_355.npy', 'fuji-norm-dataset/train/data_patch_288.npy', 'fuji-norm-dataset/train/data_patch_278.npy', 'fuji-norm-dataset/train/data_patch_287.npy', 'fuji-norm-dataset/train/data_patch_365.npy', 'fuji-norm-dataset/train/data_patch_335.npy', 'fuji-norm-dataset/train/data_patch_315.npy', 'fuji-norm-dataset/train/data_patch_172.npy', 'fuji-norm-dataset/train/data_patch_350.npy', 'fuji-norm-dataset/train/data_patch_202.npy', 'fuji-norm-dataset/train/data_patch_190.npy', 'fuji-norm-dataset/train/data_patch_182.npy', 'fuji-norm-dataset/train/data_patch_331.npy', 'fuji-norm-dataset/train/data_patch_304.npy', 'fuji-norm-dataset/train/data_patch_284.npy', 'fuji-norm-dataset/train/data_patch_213.npy', 'fuji-norm-dataset/train/data_patch_242.npy', 'fuji-norm-dataset/train/data_patch_254.npy', 'fuji-norm-dataset/train/data_patch_325.npy', 'fuji-norm-dataset/train/data_patch_175.npy', 'fuji-norm-dataset/train/data_patch_329.npy', 'fuji-norm-dataset/train/data_patch_360.npy', 'fuji-norm-dataset/train/data_patch_338.npy', 'fuji-norm-dataset/train/data_patch_240.npy', 'fuji-norm-dataset/train/data_patch_307.npy', 'fuji-norm-dataset/train/data_patch_311.npy', 'fuji-norm-dataset/train/data_patch_245.npy', 'fuji-norm-dataset/train/data_patch_270.npy', 'fuji-norm-dataset/train/data_patch_378.npy', 'fuji-norm-dataset/train/data_patch_187.npy', 'fuji-norm-dataset/train/data_patch_226.npy', 'fuji-norm-dataset/train/data_patch_308.npy', 'fuji-norm-dataset/train/data_patch_371.npy', 'fuji-norm-dataset/train/data_patch_397.npy', 'fuji-norm-dataset/train/data_patch_241.npy', 'fuji-norm-dataset/train/data_patch_384.npy', 'fuji-norm-dataset/train/data_patch_191.npy', 'fuji-norm-dataset/train/data_patch_180.npy', 'fuji-norm-dataset/train/data_patch_290.npy', 'fuji-norm-dataset/train/data_patch_349.npy', 'fuji-norm-dataset/train/data_patch_354.npy', 'fuji-norm-dataset/train/data_patch_301.npy', 'fuji-norm-dataset/train/data_patch_229.npy', 'fuji-norm-dataset/train/data_patch_243.npy', 'fuji-norm-dataset/train/data_patch_362.npy', 'fuji-norm-dataset/train/data_patch_283.npy', 'fuji-norm-dataset/train/data_patch_322.npy', 'fuji-norm-dataset/train/data_patch_361.npy', 'fuji-norm-dataset/train/data_patch_164.npy', 'fuji-norm-dataset/train/data_patch_177.npy', 'fuji-norm-dataset/train/data_patch_261.npy', 'fuji-norm-dataset/train/data_patch_163.npy', 'fuji-norm-dataset/train/data_patch_373.npy', 'fuji-norm-dataset/train/data_patch_201.npy', 'fuji-norm-dataset/train/data_patch_376.npy', 'fuji-norm-dataset/train/data_patch_264.npy', 'fuji-norm-dataset/train/data_patch_210.npy', 'fuji-norm-dataset/train/data_patch_356.npy', 'fuji-norm-dataset/train/data_patch_291.npy', 'fuji-norm-dataset/train/data_patch_313.npy', 'fuji-norm-dataset/train/data_patch_195.npy', 'fuji-norm-dataset/train/data_patch_374.npy', 'fuji-norm-dataset/train/data_patch_398.npy', 'fuji-norm-dataset/train/data_patch_216.npy', 'fuji-norm-dataset/train/data_patch_239.npy', 'fuji-norm-dataset/train/data_patch_303.npy', 'fuji-norm-dataset/train/data_patch_305.npy', 'fuji-norm-dataset/train/data_patch_265.npy', 'fuji-norm-dataset/train/data_patch_302.npy', 'fuji-norm-dataset/train/data_patch_205.npy', 'fuji-norm-dataset/train/data_patch_260.npy', 'fuji-norm-dataset/train/data_patch_370.npy', 'fuji-norm-dataset/train/data_patch_214.npy', 'fuji-norm-dataset/train/data_patch_188.npy', 'fuji-norm-dataset/train/data_patch_403.npy', 'fuji-norm-dataset/train/data_patch_230.npy', 'fuji-norm-dataset/train/data_patch_186.npy', 'fuji-norm-dataset/train/data_patch_252.npy', 'fuji-norm-dataset/train/data_patch_162.npy', 'fuji-norm-dataset/train/data_patch_221.npy', 'fuji-norm-dataset/train/data_patch_247.npy', 'fuji-norm-dataset/train/data_patch_234.npy', 'fuji-norm-dataset/train/data_patch_179.npy', 'fuji-norm-dataset/train/data_patch_346.npy', 'fuji-norm-dataset/train/data_patch_337.npy', 'fuji-norm-dataset/train/data_patch_271.npy', 'fuji-norm-dataset/train/data_patch_300.npy', 'fuji-norm-dataset/train/data_patch_385.npy', 'fuji-norm-dataset/train/data_patch_359.npy', 'fuji-norm-dataset/train/data_patch_323.npy', 'fuji-norm-dataset/train/data_patch_344.npy', 'fuji-norm-dataset/train/data_patch_273.npy', 'fuji-norm-dataset/train/data_patch_345.npy', 'fuji-norm-dataset/train/data_patch_369.npy', 'fuji-norm-dataset/train/data_patch_259.npy', 'fuji-norm-dataset/train/data_patch_342.npy', 'fuji-norm-dataset/train/data_patch_207.npy', 'fuji-norm-dataset/train/data_patch_237.npy', 'fuji-norm-dataset/train/data_patch_289.npy', 'fuji-norm-dataset/train/data_patch_339.npy', 'fuji-norm-dataset/train/data_patch_404.npy', 'fuji-norm-dataset/train/data_patch_310.npy', 'fuji-norm-dataset/train/data_patch_285.npy', 'fuji-norm-dataset/train/data_patch_299.npy', 'fuji-norm-dataset/train/data_patch_251.npy', 'fuji-norm-dataset/train/data_patch_276.npy', 'fuji-norm-dataset/train/data_patch_198.npy', 'fuji-norm-dataset/train/data_patch_212.npy', 'fuji-norm-dataset/train/data_patch_228.npy', 'fuji-norm-dataset/train/data_patch_294.npy', 'fuji-norm-dataset/train/data_patch_314.npy', 'fuji-norm-dataset/train/data_patch_196.npy', 'fuji-norm-dataset/train/data_patch_333.npy', 'fuji-norm-dataset/train/data_patch_253.npy', 'fuji-norm-dataset/train/data_patch_400.npy', 'fuji-norm-dataset/train/data_patch_324.npy', 'fuji-norm-dataset/train/data_patch_199.npy', 'fuji-norm-dataset/train/data_patch_170.npy', 'fuji-norm-dataset/train/data_patch_394.npy', 'fuji-norm-dataset/train/data_patch_204.npy', 'fuji-norm-dataset/train/data_patch_211.npy', 'fuji-norm-dataset/train/data_patch_380.npy', 'fuji-norm-dataset/train/data_patch_222.npy', 'fuji-norm-dataset/train/data_patch_250.npy', 'fuji-norm-dataset/train/data_patch_244.npy', 'fuji-norm-dataset/train/data_patch_282.npy', 'fuji-norm-dataset/train/data_patch_218.npy', 'fuji-norm-dataset/train/data_patch_255.npy', 'fuji-norm-dataset/train/data_patch_391.npy', 'fuji-norm-dataset/train/data_patch_319.npy', 'fuji-norm-dataset/train/data_patch_225.npy', 'fuji-norm-dataset/train/data_patch_392.npy', 'fuji-norm-dataset/train/data_patch_257.npy', 'fuji-norm-dataset/train/data_patch_183.npy', 'fuji-norm-dataset/train/data_patch_268.npy', 'fuji-norm-dataset/train/data_patch_168.npy', 'fuji-norm-dataset/train/data_patch_223.npy', 'fuji-norm-dataset/train/data_patch_266.npy', 'fuji-norm-dataset/train/data_patch_258.npy', 'fuji-norm-dataset/train/data_patch_232.npy', 'fuji-norm-dataset/train/data_patch_309.npy', 'fuji-norm-dataset/train/data_patch_340.npy', 'fuji-norm-dataset/train/data_patch_173.npy', 'fuji-norm-dataset/train/data_patch_176.npy', 'fuji-norm-dataset/train/data_patch_169.npy', 'fuji-norm-dataset/train/data_patch_386.npy', 'fuji-norm-dataset/train/data_patch_275.npy', 'fuji-norm-dataset/train/data_patch_358.npy', 'fuji-norm-dataset/train/data_patch_399.npy', 'fuji-norm-dataset/train/data_patch_246.npy', 'fuji-norm-dataset/train/data_patch_292.npy', 'fuji-norm-dataset/train/data_patch_366.npy', 'fuji-norm-dataset/train/data_patch_193.npy', 'fuji-norm-dataset/train/data_patch_231.npy', 'fuji-norm-dataset/train/data_patch_390.npy', 'fuji-norm-dataset/train/data_patch_194.npy', 'fuji-norm-dataset/train/data_patch_280.npy', 'fuji-norm-dataset/train/data_patch_405.npy', 'fuji-norm-dataset/train/data_patch_272.npy', 'fuji-norm-dataset/train/data_patch_395.npy', 'fuji-norm-dataset/train/data_patch_200.npy', 'fuji-norm-dataset/train/data_patch_220.npy', 'fuji-norm-dataset/train/data_patch_341.npy', 'fuji-norm-dataset/train/data_patch_321.npy', 'fuji-norm-dataset/train/data_patch_387.npy', 'fuji-norm-dataset/train/data_patch_219.npy', 'fuji-norm-dataset/train/data_patch_206.npy', 'fuji-norm-dataset/train/data_patch_167.npy', 'fuji-norm-dataset/train/data_patch_401.npy', 'fuji-norm-dataset/train/data_patch_320.npy', 'fuji-norm-dataset/train/data_patch_274.npy', 'fuji-norm-dataset/train/data_patch_377.npy', 'fuji-norm-dataset/train/data_patch_233.npy', 'fuji-norm-dataset/train/data_patch_224.npy', 'fuji-norm-dataset/train/data_patch_184.npy', 'fuji-norm-dataset/train/data_patch_192.npy', 'fuji-norm-dataset/train/data_patch_293.npy', 'fuji-norm-dataset/train/data_patch_203.npy', 'fuji-norm-dataset/train/data_patch_215.npy', 'fuji-norm-dataset/train/data_patch_181.npy', 'fuji-norm-dataset/train/data_patch_295.npy', 'fuji-norm-dataset/train/data_patch_406.npy', 'fuji-norm-dataset/train/data_patch_332.npy', 'fuji-norm-dataset/train/data_patch_312.npy', 'fuji-norm-dataset/train/data_patch_279.npy', 'fuji-norm-dataset/train/data_patch_306.npy', 'fuji-norm-dataset/train/data_patch_171.npy', 'fuji-norm-dataset/train/data_patch_269.npy', 'fuji-norm-dataset/train/data_patch_382.npy', 'fuji-norm-dataset/train/data_patch_407.npy', 'fuji-norm-dataset/train/data_patch_317.npy', 'fuji-norm-dataset/train/data_patch_389.npy', 'fuji-norm-dataset/train/data_patch_353.npy', 'fuji-norm-dataset/train/data_patch_351.npy', 'fuji-norm-dataset/train/data_patch_318.npy'] \n",
            "Validation Data List:  ['fuji-norm-dataset/valid/data_patch_298.npy', 'fuji-norm-dataset/valid/data_patch_343.npy', 'fuji-norm-dataset/valid/data_patch_249.npy', 'fuji-norm-dataset/valid/data_patch_297.npy', 'fuji-norm-dataset/valid/data_patch_236.npy', 'fuji-norm-dataset/valid/data_patch_256.npy', 'fuji-norm-dataset/valid/data_patch_327.npy', 'fuji-norm-dataset/valid/data_patch_379.npy', 'fuji-norm-dataset/valid/data_patch_347.npy', 'fuji-norm-dataset/valid/data_patch_352.npy', 'fuji-norm-dataset/valid/data_patch_368.npy'] \n",
            "Testing Data List:  ['fuji-norm-dataset/test/data_patch_334.npy', 'fuji-norm-dataset/test/data_patch_248.npy', 'fuji-norm-dataset/test/data_patch_372.npy', 'fuji-norm-dataset/test/data_patch_296.npy', 'fuji-norm-dataset/test/data_patch_330.npy', 'fuji-norm-dataset/test/data_patch_165.npy', 'fuji-norm-dataset/test/data_patch_375.npy', 'fuji-norm-dataset/test/data_patch_363.npy', 'fuji-norm-dataset/test/data_patch_185.npy', 'fuji-norm-dataset/test/data_patch_235.npy', 'fuji-norm-dataset/test/data_patch_277.npy', 'fuji-norm-dataset/test/data_patch_161.npy', 'fuji-norm-dataset/test/data_patch_393.npy', 'fuji-norm-dataset/test/data_patch_336.npy']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# exploration: verifying whether loaded point cloud data is correct\n",
        "# get the 'all' split that combines training, validation and test set\n",
        "all_split = dataset.get_split('all')\n",
        "print(all_split)\n",
        "# print the attributes of the first datum\n",
        "print(all_split.get_attr(0))\n",
        "# print the shape of the first point cloud\n",
        "print(all_split.get_data(0)['point'].shape)"
      ],
      "metadata": {
        "id": "JpCzsdNY9dMJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16e7e461-8487-4a49-b45d-9feec1914cd4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<__main__.Custom3DSplit object at 0x7fb70284ca30>\n",
            "{'name': 'data_patch_298', 'path': 'fuji-norm-dataset/valid/data_patch_298.npy', 'split': 'all'}\n",
            "(50509, 3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# semseg file updation for adding label smoothing and\n",
        "# additional penalization for apple class misclassification\n",
        "!cat /usr/local/lib/python3.9/dist-packages/open3d/_ml3d/torch/modules/losses/semseg_loss.py"
      ],
      "metadata": {
        "id": "92cWFrEd7cBY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile /usr/local/lib/python3.9/dist-packages/open3d/_ml3d/torch/modules/losses/semseg_loss.py\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from ....datasets.utils import DataProcessing\n",
        "\n",
        "\n",
        "def filter_valid_label(scores, labels, num_classes, ignored_label_inds, device):\n",
        "    \"\"\"Loss functions for semantic segmentation.\"\"\"\n",
        "    valid_scores = scores.reshape(-1, num_classes).to(device)\n",
        "    valid_labels = labels.reshape(-1).to(device)\n",
        "\n",
        "    ignored_bool = torch.zeros_like(valid_labels, dtype=torch.bool)\n",
        "    for ign_label in ignored_label_inds:\n",
        "        ignored_bool = torch.logical_or(ignored_bool,\n",
        "                                        torch.eq(valid_labels, ign_label))\n",
        "\n",
        "    valid_idx = torch.where(torch.logical_not(ignored_bool))[0].to(device)\n",
        "\n",
        "    valid_scores = torch.gather(valid_scores, 0,\n",
        "                                valid_idx.unsqueeze(-1).expand(-1, num_classes))\n",
        "    valid_labels = torch.gather(valid_labels, 0, valid_idx)\n",
        "\n",
        "    # Reduce label values in the range of logit shape\n",
        "    reducing_list = torch.arange(0, num_classes, dtype=torch.int64)\n",
        "    inserted_value = torch.zeros([1], dtype=torch.int64)\n",
        "\n",
        "    for ign_label in ignored_label_inds:\n",
        "        if ign_label >= 0:\n",
        "\n",
        "            reducing_list = torch.cat([\n",
        "                reducing_list[:ign_label], inserted_value,\n",
        "                reducing_list[ign_label:]\n",
        "            ], 0)\n",
        "    valid_labels = torch.gather(reducing_list.to(device), 0,\n",
        "                                valid_labels.long())\n",
        "\n",
        "    return valid_scores, valid_labels\n",
        "\n",
        "\n",
        "class SemSegLoss(object):\n",
        "    \"\"\"Loss functions for semantic segmentation.\"\"\"\n",
        "\n",
        "    def __init__(self, pipeline, model, dataset, device):\n",
        "        super(SemSegLoss, self).__init__()\n",
        "        # weighted_CrossEntropyLoss\n",
        "        if 'class_weights' in dataset.cfg.keys() and len(\n",
        "                dataset.cfg.class_weights) != 0:\n",
        "            class_wt = DataProcessing.get_class_weights(\n",
        "                dataset.cfg.class_weights)\n",
        "            weights = torch.tensor(class_wt, dtype=torch.float, device=device)\n",
        "\n",
        "            self.weighted_CrossEntropyLoss = nn.CrossEntropyLoss(weight=weights)\n",
        "        else:\n",
        "            weights = torch.tensor([1.0, 1.618], dtype=torch.float).squeeze(-1)\n",
        "            self.weighted_CrossEntropyLoss = nn.CrossEntropyLoss(weight=weights, ignore_index=2, label_smoothing=0.0625)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d2jzQSGd215T",
        "outputId": "cd80d88c-72f7-4e12-e90e-1ef0edb911e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting /usr/local/lib/python3.9/dist-packages/open3d/_ml3d/torch/modules/losses/semseg_loss.py\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# creating the segmentation pipeline for RandLA-Net model training\n",
        "pipeline = ml3d.pipelines.SemanticSegmentation(model, dataset=dataset, device=\"auto\", **cfg.pipeline)\n",
        "# ckpt_path = \"drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/RandLANet_fuji-apple-norm-segmentation_torch/checkpoint/ckpt_00010.pth\"\n",
        "# pipeline.load_ckpt(ckpt_path=ckpt_path)"
      ],
      "metadata": {
        "id": "u4em3Oym-SzY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# training the model from scratch with the pipe dataset, keep connection active\n",
        "pipeline.run_train()\n",
        "# known bug, adding 'class_weights' to configuration file give shape head error"
      ],
      "metadata": {
        "id": "PuA73Pr_218A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Mns-G9zjZ1S"
      },
      "outputs": [],
      "source": [
        "# for loading the training logs to observe the learning curve during training \n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir drive/MyDrive/point-cloud-prototyping/fuji-seg-logs/train-logs"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JFhQTjrn-iQF"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}