From f6c5d6751600d4f8950e34a08a311b0d84bfb491 Mon Sep 17 00:00:00 2001 From: hsj576 Date: Mon, 30 Oct 2023 15:32:18 +0800 Subject: [PATCH] merge robot and robot-sam, add leaderboard Signed-off-by: hsj576 add docstring for codes Signed-off-by: hsj576 remove hardcore path and TEMP PRINT lines Signed-off-by: hsj576 --- ...d-of-SAM-based-Edge-Cloud-Collaboration.md | 6 + .../leaderboard-of-lifelong-learning.md | 5 + .../lifelong_learning_bench/README.md | 149 ----- .../testalgorithms/rfnet/RFNet/accuracy.py | 38 -- .../testalgorithms/rfnet/RFNet/basemodel.py | 315 ---------- .../rfnet/RFNet/dataloaders/__init__.py | 116 ---- .../RFNet/dataloaders/custom_transforms.py | 240 -------- .../dataloaders/custom_transforms_rgb.py | 230 -------- .../RFNet/dataloaders/datasets/__init__.py | 0 .../dataloaders/datasets/citylostfound.py | 276 --------- .../RFNet/dataloaders/datasets/cityrand.py | 151 ----- .../RFNet/dataloaders/datasets/cityscapes.py | 160 ------ .../rfnet/RFNet/dataloaders/datasets/e1.py | 151 ----- .../RFNet/dataloaders/datasets/mapillary.py | 152 ----- .../RFNet/dataloaders/datasets/target.py | 152 ----- .../rfnet/RFNet/dataloaders/datasets/temp.txt | 38 -- .../rfnet/RFNet/dataloaders/datasets/xrlab.py | 152 ----- .../rfnet/RFNet/dataloaders/utils.py | 244 -------- .../testalgorithms/rfnet/RFNet/eval.py | 537 ------------------ .../rfnet/RFNet/models/replicate.py | 88 --- .../rfnet/RFNet/models/resnet/__init__.py | 0 .../resnet_single_scale_single_attention.py | 391 ------------- ...et_single_scale_single_attention_unseen.py | 396 ------------- .../rfnet/RFNet/models/rfnet.py | 27 - .../rfnet/RFNet/models/rfnet_for_unseen.py | 33 -- .../testalgorithms/rfnet/RFNet/models/util.py | 99 ---- .../testalgorithms/rfnet/RFNet/mypath.py | 20 - .../testalgorithms/rfnet/RFNet/predict.py | 100 ---- .../testalgorithms/rfnet/RFNet/run_server.py | 254 --------- .../rfnet/RFNet/sedna_evaluate.py | 50 -- .../rfnet/RFNet/sedna_predict.py | 132 ----- .../testalgorithms/rfnet/RFNet/sedna_train.py | 78 --- .../testalgorithms/rfnet/RFNet/test.py | 52 -- .../testalgorithms/rfnet/RFNet/train.py | 310 ---------- .../rfnet/RFNet/utils/__init__.py | 0 .../testalgorithms/rfnet/RFNet/utils/args.py | 52 -- .../rfnet/RFNet/utils/calculate_weights.py | 29 - .../rfnet/RFNet/utils/iouEval.py | 141 ----- .../testalgorithms/rfnet/RFNet/utils/loss.py | 65 --- .../rfnet/RFNet/utils/lr_scheduler.py | 70 --- .../rfnet/RFNet/utils/metrics.py | 203 ------- .../testalgorithms/rfnet/RFNet/utils/saver.py | 68 --- .../rfnet/RFNet/utils/summaries.py | 39 -- .../testalgorithms/rfnet/basemodel-simple.py | 200 ------- .../testalgorithms/rfnet/basemodel.py | 164 ------ .../rfnet/rfnet_algorithm-simple.yaml | 63 -- .../testalgorithms/rfnet/rfnet_algorithm.yaml | 74 --- .../rfnet/task_allocation_by_origin-simple.py | 49 -- .../rfnet/task_allocation_by_origin.py | 49 -- .../rfnet/task_definition_by_origin-simple.py | 64 --- .../rfnet/task_definition_by_origin.py | 65 --- .../testenv/accuracy.py | 58 -- .../testenv/testenv-all.yaml | 35 -- .../testenv/testenv-city.yaml | 35 -- .../testenv/testenv-robot-small.yaml | 40 -- .../testenv/testenv-robot.yaml | 40 -- .../testenv/testenv.yaml | 35 -- .../robot/lifelong_learning_bench/README.md | 115 +++- .../benchmarkingjob-sam.yaml} | 12 +- ...ngjob.yaml => benchmarkingjob-simple.yaml} | 2 +- .../rfnet/RFNet/cityscapes_id2label.py | 0 .../testalgorithms/rfnet/RFNet/eval.py | 389 ++++++++++++- .../rfnet/RFNet/utils/metrics.py | 2 + .../testalgorithms/rfnet/basemodel-sam.py | 38 +- .../testalgorithms/rfnet/basemodel-simple.py | 59 +- .../testalgorithms/rfnet/basemodel-vit-sam.py | 38 +- .../rfnet/basemodel-vit-simple.py | 38 +- .../testalgorithms/rfnet/basemodel.py | 3 + .../rfnet/hard_sample_mining.py | 0 .../testalgorithms/rfnet/rfnet_algorithm.yaml | 27 +- .../testalgorithms/rfnet/sam_algorithm.yaml | 8 +- .../rfnet/sam_vit_algorithm.yaml | 8 +- .../testalgorithms/rfnet/vit_algorithm.yaml | 8 +- .../testenv/testenv-robot-small.yaml | 3 +- .../testenv/testenv-robot.yaml | 3 +- 75 files changed, 578 insertions(+), 6955 deletions(-) create mode 100644 docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md create mode 100644 docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md delete mode 100755 examples/robot-sam/lifelong_learning_bench/README.md delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/accuracy.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/basemodel.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/__init__.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms_rgb.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/__init__.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/citylostfound.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityrand.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityscapes.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/e1.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/mapillary.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/target.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/temp.txt delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/xrlab.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/utils.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/replicate.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/__init__.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention_unseen.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet_for_unseen.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/util.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/mypath.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/predict.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/run_server.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_evaluate.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_predict.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_train.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/test.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/train.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/__init__.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/calculate_weights.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/iouEval.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/loss.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/lr_scheduler.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/saver.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/summaries.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm-simple.yaml delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testenv/accuracy.py delete mode 100755 examples/robot-sam/lifelong_learning_bench/testenv/testenv-all.yaml delete mode 100755 examples/robot-sam/lifelong_learning_bench/testenv/testenv-city.yaml delete mode 100755 examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml delete mode 100755 examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml delete mode 100755 examples/robot-sam/lifelong_learning_bench/testenv/testenv.yaml rename examples/{robot-sam/lifelong_learning_bench/benchmarkingjob.yaml => robot/lifelong_learning_bench/benchmarkingjob-sam.yaml} (85%) mode change 100755 => 100644 rename examples/robot/lifelong_learning_bench/{benchmarkingjob.yaml => benchmarkingjob-simple.yaml} (97%) mode change 100755 => 100644 rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py (100%) rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py (85%) mode change 100755 => 100644 rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py (85%) mode change 100755 => 100644 rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py (85%) mode change 100755 => 100644 rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py (100%) rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml (86%) mode change 100755 => 100644 rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml (86%) mode change 100755 => 100644 rename examples/{robot-sam => robot}/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml (86%) mode change 100755 => 100644 diff --git a/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md new file mode 100644 index 00000000..d331d857 --- /dev/null +++ b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md @@ -0,0 +1,6 @@ +# Leaderboard of SAM-based Edge-Cloud Collaboration + + +| rank | algorithm | accuracy | Task_Avg_Acc | paradigm | basemodel | task_definition | task_allocation | unseen_sample_recognition | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | unseen_sample_recognition-threhold | time | +| :--: | :-------------------------: | :----------------: | :----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :-----------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :--------------------------------: | ------------------- | +| 1 | sam_rfnet_lifelong_learning | 0.7052917006987501 | 0.6258875117354328 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | HardSampleMining | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 0.95 | 2023-08-24 12:43:19 | \ No newline at end of file diff --git a/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md new file mode 100644 index 00000000..7691ef2c --- /dev/null +++ b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md @@ -0,0 +1,5 @@ +# Leaderboard of lifelong learning + +| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | +| :--: | :---------------------: | :----------------: | :-----------------: | :-----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :-----------------: | +| 1 | rfnet_lifelong_learning | 0.5206033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 10 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | diff --git a/examples/robot-sam/lifelong_learning_bench/README.md b/examples/robot-sam/lifelong_learning_bench/README.md deleted file mode 100755 index 399d651f..00000000 --- a/examples/robot-sam/lifelong_learning_bench/README.md +++ /dev/null @@ -1,149 +0,0 @@ -# Quick Start - -Welcome to Ianvs! Ianvs aims to test the performance of distributed synergy AI solutions following recognized standards, -in order to facilitate more efficient and effective development. Quick start helps you to test your algorithm on Ianvs -with a simple example of industrial defect detection. You can reduce manual procedures to just a few steps so that you can -build and start your distributed synergy AI solution development within minutes. - -Before using Ianvs, you might want to have the device ready: -- One machine is all you need, i.e., a laptop or a virtual machine is sufficient and a cluster is not necessary -- 2 CPUs or more -- 4GB+ free memory, depends on algorithm and simulation setting -- 10GB+ free disk space -- Internet connection for GitHub and pip, etc -- Python 3.6+ installed - - -In this example, we are using the Linux platform with Python 3.9. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. - -## Step 1. Ianvs Preparation - -First, we download the code of Ianvs. Assuming that we are using `/ianvs` as workspace, Ianvs can be cloned with `Git` -as: - -``` shell -mkdir /ianvs -cd /ianvs #One might use another path preferred - -mkdir project -cd project -git clone https://github.com/kubeedge/ianvs.git -``` - - -Then, we install third-party dependencies for ianvs. -``` shell -sudo apt-get update -sudo apt-get install libgl1-mesa-glx -y -python -m pip install --upgrade pip - -cd ianvs -python -m pip install ./examples/resources/third_party/* -python -m pip install -r requirements.txt -``` - -We are now ready to install Ianvs. -``` shell -python setup.py install -``` - -## Step 2. Dataset Preparation - -Datasets and models can be large. To avoid over-size projects in the Github repository of Ianvs, the Ianvs code base does -not include origin datasets. Then developers do not need to download non-necessary datasets for a quick start. - -``` shell -mkdir /data -cd /data -mkdir datasets -cd datasets -python -m pip install kaggle -kaggle datasets download -d hsj576/cloud-robotics -unzip cloud-robotics.zip -``` - -The URL address of this dataset then should be filled in the configuration file ``testenv.yaml``. In this quick start, -we have done that for you and the interested readers can refer to [testenv.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. - - - - -Related algorithm is also ready in this quick start. -``` shell -export PYTHONPATH=$PYTHONPATH:/ianvs/project/ianvs/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet -``` - -The URL address of this algorithm then should be filled in the configuration file ``algorithm.yaml``. In this quick -start, we have done that for you and the interested readers can refer to [algorithm.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. - -In this example, we use [SAM model](https://segment-anything.com/) as the cloud large model. So, we need to install SAM by the following instructions: - -~~~bash -cd /ianvs/project -git clone https://github.com/facebookresearch/segment-anything.git -cd segment-anything -python -m pip install -e . -~~~ - -Then, we need to download the pretrained SAM model: - -~~~bash -wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth -~~~ - -In order to save the inference result, we need to install mmcv and mmdetection by the following instructions: - -~~~bash -python -m pip install https://download.openmmlab.com/mmcv/dist/cu118/torch2.0.0/mmcv-2.0.0-cp39-cp39-manylinux1_x86_64.whl -cd /ianvs/project -git clone https://github.com/hsj576/mmdetection.git -cd mmdetection -python -m pip install -v -e . -~~~ - -In case that your computer couldn't run SAM model, we prepare a cache for all the SAM inference results in Cloud-Robotics dataset. You could download the cache from [this link](https://pan.baidu.com/s/1oGGBa8TjZn0ccbznQsl48g?pwd=wpp1) and put the cache file in "/ianvs/project/": - -~~~bash -cp cache.pickle /ianvs/project -~~~ - - By using the cache, you could simulate the edge-cloud joint inference without installing SAM model. - -Besides that, we also provided you a pretrained RFNet model in [this link](https://pan.baidu.com/s/1h8JnUgr1hfx5QnaFLLkMAg?pwd=jts4), you could use it if you don't want to train the RFNet model from zero. This instruction is optional: - -~~~bash -cd /ianvs/project -mkdir pretrain -cp pretrain_model.pth /ianvs/project/pretrain -~~~ - -## Step 3. Ianvs Execution and Presentation - -We are now ready to run the ianvs for benchmarking. - -``` shell -cd /ianvs/project/ianvs -ianvs -f examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml -``` - -Finally, the user can check the result of benchmarking on the console and also in the output path( -e.g. `/ianvs/lifelong_learning_bench/workspace`) defined in the benchmarking config file ( -e.g. `benchmarkingjob.yaml`). In this quick start, we have done all configurations for you and the interested readers -can refer to [benchmarkingJob.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. - -The final output might look like this: - - -| rank | algorithm | accuracy | Task_Avg_Acc | paradigm | basemodel | task_definition | task_allocation | unseen_sample_recognition | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | unseen_sample_recognition-threhold | time | url | -|:------:|:-------------------------:|:--------------------:|:---------------------:|:------------------:|:-----------:|:------------------------:|:------------------------:|:-------------------------:|:------------------:|:-------------------------:|:-------------------------:|:---------------------:|:---------------------------------------------------------------------------------------------------------------------------------:|-----------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------:| -| 1 | sam_rfnet_lifelong_learning | 0.7052917006987501 | 0.6258875117354328 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | HardSampleMining | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 0.95 | 2023-08-24 12:43:19 | /ianvs/sam_bench/robot-workspace/benchmarkingjob/sam_rfnet_lifelong_learning/9465c47a-4235-11ee-8519-ec2a724ccd3e | - - - -This ends the quick start experiment. - -# What is next - -If any problems happen, the user can refer to [the issue page on Github](https://github.com/kubeedge/ianvs/issues) for help and are also welcome to raise any new issue. - -Enjoy your journey on Ianvs! \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/accuracy.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/accuracy.py deleted file mode 100755 index 4fb068f1..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/accuracy.py +++ /dev/null @@ -1,38 +0,0 @@ -from basemodel import val_args -from utils.metrics import Evaluator -from tqdm import tqdm -from dataloaders import make_data_loader -from sedna.common.class_factory import ClassType, ClassFactory - -__all__ = ('accuracy') - -@ClassFactory.register(ClassType.GENERAL) -def accuracy(y_true, y_pred, **kwargs): - args = val_args() - _, _, test_loader, num_class = make_data_loader(args, test_data=y_true) - evaluator = Evaluator(num_class) - - tbar = tqdm(test_loader, desc='\r') - for i, (sample, img_path) in enumerate(tbar): - if args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - image, target = sample['image'], sample['label'] - if args.cuda: - image, target = image.cuda(args.gpu_ids), target.cuda(args.gpu_ids) - if args.depth: - depth = depth.cuda(args.gpu_ids) - - target[target > evaluator.num_class-1] = 255 - target = target.cpu().numpy() - # Add batch sample into evaluator - evaluator.add_batch(target, y_pred[i]) - - # Test during the training - # Acc = evaluator.Pixel_Accuracy() - CPA = evaluator.Pixel_Accuracy_Class() - mIoU = evaluator.Mean_Intersection_over_Union() - FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union() - - print("CPA:{}, mIoU:{}, fwIoU: {}".format(CPA, mIoU, FWIoU)) - return CPA diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/basemodel.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/basemodel.py deleted file mode 100755 index 2684a997..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/basemodel.py +++ /dev/null @@ -1,315 +0,0 @@ -import os -import numpy as np -import torch -from PIL import Image -import argparse -from train import Trainer -from eval import Validator -from tqdm import tqdm -from eval import load_my_state_dict -from utils.metrics import Evaluator -from dataloaders import make_data_loader -from dataloaders import custom_transforms as tr -from torchvision import transforms -from sedna.common.class_factory import ClassType, ClassFactory -from sedna.common.config import Context -from sedna.datasources import TxtDataParse -from torch.utils.data import DataLoader -from sedna.common.file_ops import FileOps -from utils.lr_scheduler import LR_Scheduler - -def preprocess(image_urls): - transformed_images = [] - for paths in image_urls: - if len(paths) == 2: - img_path, depth_path = paths - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(depth_path) - else: - img_path = paths[0] - _img = Image.open(img_path).convert('RGB') - _depth = _img - - sample = {'image': _img, 'depth': _depth, 'label': _img} - composed_transforms = transforms.Compose([ - # tr.CropBlackArea(), - # tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - transformed_images.append((composed_transforms(sample), img_path)) - - return transformed_images - -class Model: - def __init__(self, **kwargs): - self.val_args = val_args() - self.train_args = train_args() - - self.train_args.lr = kwargs.get("learning_rate", 1e-4) - self.train_args.epochs = kwargs.get("epochs", 2) - self.train_args.eval_interval = kwargs.get("eval_interval", 2) - self.train_args.no_val = kwargs.get("no_val", True) - # self.train_args.resume = Context.get_parameters("PRETRAINED_MODEL_URL", None) - self.trainer = None - - label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") - self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") - self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") - self.val_args.label_save_path = os.path.join(label_save_dir, "label") - self.validator = Validator(self.val_args) - - def train(self, train_data, valid_data=None, **kwargs): - self.trainer = Trainer(self.train_args, train_data=train_data) - print("Total epoches:", self.trainer.args.epochs) - for epoch in range(self.trainer.args.start_epoch, self.trainer.args.epochs): - if epoch == 0 and self.trainer.val_loader: - self.trainer.validation(epoch) - self.trainer.training(epoch) - - if self.trainer.args.no_val and \ - (epoch % self.trainer.args.eval_interval == (self.trainer.args.eval_interval - 1) - or epoch == self.trainer.args.epochs - 1): - # save checkpoint when it meets eval_interval or the training finished - is_best = False - checkpoint_path = self.trainer.saver.save_checkpoint({ - 'epoch': epoch + 1, - 'state_dict': self.trainer.model.state_dict(), - 'optimizer': self.trainer.optimizer.state_dict(), - 'best_pred': self.trainer.best_pred, - }, is_best) - - # if not self.trainer.args.no_val and \ - # epoch % self.train_args.eval_interval == (self.train_args.eval_interval - 1) \ - # and self.trainer.val_loader: - # self.trainer.validation(epoch) - - self.trainer.writer.close() - - return checkpoint_path - - def predict(self, data, **kwargs): - if not isinstance(data[0][0], dict): - data = preprocess(data) - - if type(data) is np.ndarray: - data = data.tolist() - - self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, - pin_memory=True) - return self.validator.validate() - - def evaluate(self, data, **kwargs): - self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) - samples = preprocess(data.x) - predictions = self.predict(samples) - return accuracy(data.y, predictions) - - def load(self, model_url, **kwargs): - if model_url: - self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) - self.train_args.resume = model_url - else: - raise Exception("model url does not exist.") - self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) - - def save(self, model_path=None): - # TODO: how to save unstructured data model - pass - -def train_args(): - parser = argparse.ArgumentParser(description="PyTorch RFNet Training") - parser.add_argument('--depth', action="store_true", default=False, - help='training with depth image or not (default: False)') - parser.add_argument('--dataset', type=str, default='cityscapes', - choices=['citylostfound', 'cityscapes', 'cityrand', 'target', 'xrlab', 'e1', 'mapillary'], - help='dataset name (default: cityscapes)') - parser.add_argument('--workers', type=int, default=4, - metavar='N', help='dataloader threads') - parser.add_argument('--base-size', type=int, default=1024, - help='base image size') - parser.add_argument('--crop-size', type=int, default=768, - help='crop image size') - parser.add_argument('--loss-type', type=str, default='ce', - choices=['ce', 'focal'], - help='loss func type (default: ce)') - # training hyper params - # parser.add_argument('--epochs', type=int, default=None, metavar='N', - # help='number of epochs to train (default: auto)') - parser.add_argument('--epochs', type=int, default=None, metavar='N', - help='number of epochs to train (default: auto)') - parser.add_argument('--start_epoch', type=int, default=0, - metavar='N', help='start epochs (default:0)') - parser.add_argument('--batch-size', type=int, default=None, - metavar='N', help='input batch size for \ - training (default: auto)') - parser.add_argument('--val-batch-size', type=int, default=1, - metavar='N', help='input batch size for \ - testing (default: auto)') - parser.add_argument('--test-batch-size', type=int, default=1, - metavar='N', help='input batch size for \ - testing (default: auto)') - parser.add_argument('--use-balanced-weights', action='store_true', default=False, - help='whether to use balanced weights (default: True)') - parser.add_argument('--num-class', type=int, default=24, - help='number of training classes (default: 24') - # optimizer params - parser.add_argument('--lr', type=float, default=1e-4, metavar='LR', - help='learning rate (default: auto)') - parser.add_argument('--lr-scheduler', type=str, default='cos', - choices=['poly', 'step', 'cos', 'inv'], - help='lr scheduler mode: (default: cos)') - parser.add_argument('--momentum', type=float, default=0.9, - metavar='M', help='momentum (default: 0.9)') - parser.add_argument('--weight-decay', type=float, default=2.5e-5, - metavar='M', help='w-decay (default: 5e-4)') - # cuda, seed and logging - parser.add_argument('--no-cuda', action='store_true', default= - False, help='disables CUDA training') - parser.add_argument('--gpu-ids', type=str, default='0', - help='use which gpu to train, must be a \ - comma-separated list of integers only (default=0)') - parser.add_argument('--seed', type=int, default=1, metavar='S', - help='random seed (default: 1)') - # checking point - parser.add_argument('--resume', type=str, - default=None, - help='put the path to resuming file if needed') - parser.add_argument('--checkname', type=str, default=None, - help='set the checkpoint name') - # finetuning pre-trained models - parser.add_argument('--ft', action='store_true', default=True, - help='finetuning on a different dataset') - # evaluation option - parser.add_argument('--eval-interval', type=int, default=1, - help='evaluation interval (default: 1)') - parser.add_argument('--no-val', action='store_true', default=False, - help='skip validation during training') - - args = parser.parse_args() - args.cuda = not args.no_cuda and torch.cuda.is_available() - print(torch.cuda.is_available()) - if args.cuda: - try: - args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')] - except ValueError: - raise ValueError('Argument --gpu_ids must be a comma-separated list of integers only') - - if args.epochs is None: - epoches = { - 'cityscapes': 200, - 'citylostfound': 200, - } - args.epochs = epoches[args.dataset.lower()] - - if args.batch_size is None: - args.batch_size = 4 * len(args.gpu_ids) - - if args.test_batch_size is None: - args.test_batch_size = args.batch_size - - if args.lr is None: - lrs = { - 'cityscapes': 0.0001, - 'citylostfound': 0.0001, - 'cityrand': 0.0001 - } - args.lr = lrs[args.dataset.lower()] / (4 * len(args.gpu_ids)) * args.batch_size - - if args.checkname is None: - args.checkname = 'RFNet' - print(args) - torch.manual_seed(args.seed) - - return args - -def val_args(): - parser = argparse.ArgumentParser(description="PyTorch RFNet validation") - parser.add_argument('--dataset', type=str, default='cityscapes', - choices=['citylostfound', 'cityscapes', 'xrlab', 'mapillary'], - help='dataset name (default: cityscapes)') - parser.add_argument('--workers', type=int, default=4, - metavar='N', help='dataloader threads') - parser.add_argument('--base-size', type=int, default=1024, - help='base image size') - parser.add_argument('--crop-size', type=int, default=768, - help='crop image size') - parser.add_argument('--batch-size', type=int, default=6, - help='batch size for training') - parser.add_argument('--val-batch-size', type=int, default=1, - metavar='N', help='input batch size for \ - validating (default: auto)') - parser.add_argument('--test-batch-size', type=int, default=1, - metavar='N', help='input batch size for \ - testing (default: auto)') - parser.add_argument('--num-class', type=int, default=24, - help='number of training classes (default: 24') - parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') - parser.add_argument('--gpu-ids', type=str, default='0', - help='use which gpu to train, must be a \ - comma-separated list of integers only (default=0)') - parser.add_argument('--checkname', type=str, default=None, - help='set the checkpoint name') - parser.add_argument('--weight-path', type=str, default="./models/530_exp3_2.pth", - help='enter your path of the weight') - parser.add_argument('--save-predicted-image', action='store_true', default=False, - help='save predicted images') - parser.add_argument('--color-label-save-path', type=str, - default='./test/color/', - help='path to save label') - parser.add_argument('--merge-label-save-path', type=str, - default='./test/merge/', - help='path to save merged label') - parser.add_argument('--label-save-path', type=str, default='./test/label/', - help='path to save merged label') - parser.add_argument('--merge', action='store_true', default=True, help='merge image and label') - parser.add_argument('--depth', action='store_true', default=False, help='add depth image or not') - - args = parser.parse_args() - args.cuda = not args.no_cuda and torch.cuda.is_available() - if args.cuda: - try: - args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')] - except ValueError: - raise ValueError('Argument --gpu_ids must be a comma-separated list of integers only') - - return args - -def accuracy(y_true, y_pred, **kwargs): - args = val_args() - _, _, test_loader, num_class = make_data_loader(args, test_data=y_true) - evaluator = Evaluator(num_class) - - tbar = tqdm(test_loader, desc='\r') - for i, (sample, img_path) in enumerate(tbar): - if args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - image, target = sample['image'], sample['label'] - if args.cuda: - image, target = image.cuda(args.gpu_ids), target.cuda(args.gpu_ids) - if args.depth: - depth = depth.cuda(args.gpu_ids) - - target[target > evaluator.num_class-1] = 255 - target = target.cpu().numpy() - # Add batch sample into evaluator - evaluator.add_batch(target, y_pred[i]) - - # Test during the training - # Acc = evaluator.Pixel_Accuracy() - CPA = evaluator.Pixel_Accuracy_Class() - mIoU = evaluator.Mean_Intersection_over_Union() - FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union() - - print("CPA:{}, mIoU:{}, fwIoU: {}".format(CPA, mIoU, FWIoU)) - return CPA - -if __name__ == '__main__': - model_path = "/tmp/RFNet/" - if not os.path.exists(model_path): - os.makedirs(model_path) - - p1 = Process(target=exp_train, args=(10,)) - p1.start() - p1.join() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/__init__.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/__init__.py deleted file mode 100755 index ec1f25bf..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/__init__.py +++ /dev/null @@ -1,116 +0,0 @@ -from dataloaders.datasets import cityscapes, citylostfound, cityrand, target, xrlab, e1, mapillary -from torch.utils.data import DataLoader - -def make_data_loader(args, train_data=None, valid_data=None, test_data=None, **kwargs): - - if args.dataset == 'cityscapes': - if train_data is not None: - train_set = cityscapes.CityscapesSegmentation(args, data=train_data, split='train') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - else: - train_loader, num_class = None, cityscapes.CityscapesSegmentation.NUM_CLASSES - - if valid_data is not None: - val_set = cityscapes.CityscapesSegmentation(args, data=valid_data, split='val') - num_class = val_set.NUM_CLASSES - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - else: - val_loader, num_class = None, cityscapes.CityscapesSegmentation.NUM_CLASSES - - if test_data is not None: - test_set = cityscapes.CityscapesSegmentation(args, data=test_data, split='test') - num_class = test_set.NUM_CLASSES - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - else: - test_loader, num_class = None, cityscapes.CityscapesSegmentation.NUM_CLASSES - - return train_loader, val_loader, test_loader, num_class - - if args.dataset == 'citylostfound': - if args.depth: - train_set = citylostfound.CitylostfoundSegmentation(args, split='train') - val_set = citylostfound.CitylostfoundSegmentation(args, split='val') - test_set = citylostfound.CitylostfoundSegmentation(args, split='test') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - else: - train_set = citylostfound.CitylostfoundSegmentation_rgb(args, split='train') - val_set = citylostfound.CitylostfoundSegmentation_rgb(args, split='val') - test_set = citylostfound.CitylostfoundSegmentation_rgb(args, split='test') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - - return train_loader, val_loader, test_loader, num_class - if args.dataset == 'cityrand': - train_set = cityrand.CityscapesSegmentation(args, split='train') - val_set = cityrand.CityscapesSegmentation(args, split='val') - test_set = cityrand.CityscapesSegmentation(args, split='test') - custom_set = cityrand.CityscapesSegmentation(args, split='custom_resize') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - custom_loader = DataLoader(custom_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - - return train_loader, val_loader, test_loader, custom_loader, num_class - - if args.dataset == 'target': - train_set = target.CityscapesSegmentation(args, split='train') - val_set = target.CityscapesSegmentation(args, split='val') - test_set = target.CityscapesSegmentation(args, split='test') - custom_set = target.CityscapesSegmentation(args, split='test') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - custom_loader = DataLoader(custom_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - - return train_loader, val_loader, test_loader, custom_loader, num_class - - if args.dataset == 'xrlab': - train_set = xrlab.CityscapesSegmentation(args, split='train') - val_set = xrlab.CityscapesSegmentation(args, split='val') - test_set = xrlab.CityscapesSegmentation(args, split='test') - custom_set = xrlab.CityscapesSegmentation(args, split='test') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - custom_loader = DataLoader(custom_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - - return train_loader, val_loader, test_loader, custom_loader, num_class - - if args.dataset == 'e1': - train_set = e1.CityscapesSegmentation(args, split='train') - val_set = e1.CityscapesSegmentation(args, split='val') - test_set = e1.CityscapesSegmentation(args, split='test') - custom_set = e1.CityscapesSegmentation(args, split='test') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - custom_loader = DataLoader(custom_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - - return train_loader, val_loader, test_loader, custom_loader, num_class - - if args.dataset == 'mapillary': - train_set = mapillary.CityscapesSegmentation(args, split='train') - val_set = mapillary.CityscapesSegmentation(args, split='val') - test_set = mapillary.CityscapesSegmentation(args, split='test') - custom_set = mapillary.CityscapesSegmentation(args, split='test') - num_class = train_set.NUM_CLASSES - train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, **kwargs) - val_loader = DataLoader(val_set, batch_size=args.val_batch_size, shuffle=False, **kwargs) - test_loader = DataLoader(test_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - custom_loader = DataLoader(custom_set, batch_size=args.test_batch_size, shuffle=False, **kwargs) - - return train_loader, val_loader, test_loader, custom_loader, num_class - - else: - raise NotImplementedError - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms.py deleted file mode 100755 index d63f200a..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms.py +++ /dev/null @@ -1,240 +0,0 @@ -import torch -import random -import numpy as np - -from PIL import Image, ImageOps, ImageFilter - -class Normalize(object): - """Normalize a tensor image with mean and standard deviation. - Args: - mean (tuple): means for each channel. - std (tuple): standard deviations for each channel. - """ - def __init__(self, mean=(0., 0., 0.), std=(1., 1., 1.)): - self.mean = mean - self.std = std - - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - img = np.array(img).astype(np.float32) - depth = np.array(depth).astype(np.float32) - mask = np.array(mask).astype(np.float32) - img /= 255.0 - img -= self.mean - img /= self.std - - # mean and std for original depth images - mean_depth = 0.12176 - std_depth = 0.09752 - - depth /= 255.0 - depth -= mean_depth - depth /= std_depth - - return {'image': img, - 'depth': depth, - 'label': mask} - - -class ToTensor(object): - """Convert Image object in sample to Tensors.""" - - def __call__(self, sample): - # swap color axis because - # numpy image: H x W x C - # torch image: C X H X W - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - img = np.array(img).astype(np.float32).transpose((2, 0, 1)) - depth = np.array(depth).astype(np.float32) - mask = np.array(mask).astype(np.float32) - - img = torch.from_numpy(img).float() - depth = torch.from_numpy(depth).float() - mask = torch.from_numpy(mask).float() - - return {'image': img, - 'depth': depth, - 'label': mask} - -class CropBlackArea(object): - """ - crop black area for depth image - """ - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - width, height = img.size - left = 140 - top = 30 - right = 2030 - bottom = 900 - # crop - img = img.crop((left, top, right, bottom)) - depth = depth.crop((left, top, right, bottom)) - mask = mask.crop((left, top, right, bottom)) - # resize - img = img.resize((width,height), Image.BILINEAR) - depth = depth.resize((width,height), Image.BILINEAR) - mask = mask.resize((width,height), Image.NEAREST) - # img = img.resize((512,1024), Image.BILINEAR) - # depth = depth.resize((512,1024), Image.BILINEAR) - # mask = mask.resize((512,1024), Image.NEAREST) - - return {'image': img, - 'depth': depth, - 'label': mask} - - -class RandomHorizontalFlip(object): - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - if random.random() < 0.5: - img = img.transpose(Image.FLIP_LEFT_RIGHT) - depth = depth.transpose(Image.FLIP_LEFT_RIGHT) - mask = mask.transpose(Image.FLIP_LEFT_RIGHT) - - return {'image': img, - 'depth': depth, - 'label': mask} - - -class RandomRotate(object): - def __init__(self, degree): - self.degree = degree - - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - rotate_degree = random.uniform(-1*self.degree, self.degree) - img = img.rotate(rotate_degree, Image.BILINEAR) - depth = depth.rotate(rotate_degree, Image.BILINEAR) - mask = mask.rotate(rotate_degree, Image.NEAREST) - - return {'image': img, - 'depth': depth, - 'label': mask} - - -class RandomGaussianBlur(object): - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - if random.random() < 0.5: - img = img.filter(ImageFilter.GaussianBlur( - radius=random.random())) - - return {'image': img, - 'depth': depth, - 'label': mask} - - -class RandomScaleCrop(object): - def __init__(self, base_size, crop_size, fill=0): - self.base_size = base_size - self.crop_size = crop_size - self.fill = fill - - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - # random scale (short edge) - short_size = random.randint(int(self.base_size * 0.5), int(self.base_size * 2.0)) - w, h = img.size - if h > w: - ow = short_size - oh = int(1.0 * h * ow / w) - else: - oh = short_size - ow = int(1.0 * w * oh / h) - img = img.resize((ow, oh), Image.BILINEAR) - depth = depth.resize((ow, oh), Image.BILINEAR) - mask = mask.resize((ow, oh), Image.NEAREST) - # pad crop - if short_size < self.crop_size: - padh = self.crop_size - oh if oh < self.crop_size else 0 - padw = self.crop_size - ow if ow < self.crop_size else 0 - img = ImageOps.expand(img, border=(0, 0, padw, padh), fill=0) - depth = ImageOps.expand(depth, border=(0, 0, padw, padh), fill=0) # depth多余的部分填0 - mask = ImageOps.expand(mask, border=(0, 0, padw, padh), fill=self.fill) - # random crop crop_size - w, h = img.size - x1 = random.randint(0, w - self.crop_size) - y1 = random.randint(0, h - self.crop_size) - img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - depth = depth.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - - return {'image': img, - 'depth': depth, - 'label': mask} - - -class FixScaleCrop(object): - def __init__(self, crop_size): - self.crop_size = crop_size - - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - w, h = img.size - if w > h: - oh = self.crop_size - ow = int(1.0 * w * oh / h) - else: - ow = self.crop_size - oh = int(1.0 * h * ow / w) - img = img.resize((ow, oh), Image.BILINEAR) - depth = depth.resize((ow, oh), Image.BILINEAR) - mask = mask.resize((ow, oh), Image.NEAREST) - # center crop - w, h = img.size - x1 = int(round((w - self.crop_size) / 2.)) - y1 = int(round((h - self.crop_size) / 2.)) - img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - depth = depth.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - - return {'image': img, - 'depth': depth, - 'label': mask} - -class FixedResize(object): - def __init__(self, size): - self.size = (size, size) # size: (h, w) - - def __call__(self, sample): - img = sample['image'] - depth = sample['depth'] - mask = sample['label'] - - assert img.size == depth.size == mask.size - - img = img.resize(self.size, Image.BILINEAR) - depth = depth.resize(self.size, Image.BILINEAR) - mask = mask.resize(self.size, Image.NEAREST) - - return {'image': img, - 'depth': depth, - 'label': mask} - -class Relabel(object): - def __init__(self, olabel, nlabel): # change trainid label from olabel to nlabel - self.olabel = olabel - self.nlabel = nlabel - - def __call__(self, tensor): - # assert (isinstance(tensor, torch.LongTensor) or isinstance(tensor, - # torch.ByteTensor)), 'tensor needs to be LongTensor' - tensor[tensor == self.olabel] = self.nlabel - return tensor \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms_rgb.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms_rgb.py deleted file mode 100755 index e04ef5a3..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/custom_transforms_rgb.py +++ /dev/null @@ -1,230 +0,0 @@ -import torch -import random -import numpy as np - -from PIL import Image, ImageOps, ImageFilter - -class Normalize(object): - """Normalize a tensor image with mean and standard deviation. - Args: - mean (tuple): means for each channel. - std (tuple): standard deviations for each channel. - """ - def __init__(self, mean=(0., 0., 0.), std=(1., 1., 1.)): - self.mean = mean - self.std = std - - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - img = np.array(img).astype(np.float32) - mask = np.array(mask).astype(np.float32) - img /= 255.0 - img -= self.mean - img /= self.std - - return {'image': img, - 'label': mask} - - -class Normalize_test(object): - def __init__(self, mean=(0., 0., 0.), std=(1., 1., 1.)): - self.mean = mean - self.std = std - - def __call__(self, sample): - img = sample - img = np.array(img).astype(np.float32) - img /= 255.0 - img -= self.mean - img /= self.std - - return img - - -class ToTensor(object): - """Convert Image object in sample to Tensors.""" - - def __call__(self, sample): - # swap color axis because - # numpy image: H x W x C - # torch image: C X H X W - img = sample['image'] - mask = sample['label'] - img = np.array(img).astype(np.float32).transpose((2, 0, 1)) - mask = np.array(mask).astype(np.float32) - - img = torch.from_numpy(img).float() - mask = torch.from_numpy(mask).float() - - return {'image': img, - 'label': mask} - -class CropBlackArea(object): - """ - crop black area for depth image - """ - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - width, height = img.size - left = 140 - top = 30 - right = 2030 - bottom = 900 - # crop - img = img.crop((left, top, right, bottom)) - mask = mask.crop((left, top, right, bottom)) - # resize - img = img.resize((width,height), Image.BILINEAR) - mask = mask.resize((width,height), Image.NEAREST) - # img = img.resize((512,1024), Image.BILINEAR) - # mask = mask.resize((512,1024), Image.NEAREST) - print(img.size) - - return {'image': img, - 'label': mask} - -class ToTensor_test(object): - """Convert Image object in sample to Tensors.""" - - def __call__(self, sample): - # swap color axis because - # numpy image: H x W x C - # torch image: C X H X W - img = sample - img = np.array(img).astype(np.float32).transpose((2, 0, 1)) - - img = torch.from_numpy(img).float() - - return img - - -class RandomHorizontalFlip(object): - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - if random.random() < 0.5: - img = img.transpose(Image.FLIP_LEFT_RIGHT) - mask = mask.transpose(Image.FLIP_LEFT_RIGHT) - - return {'image': img, - 'label': mask} - - -class RandomRotate(object): - def __init__(self, degree): - self.degree = degree - - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - rotate_degree = random.uniform(-1*self.degree, self.degree) - img = img.rotate(rotate_degree, Image.BILINEAR) - mask = mask.rotate(rotate_degree, Image.NEAREST) - - return {'image': img, - 'label': mask} - - -class RandomGaussianBlur(object): - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - if random.random() < 0.5: - img = img.filter(ImageFilter.GaussianBlur( - radius=random.random())) - - return {'image': img, - 'label': mask} - - -class RandomScaleCrop(object): - def __init__(self, base_size, crop_size, fill=0): - self.base_size = base_size - self.crop_size = crop_size - self.fill = fill - - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - # random scale (short edge) - short_size = random.randint(int(self.base_size * 0.5), int(self.base_size * 2.0)) - w, h = img.size - if h > w: - ow = short_size - oh = int(1.0 * h * ow / w) - else: - oh = short_size - ow = int(1.0 * w * oh / h) - img = img.resize((ow, oh), Image.BILINEAR) - mask = mask.resize((ow, oh), Image.NEAREST) - # pad crop - if short_size < self.crop_size: - padh = self.crop_size - oh if oh < self.crop_size else 0 - padw = self.crop_size - ow if ow < self.crop_size else 0 - img = ImageOps.expand(img, border=(0, 0, padw, padh), fill=0) - mask = ImageOps.expand(mask, border=(0, 0, padw, padh), fill=self.fill) - # random crop crop_size - w, h = img.size - x1 = random.randint(0, w - self.crop_size) - y1 = random.randint(0, h - self.crop_size) - img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - - return {'image': img, - 'label': mask} - - -class FixScaleCrop(object): - def __init__(self, crop_size): - self.crop_size = crop_size - - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - w, h = img.size - if w > h: - oh = self.crop_size - ow = int(1.0 * w * oh / h) - else: - ow = self.crop_size - oh = int(1.0 * h * ow / w) - img = img.resize((ow, oh), Image.BILINEAR) - mask = mask.resize((ow, oh), Image.NEAREST) - # center crop - w, h = img.size - x1 = int(round((w - self.crop_size) / 2.)) - y1 = int(round((h - self.crop_size) / 2.)) - img = img.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - mask = mask.crop((x1, y1, x1 + self.crop_size, y1 + self.crop_size)) - - return {'image': img, - 'label': mask} - -class FixedResize(object): - def __init__(self, size): - self.size = (size, size) # size: (h, w) - - def __call__(self, sample): - img = sample['image'] - mask = sample['label'] - - assert img.size == mask.size - - img = img.resize(self.size, Image.BILINEAR) - mask = mask.resize(self.size, Image.NEAREST) - - return {'image': img, - 'label': mask} - -class Relabel(object): - def __init__(self, olabel, nlabel): # change trainid label from olabel to nlabel - self.olabel = olabel - self.nlabel = nlabel - - def __call__(self, tensor): - # assert (isinstance(tensor, torch.LongTensor) or isinstance(tensor, - # torch.ByteTensor)), 'tensor needs to be LongTensor' - tensor[tensor == self.olabel] = self.nlabel - return tensor \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/__init__.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/citylostfound.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/citylostfound.py deleted file mode 100755 index 6ffd0a4b..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/citylostfound.py +++ /dev/null @@ -1,276 +0,0 @@ -import os -import numpy as np -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr -from dataloaders import custom_transforms_rgb as tr_rgb - -class CitylostfoundSegmentation(data.Dataset): - NUM_CLASSES = 20 - - def __init__(self, args, root=Path.db_root_dir('citylostfound'), split="train"): - - self.root = root - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.disparities_base = os.path.join(self.root,'disparity',self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.images[split] = self.recursive_glob(rootdir=self.images_base, suffix= '.png') - self.images[split].sort() - - self.disparities[split] = self.recursive_glob(rootdir=self.disparities_base, suffix= '.png') - self.disparities[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, - suffix='labelTrainIds.png') - self.labels[split].sort() - - self.ignore_index = 255 - - if not self.images[split]: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if not self.disparities[split]: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - lbl_path = self.labels[self.split][index].rstrip() - - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _tmp = np.array(Image.open(lbl_path), dtype=np.uint8) - if self.split == 'train': - if index < 1036: # lostandfound - _tmp = self.relabel_lostandfound(_tmp) - else: # cityscapes - pass - elif self.split == 'val': - if index < 1203: # lostandfound - _tmp = self.relabel_lostandfound(_tmp) - else: # cityscapes - pass - _target = Image.fromarray(_tmp) - - sample = {'image': _img, 'depth': _depth, 'label': _target} - - # data augment - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample) - - - def relabel_lostandfound(self, input): - input = tr.Relabel(0, self.ignore_index)(input) # background->255 ignore - input = tr.Relabel(1, 0)(input) # road 1->0 - # input = Relabel(255, 20)(input) # unlabel 20 - input = tr.Relabel(2, 19)(input) # obstacle 19 - return input - - def recursive_glob(self, rootdir='.', suffix=None): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - if isinstance(suffix, str): - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - elif isinstance(suffix, list): - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for x in suffix for filename in filenames if filename.startswith(x)] - - - def transform_tr(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - # tr.CropBlackArea(), - tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - -class CitylostfoundSegmentation_rgb(data.Dataset): - NUM_CLASSES = 19 - - def __init__(self, args, root=Path.db_root_dir('citylostfound'), split="train"): - - self.root = root - self.split = split - self.args = args - self.files = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.files[split] = self.recursive_glob(rootdir=self.images_base, suffix='.png') - self.files[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, suffix='labelTrainIds.png') - self.labels[split].sort() - - self.ignore_index = 255 - - if not self.files[split]: - raise Exception("No files for split=[%s] found in %s" % (split, self.images_base)) - - print("Found %d %s images" % (len(self.files[split]), split)) - - def __len__(self): - return len(self.files[self.split]) - - def __getitem__(self, index): - - img_path = self.files[self.split][index].rstrip() - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _tmp = np.array(Image.open(lbl_path), dtype=np.uint8) - if self.split == 'train': - if index < 1036: # lostandfound - _tmp = self.relabel_lostandfound(_tmp) - else: # cityscapes - pass - elif self.split == 'val': - if index < 1203: # lostandfound - _tmp = self.relabel_lostandfound(_tmp) - else: # cityscapes - pass - _target = Image.fromarray(_tmp) - - sample = {'image': _img, 'label': _target} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample) - - - def relabel_lostandfound(self, input): - input = tr.Relabel(0, self.ignore_index)(input) - input = tr.Relabel(1, 0)(input) # road 1->0 - input = tr.Relabel(2, 19)(input) # obstacle 19 - return input - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - tr_rgb.CropBlackArea(), - tr_rgb.RandomHorizontalFlip(), - tr_rgb.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr_rgb.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr_rgb.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr_rgb.CropBlackArea(), - tr_rgb.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr_rgb.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - tr_rgb.FixedResize(size=self.args.crop_size), - tr_rgb.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr_rgb.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 513 - args.crop_size = 513 - - cityscapes_train = CitylostfoundSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityrand.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityrand.py deleted file mode 100755 index 74eddb67..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityrand.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import numpy as np -import scipy.misc as m -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr - -class CityscapesSegmentation(data.Dataset): - NUM_CLASSES = 19 - - def __init__(self, args, root=Path.db_root_dir('cityrand'), split="train"): - - self.root = root - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.disparities_base = os.path.join(self.root, 'disparity', self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.images[split] = self.recursive_glob(rootdir=self.images_base, suffix='.png') - self.images[split].sort() - - self.disparities[split] = self.recursive_glob(rootdir=self.disparities_base, suffix='.png') - self.disparities[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, suffix='TrainIds.png') - self.labels[split].sort() - - - self.ignore_index = 255 - - if not self.images[split]: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if not self.disparities[split]: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - #print(index) - try: - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _target = Image.open(lbl_path) - sample = {'image': _img,'depth':_depth, 'label': _target} - except: - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - sample = {'image': _img,'depth':_depth, 'label': _img} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample), img_path - elif self.split == 'custom_resize': - return self.transform_ts(sample), img_path - - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - #tr.CropBlackArea(), - #tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 513 - args.crop_size = 513 - - cityscapes_train = CityscapesSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityscapes.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityscapes.py deleted file mode 100755 index 4396f5f4..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/cityscapes.py +++ /dev/null @@ -1,160 +0,0 @@ -import os -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" -import numpy as np -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr - -class CityscapesSegmentation(data.Dataset): - NUM_CLASSES = 30 # 25 - - def __init__(self, args, root=Path.db_root_dir('cityscapes'), data=None, split="train"): - - # self.root = root - self.root = "/home/lsq/Dataset/" - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.disparities_base = os.path.join(self.root, self.split, "depth", "cityscapes_real") - self.images[split] = [img[0] for img in data.x] if hasattr(data, "x") else data - - - if hasattr(data, "x") and len(data.x[0]) == 1: - # TODO: fit the case that depth images don't exist. - self.disparities[split] = self.images[split] - elif hasattr(data, "x") and len(data.x[0]) == 2: - self.disparities[split] = [img[1] for img in data.x] - else: - if len(data[0]) == 2: - self.images[split] = [img[0] for img in data] - self.disparities[split] = [img[1] for img in data] - elif len(data[0]) == 1: - self.images[split] = [img[0] for img in data] - self.disparities[split] = [img[0] for img in data] - else: - self.images[split] = data - self.disparities[split] = data - - self.labels[split] = data.y if hasattr(data, "y") else data - - self.ignore_index = 255 - - if len(self.images[split]) == 0: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if len(self.disparities[split]) == 0: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - try: - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _target = Image.open(lbl_path) - sample = {'image': _img,'depth':_depth, 'label': _target} - except: - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - sample = {'image': _img,'depth':_depth, 'label': _img} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample), img_path - elif self.split == 'custom_resize': - return self.transform_ts(sample), img_path - - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - #tr.CropBlackArea(), - #tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 513 - args.crop_size = 513 - - cityscapes_train = CityscapesSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/e1.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/e1.py deleted file mode 100755 index 40e06e98..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/e1.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import numpy as np -import scipy.misc as m -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr - -class CityscapesSegmentation(data.Dataset): - NUM_CLASSES = 24 - - def __init__(self, args, root=Path.db_root_dir('e1'), split="train"): - - self.root = root - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.disparities_base = os.path.join(self.root, 'disparity', self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.images[split] = self.recursive_glob(rootdir=self.images_base, suffix='.png') - self.images[split].sort() - - self.disparities[split] = self.recursive_glob(rootdir=self.disparities_base, suffix='.png') - self.disparities[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, suffix='.png') - self.labels[split].sort() - - self.ignore_index = 255 - - if not self.images[split]: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if not self.disparities[split]: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - #print(index) - try: - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _target = Image.open(lbl_path) - sample = {'image': _img,'depth':_depth, 'label': _target} - except: - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - sample = {'image': _img,'depth':_depth, 'label': _img} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample), img_path - elif self.split == 'custom_resize': - return self.transform_ts(sample), img_path - - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - #tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - #tr.CropBlackArea(), - #tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 513 - args.crop_size = 513 - - cityscapes_train = CityscapesSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/mapillary.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/mapillary.py deleted file mode 100755 index d665649b..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/mapillary.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import numpy as np -import scipy.misc as m -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr - -class CityscapesSegmentation(data.Dataset): - NUM_CLASSES = 24 - - def __init__(self, args, root=Path.db_root_dir('mapillary'), split="train"): - - self.root = root - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.disparities_base = os.path.join(self.root, 'disparity', self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.images[split] = self.recursive_glob(rootdir=self.images_base, suffix='.png') - self.images[split].sort() - - self.disparities[split] = self.recursive_glob(rootdir=self.disparities_base, suffix='.png') - self.disparities[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, suffix='.png') - self.labels[split].sort() - - - self.ignore_index = 255 - - if not self.images[split]: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if not self.disparities[split]: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - #print(index) - try: - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _target = Image.open(lbl_path) - sample = {'image': _img,'depth':_depth, 'label': _target} - except: - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - sample = {'image': _img,'depth':_depth, 'label': _img} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample), img_path - elif self.split == 'custom_resize': - return self.transform_ts(sample), img_path - - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - #tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 768 - args.crop_size = 768 - - cityscapes_train = CityscapesSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/target.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/target.py deleted file mode 100755 index 739e85f8..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/target.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import numpy as np -import scipy.misc as m -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr - -class CityscapesSegmentation(data.Dataset): - NUM_CLASSES = 24 - - def __init__(self, args, root=Path.db_root_dir('target'), split="train"): - - self.root = root - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.disparities_base = os.path.join(self.root, 'disparity', self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.images[split] = self.recursive_glob(rootdir=self.images_base, suffix='.png') - self.images[split].sort() - - self.disparities[split] = self.recursive_glob(rootdir=self.disparities_base, suffix='.png') - self.disparities[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, suffix='TrainIds.png') - self.labels[split].sort() - - - self.ignore_index = 255 - - if not self.images[split]: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if not self.disparities[split]: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - #print(index) - try: - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _target = Image.open(lbl_path) - sample = {'image': _img,'depth':_depth, 'label': _target} - except: - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - sample = {'image': _img,'depth':_depth, 'label': _img} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample), img_path - elif self.split == 'custom_resize': - return self.transform_ts(sample), img_path - - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - #tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 513 - args.crop_size = 513 - - cityscapes_train = CityscapesSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/temp.txt b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/temp.txt deleted file mode 100755 index 3c81afef..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/temp.txt +++ /dev/null @@ -1,38 +0,0 @@ -for i in range(len(nam_label)): - img_label = cv2.imread(label_ori_path+nam_label[i], -1)[:,:,2] - img_label_temp = img_label.copy() - img_label_temp[img_label == 0] = 22 - img_label_temp[img_label == 1] = 10 - img_label_temp[img_label == 2] = 2 - img_label_temp[img_label == 3] = 0 - img_label_temp[img_label == 4] = 1 - img_label_temp[img_label == 5] = 4 - img_label_temp[img_label == 6] = 8 - img_label_temp[img_label == 7] = 5 - img_label_temp[img_label == 8] = 13 - img_label_temp[img_label == 9] = 7 - img_label_temp[img_label == 10] = 11 - img_label_temp[img_label == 11] = 18 - img_label_temp[img_label == 12] = 17 - img_label_temp[img_label == 13] = 21 - img_label_temp[img_label == 14] = 20 - img_label_temp[img_label == 15] = 6 - img_label_temp[img_label == 16] = 9 - img_label_temp[img_label == 17] = 12 - img_label_temp[img_label == 18] = 14 - img_label_temp[img_label == 19] = 15 - img_label_temp[img_label == 20] = 16 - img_label_temp[img_label == 21] = 3 - img_label_temp[img_label == 22] = 19 - #print(img_label) - #img_label[img_label == 0] = 10 - #img_label[img_label == 6] = 0 - #img_label[img_label == 5] = 11 - #img_label[img_label == 1] = 5 - #img_label[img_label == 2] = 1 - #img_label[img_label == 4] = 9 - #img_label[img_label == 3] = 4 - #img_label[img_label == 7] = 8 - #img_label[img_label == 11] = 2 - img_resize_lab = cv2.resize(img_label_temp, (2048,1024), interpolation=cv2.INTER_NEAREST) - cv2.imwrite(label_save_path+str(i)+'TrainIds.png', img_resize_lab.astype(np.uint16)) \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/xrlab.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/xrlab.py deleted file mode 100755 index 4b261fcd..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/datasets/xrlab.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import numpy as np -import scipy.misc as m -from PIL import Image -from torch.utils import data -from mypath import Path -from torchvision import transforms -from dataloaders import custom_transforms as tr - -class CityscapesSegmentation(data.Dataset): - NUM_CLASSES = 25 - - def __init__(self, args, root=Path.db_root_dir('xrlab'), split="train"): - - self.root = root - self.split = split - self.args = args - self.images = {} - self.disparities = {} - self.labels = {} - - self.images_base = os.path.join(self.root, 'leftImg8bit', self.split) - self.disparities_base = os.path.join(self.root, 'disparity', self.split) - self.annotations_base = os.path.join(self.root, 'gtFine', self.split) - - self.images[split] = self.recursive_glob(rootdir=self.images_base, suffix='.png') - self.images[split].sort() - - self.disparities[split] = self.recursive_glob(rootdir=self.disparities_base, suffix='.png') - self.disparities[split].sort() - - self.labels[split] = self.recursive_glob(rootdir=self.annotations_base, suffix='.png') - self.labels[split].sort() - - - self.ignore_index = 255 - - if not self.images[split]: - raise Exception("No RGB images for split=[%s] found in %s" % (split, self.images_base)) - if not self.disparities[split]: - raise Exception("No depth images for split=[%s] found in %s" % (split, self.disparities_base)) - - print("Found %d %s RGB images" % (len(self.images[split]), split)) - print("Found %d %s disparity images" % (len(self.disparities[split]), split)) - - - def __len__(self): - return len(self.images[self.split]) - - def __getitem__(self, index): - - img_path = self.images[self.split][index].rstrip() - disp_path = self.disparities[self.split][index].rstrip() - #print(index) - try: - lbl_path = self.labels[self.split][index].rstrip() - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - _target = Image.open(lbl_path) - sample = {'image': _img,'depth':_depth, 'label': _target} - except: - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(disp_path) - sample = {'image': _img,'depth':_depth, 'label': _img} - - if self.split == 'train': - return self.transform_tr(sample) - elif self.split == 'val': - return self.transform_val(sample), img_path - elif self.split == 'test': - return self.transform_ts(sample), img_path - elif self.split == 'custom_resize': - return self.transform_ts(sample), img_path - - - def recursive_glob(self, rootdir='.', suffix=''): - """Performs recursive glob with given suffix and rootdir - :param rootdir is the root directory - :param suffix is the suffix to be searched - """ - return [os.path.join(looproot, filename) - for looproot, _, filenames in os.walk(rootdir) - for filename in filenames if filename.endswith(suffix)] - - def transform_tr(self, sample): - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.RandomHorizontalFlip(), - tr.RandomScaleCrop(base_size=self.args.base_size, crop_size=self.args.crop_size, fill=255), - # tr.RandomGaussianBlur(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_val(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - - def transform_ts(self, sample): - - composed_transforms = transforms.Compose([ - tr.CropBlackArea(), - #tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - return composed_transforms(sample) - -if __name__ == '__main__': - from dataloaders.utils import decode_segmap - from torch.utils.data import DataLoader - import matplotlib.pyplot as plt - import argparse - - parser = argparse.ArgumentParser() - args = parser.parse_args() - args.base_size = 513 - args.crop_size = 513 - - cityscapes_train = CityscapesSegmentation(args, split='train') - - dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) - - for ii, sample in enumerate(dataloader): - for jj in range(sample["image"].size()[0]): - img = sample['image'].numpy() - gt = sample['label'].numpy() - tmp = np.array(gt[jj]).astype(np.uint8) - segmap = decode_segmap(tmp, dataset='cityscapes') - img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) - img_tmp *= (0.229, 0.224, 0.225) - img_tmp += (0.485, 0.456, 0.406) - img_tmp *= 255.0 - img_tmp = img_tmp.astype(np.uint8) - plt.figure() - plt.title('display') - plt.subplot(211) - plt.imshow(img_tmp) - plt.subplot(212) - plt.imshow(segmap) - - if ii == 1: - break - - plt.show(block=True) - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/utils.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/utils.py deleted file mode 100755 index ef572332..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/dataloaders/utils.py +++ /dev/null @@ -1,244 +0,0 @@ -import matplotlib.pyplot as plt -import numpy as np -import torch - -def decode_seg_map_sequence(label_masks, dataset='pascal'): - rgb_masks = [] - for label_mask in label_masks: - rgb_mask = decode_segmap(label_mask, dataset) - rgb_masks.append(rgb_mask) - rgb_masks = torch.from_numpy(np.array(rgb_masks).transpose([0, 3, 1, 2])) # change for val - return rgb_masks - - -def decode_segmap(label_mask, dataset, plot=False): - """Decode segmentation class labels into a color image - Args: - label_mask (np.ndarray): an (M,N) array of integer values denoting - the class label at each spatial location. - plot (bool, optional): whether to show the resulting color image - in a figure. - Returns: - (np.ndarray, optional): the resulting decoded color image. - """ - if dataset == 'pascal' or dataset == 'coco': - n_classes = 21 - label_colours = get_pascal_labels() - elif dataset == 'cityscapes': - n_classes = 19 - label_colours = get_cityscapes_labels() - elif dataset == 'target': - n_classes = 24 - label_colours = get_cityscapes_labels() - elif dataset == 'cityrand': - n_classes = 19 - label_colours = get_cityscapes_labels() - elif dataset == 'citylostfound': - n_classes = 20 - label_colours = get_citylostfound_labels() - elif dataset == 'xrlab': - n_classes = 25 - label_colours = get_cityscapes_labels() - elif dataset == 'e1': - n_classes = 24 - label_colours = get_cityscapes_labels() - elif dataset == 'mapillary': - n_classes = 24 - label_colours = get_cityscapes_labels() - else: - raise NotImplementedError - - r = label_mask.copy() - g = label_mask.copy() - b = label_mask.copy() - for ll in range(0, n_classes): - r[label_mask == ll] = label_colours[ll, 0] - g[label_mask == ll] = label_colours[ll, 1] - b[label_mask == ll] = label_colours[ll, 2] - rgb = np.zeros((label_mask.shape[0], label_mask.shape[1], 3)) # change for val - # rgb = torch.ByteTensor(3, label_mask.shape[0], label_mask.shape[1]).fill_(0) - rgb[:, :, 0] = r / 255.0 - rgb[:, :, 1] = g / 255.0 - rgb[:, :, 2] = b / 255.0 - # r = torch.from_numpy(r) - # g = torch.from_numpy(g) - # b = torch.from_numpy(b) - - rgb[:, :, 0] = r / 255.0 - rgb[:, :, 1] = g / 255.0 - rgb[:, :, 2] = b / 255.0 - if plot: - plt.imshow(rgb) - plt.show() - else: - return rgb - - -def encode_segmap(mask): - """Encode segmentation label images as pascal classes - Args: - mask (np.ndarray): raw segmentation label image of dimension - (M, N, 3), in which the Pascal classes are encoded as colours. - Returns: - (np.ndarray): class map with dimensions (M,N), where the value at - a given location is the integer denoting the class index. - """ - mask = mask.astype(int) - label_mask = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.int16) - for ii, label in enumerate(get_pascal_labels()): - label_mask[np.where(np.all(mask == label, axis=-1))[:2]] = ii - label_mask = label_mask.astype(int) - return label_mask - - -def get_cityscapes_labels(): - return np.array([ - [128, 64, 128], - [244, 35, 232], - [70, 70, 70], - [102, 102, 156], - [190, 153, 153], - [153, 153, 153], - [250, 170, 30], - [220, 220, 0], - [107, 142, 35], - [152, 251, 152], - [0, 130, 180], - [220, 20, 60], - [255, 0, 0], - [0, 0, 142], - [0, 0, 70], - [0, 60, 100], - [0, 80, 100], - [0, 0, 230], - [119, 11, 32], - [119, 11, 119], - [128, 64, 64], - [102, 10, 156], - [102, 102, 15], - [10, 102, 156], - [10, 102, 156], - [10, 102, 156], - [10, 102, 156]]) - -def get_citylostfound_labels(): - return np.array([ - [128, 64, 128], - [244, 35, 232], - [70, 70, 70], - [102, 102, 156], - [190, 153, 153], - [153, 153, 153], - [250, 170, 30], - [220, 220, 0], - [107, 142, 35], - [152, 251, 152], - [0, 130, 180], - [220, 20, 60], - [255, 0, 0], - [0, 0, 142], - [0, 0, 70], - [0, 60, 100], - [0, 80, 100], - [0, 0, 230], - [119, 11, 32], - [111, 74, 0]]) - - -def get_pascal_labels(): - """Load the mapping that associates pascal classes with label colors - Returns: - np.ndarray with dimensions (21, 3) - """ - return np.asarray([[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], - [0, 0, 128], [128, 0, 128], [0, 128, 128], [128, 128, 128], - [64, 0, 0], [192, 0, 0], [64, 128, 0], [192, 128, 0], - [64, 0, 128], [192, 0, 128], [64, 128, 128], [192, 128, 128], - [0, 64, 0], [128, 64, 0], [0, 192, 0], [128, 192, 0], - [0, 64, 128]]) - - -def colormap_bdd(n): - cmap=np.zeros([n, 3]).astype(np.uint8) - cmap[0,:] = np.array([128, 64, 128]) - cmap[1,:] = np.array([244, 35, 232]) - cmap[2,:] = np.array([ 70, 70, 70]) - cmap[3,:] = np.array([102, 102, 156]) - cmap[4,:] = np.array([190, 153, 153]) - cmap[5,:] = np.array([153, 153, 153]) - - cmap[6,:] = np.array([250, 170, 30]) - cmap[7,:] = np.array([220, 220, 0]) - cmap[8,:] = np.array([107, 142, 35]) - cmap[9,:] = np.array([152, 251, 152]) - cmap[10,:]= np.array([70, 130, 180]) - - cmap[11,:]= np.array([220, 20, 60]) - cmap[12,:]= np.array([255, 0, 0]) - cmap[13,:]= np.array([0, 0, 142]) - cmap[14,:]= np.array([0, 0, 70]) - cmap[15,:]= np.array([0, 60, 100]) - - cmap[16,:]= np.array([0, 80, 100]) - cmap[17,:]= np.array([0, 0, 230]) - cmap[18,:]= np.array([119, 11, 32]) - cmap[19,:]= np.array([111, 74, 0]) #多加了一类small obstacle - - return cmap - -def colormap_bdd0(n): - cmap=np.zeros([n, 3]).astype(np.uint8) - cmap[0,:] = np.array([0, 0, 0]) - cmap[1,:] = np.array([70, 130, 180]) - cmap[2,:] = np.array([70, 70, 70]) - cmap[3,:] = np.array([128, 64, 128]) - cmap[4,:] = np.array([244, 35, 232]) - cmap[5,:] = np.array([64, 64, 128]) - - cmap[6,:] = np.array([107, 142, 35]) - cmap[7,:] = np.array([153, 153, 153]) - cmap[8,:] = np.array([0, 0, 142]) - cmap[9,:] = np.array([220, 220, 0]) - cmap[10,:]= np.array([220, 20, 60]) - - cmap[11,:]= np.array([119, 11, 32]) - cmap[12,:]= np.array([0, 0, 230]) - cmap[13,:]= np.array([250, 170, 160]) - cmap[14,:]= np.array([128, 64, 64]) - cmap[15,:]= np.array([250, 170, 30]) - - cmap[16,:]= np.array([152, 251, 152]) - cmap[17,:]= np.array([255, 0, 0]) - cmap[18,:]= np.array([0, 0, 70]) - cmap[19,:]= np.array([0, 60, 100]) #small obstacle - cmap[20,:]= np.array([0, 80, 100]) - cmap[21,:]= np.array([102, 102, 156]) - cmap[22,:]= np.array([102, 102, 156]) - - return cmap - -class Colorize: - - def __init__(self, n=24): # n = nClasses - # self.cmap = colormap(256) - self.cmap = colormap_bdd(256) - self.cmap[n] = self.cmap[-1] - self.cmap = torch.from_numpy(self.cmap[:n]) - - def __call__(self, gray_image): - size = gray_image.size() - # print(size) - color_images = torch.ByteTensor(size[0], 3, size[1], size[2]).fill_(0) - # color_image = torch.ByteTensor(3, size[0], size[1]).fill_(0) - - # for label in range(1, len(self.cmap)): - for i in range(color_images.shape[0]): - for label in range(0, len(self.cmap)): - mask = gray_image[0] == label - # mask = gray_image == label - - color_images[i][0][mask] = self.cmap[label][0] - color_images[i][1][mask] = self.cmap[label][1] - color_images[i][2][mask] = self.cmap[label][2] - - return color_images diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py deleted file mode 100755 index da3538b8..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py +++ /dev/null @@ -1,537 +0,0 @@ -import argparse -import os -import numpy as np -from tqdm import tqdm -import time -import torch -from torchvision.transforms import ToPILImage -from PIL import Image - -from segment_anything import sam_model_registry, SamAutomaticMaskGenerator -import mmcv -import pycocotools.mask as maskUtils -from mmdet.visualization.image import imshow_det_bboxes -import pickle - -from dataloaders import make_data_loader -from dataloaders.utils import decode_seg_map_sequence, Colorize -from utils.metrics import Evaluator -from models.rfnet import RFNet -from models import rfnet_for_unseen -from models.resnet.resnet_single_scale_single_attention import * -from models.resnet import resnet_single_scale_single_attention_unseen -import torch.backends.cudnn as cudnn -from cityscapes_id2label import CONFIG as CONFIG_CITYSCAPES_ID2LABEL -import torch.nn.functional as F -from transformers import SegformerFeatureExtractor, SegformerForSemanticSegmentation - -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" - -class Validator(object): - def __init__(self, args, data=None, unseen_detection=False): - self.args = args - self.time_train = [] - self.num_class = args.num_class - - # Define Dataloader - kwargs = {'num_workers': args.workers, 'pin_memory': False} - # _, self.val_loader, _, self.custom_loader, self.num_class = make_data_loader(args, **kwargs) - _, _, self.test_loader, _ = make_data_loader(args, test_data=data, **kwargs) - print('un_classes:'+str(self.num_class)) - - # Define evaluator - self.evaluator = Evaluator(self.num_class) - - # Define network - if unseen_detection: - self.resnet = resnet_single_scale_single_attention_unseen.\ - resnet18(pretrained=False, efficient=False, use_bn=True) - self.model = rfnet_for_unseen.RFNet(self.resnet, num_classes=self.num_class, use_bn=True) - else: - self.resnet = resnet18(pretrained=False, efficient=False, use_bn=True) - self.model = RFNet(self.resnet, num_classes=self.num_class, use_bn=True) - - if args.cuda: - #self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids) - self.model = self.model.cuda(args.gpu_ids) - cudnn.benchmark = True # accelarate speed - print('Model loaded successfully!') - - # # Load weights - # assert os.path.exists(args.weight_path), 'weight-path:{} doesn\'t exit!'.format(args.weight_path) - # self.new_state_dict = torch.load(args.weight_path, map_location=torch.device("cpu")) - # self.model = load_my_state_dict(self.model, self.new_state_dict['state_dict']) - - def segformer_segmentation(self, image, processor, model, rank): - h, w, _ = image.shape - inputs = processor(images=image, return_tensors="pt").to(rank) - outputs = model(**inputs) - logits = outputs.logits - logits = F.interpolate(logits, size=(h, w), mode='bilinear', align_corners=True) - predicted_semantic_map = logits.argmax(dim=1) - return predicted_semantic_map - - def draw_mask(self, image_name, mask, output_path): - img = mmcv.imread(image_name) - anns = {'annotations': mask} - anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) - semantc_mask = torch.zeros(1024, 2048) - i = 0 - for ann in anns['annotations']: - valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() - semantc_mask[valid_mask] = i - i += 1 - sematic_class_in_img = torch.unique(semantc_mask) - semantic_bitmasks, semantic_class_names = [], [] - - # semantic prediction - for i in range(len(sematic_class_in_img)): - class_name = str(i) - class_mask = semantc_mask == sematic_class_in_img[i] - class_mask = class_mask.cpu().numpy().astype(np.uint8) - semantic_class_names.append(class_name) - semantic_bitmasks.append(class_mask) - - length = len(image_name) - for i in range(length): - if image_name[length-i-1] == '_': - break - filename = image_name[length-i:] - imshow_det_bboxes(img, - bboxes=None, - labels=np.arange(len(sematic_class_in_img)), - segms=np.stack(semantic_bitmasks), - class_names=semantic_class_names, - font_size=25, - show=False, - out_file=os.path.join(output_path, filename + '_mask.png')) - print('[Save] save mask: ', os.path.join(output_path, filename + '_mask.png')) - semantc_mask = semantc_mask.unsqueeze(0).numpy() - del img - del semantic_bitmasks - del semantic_class_names - - def draw_picture(self, image_name, semantc_mask, id2label, output_path, suffix): - img = mmcv.imread(image_name) - sematic_class_in_img = torch.unique(semantc_mask) - semantic_bitmasks, semantic_class_names = [], [] - - # semantic prediction - for i in range(len(sematic_class_in_img)): - class_name = id2label['id2label'][str(sematic_class_in_img[i].item())] - class_mask = semantc_mask == sematic_class_in_img[i] - class_mask = class_mask.cpu().numpy().astype(np.uint8) - semantic_class_names.append(class_name) - semantic_bitmasks.append(class_mask) - - #print(os.environ["OUTPUT_URL"]) - length = len(image_name) - for i in range(length): - if image_name[length-i-1] == '_': - break - filename = image_name[length-i:] - imshow_det_bboxes(img, - bboxes=None, - labels=np.arange(len(sematic_class_in_img)), - segms=np.stack(semantic_bitmasks), - class_names=semantic_class_names, - font_size=25, - show=False, - out_file=os.path.join(output_path, filename + suffix)) - print('[Save] save rfnet prediction: ', os.path.join(output_path, filename + suffix)) - #semantc_mask = semantc_mask.unsqueeze(0).numpy() - del img - del semantic_bitmasks - del semantic_class_names - - def confidence(self, input_output): - output = torch.softmax(input_output, dim=0) - highth = len(output[0]) - width = len(output[0][0]) - sum_1 = 0.0 - sum_2 = 0.0 - values, _ = torch.topk(output, k=2, dim=0) - sum_1 = torch.sum(values[0]) - value_2 = torch.sub(values[0],values[1]) - sum_2 = torch.sum(value_2) - sum_1 = sum_1/(highth*width) - sum_2 = sum_2/(highth*width) - count = (values[0] > 0.9).sum().item() - sum_3 = count/(highth*width) - return sum_3 - - def sam_predict_ssa(self, image_name, pred): - with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: - cache = pickle.load(file) - img = mmcv.imread(image_name) - if image_name in cache.keys(): - mask = cache[image_name] - print("load cache") - else: - sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') - mask_branch_model = SamAutomaticMaskGenerator( - model=sam, - #points_per_side=64, - # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side - #pred_iou_thresh=0.86, - #stability_score_thresh=0.92, - #crop_n_layers=1, - #crop_n_points_downscale_factor=2, - #min_mask_region_area=100, # Requires open-cv to run post-processing - output_mode='coco_rle', - ) - print('[Model loaded] Mask branch (SAM) is loaded.') - mask = mask_branch_model.generate(img) - cache[image_name] = mask - with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: - pickle.dump(cache, file) - print("save cache") - - anns = {'annotations': mask} - #print(len(anns['annotations']), len(anns['annotations'][0])) - #print(pred.shape) - #print(pred[0]) - class_names = [] - semantc_mask = pred.clone() - id2label = CONFIG_CITYSCAPES_ID2LABEL - anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) - for ann in anns['annotations']: - valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() - #print(valid_mask) - propose_classes_ids = pred[valid_mask] - num_class_proposals = len(torch.unique(propose_classes_ids)) - if num_class_proposals == 1: - semantc_mask[valid_mask] = propose_classes_ids[0] - ann['class_name'] = id2label['id2label'][str(propose_classes_ids[0].item())] - ann['class_proposals'] = id2label['id2label'][str(propose_classes_ids[0].item())] - class_names.append(ann['class_name']) - # bitmasks.append(maskUtils.decode(ann['segmentation'])) - continue - top_1_propose_class_ids = torch.bincount(propose_classes_ids.flatten()).topk(1).indices - top_1_propose_class_names = [id2label['id2label'][str(class_id.item())] for class_id in top_1_propose_class_ids] - #print("top_1_propose_class_ids: ", top_1_propose_class_ids) - semantc_mask[valid_mask] = top_1_propose_class_ids - ann['class_name'] = top_1_propose_class_names[0] - ann['class_proposals'] = top_1_propose_class_names[0] - class_names.append(ann['class_name']) - # bitmasks.append(maskUtils.decode(ann['segmentation'])) - - del valid_mask - del propose_classes_ids - del num_class_proposals - del top_1_propose_class_ids - del top_1_propose_class_names - - #print(semantc_mask.shape) - #print(semantc_mask) - - del img - del anns - #del semantc_mask - # del bitmasks - del class_names - return semantc_mask, mask - - def sam_predict(self, image_name, pred): - with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: - cache = pickle.load(file) - img = mmcv.imread(image_name) - if image_name in cache.keys(): - mask = cache[image_name] - print("load cache") - else: - sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') - mask_branch_model = SamAutomaticMaskGenerator( - model=sam, - #points_per_side=64, - # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side - #pred_iou_thresh=0.86, - #stability_score_thresh=0.92, - #crop_n_layers=1, - #crop_n_points_downscale_factor=2, - #min_mask_region_area=100, # Requires open-cv to run post-processing - output_mode='coco_rle', - ) - print('[Model loaded] Mask branch (SAM) is loaded.') - mask = mask_branch_model.generate(img) - cache[image_name] = mask - with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: - pickle.dump(cache, file) - print("save cache") - - anns = {'annotations': mask} - #print(len(anns['annotations']), len(anns['annotations'][0])) - #print(pred.shape) - #print(pred[0]) - class_names = [] - pred_2 = np.argmax(pred, axis=0) - semantc_mask = pred_2.clone() - id2label = CONFIG_CITYSCAPES_ID2LABEL - anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) - for ann in anns['annotations']: - valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() - #print(valid_mask) - length = len(pred) - all_scores = [0 for i in range(length)] - for i in range(length): - propose_classes_ids = pred[i][valid_mask] - #print(propose_classes_ids.shape) - all_scores[i] = torch.sum(propose_classes_ids) - #print(all_scores[i]) - top_1_propose_class_ids = np.argmax(all_scores) - #print(top_1_propose_class_ids) - top_1_propose_class_names = id2label['id2label'][str(top_1_propose_class_ids)] - - semantc_mask[valid_mask] = top_1_propose_class_ids - ann['class_name'] = top_1_propose_class_names - ann['class_proposals'] = top_1_propose_class_names - class_names.append(ann['class_name']) - # bitmasks.append(maskUtils.decode(ann['segmentation'])) - - del valid_mask - del propose_classes_ids - del top_1_propose_class_ids - del top_1_propose_class_names - - #print(semantc_mask.shape) - #print(semantc_mask) - - #self.draw_picture(img, image_name, pred_2, id2label, output_path, "_origin.png") - #self.draw_picture(img, image_name, semantc_mask, id2label, output_path, "_sam.png") - - del img - del anns - #del semantc_mask - # del bitmasks - del class_names - return semantc_mask, mask - - def validate(self): - #print("start validating 55") - self.model.eval() - self.evaluator.reset() - tbar = tqdm(self.test_loader, desc='\r') - predictions = [] - scores = [] - for i, (sample, image_name) in enumerate(tbar):#self.test_loader: - if self.args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - # spec = time.time() - image, target = sample['image'], sample['label'] - #print(self.args.cuda, self.args.gpu_ids) - if self.args.cuda: - image = image.cuda(self.args.gpu_ids) - if self.args.depth: - depth = depth.cuda(self.args.gpu_ids) - - with torch.no_grad(): - if self.args.depth: - output = self.model(image, depth) - else: - output = self.model(image) - - if self.args.cuda: - torch.cuda.synchronize() - - if len(output) == 1: - score = self.confidence(output[0]) - else: - score = self.confidence(output) - scores.append(score) - - pred = output.data.cpu().numpy() - # todo - pred = np.argmax(pred, axis=1) - predictions.append(pred) - - output_path = os.environ["OUTPUT_URL"] - id2label = CONFIG_CITYSCAPES_ID2LABEL - self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_origin.png") - - #print("start validating 120") - return predictions, scores - - def vit_validate(self): - #print("start validating 55") - self.model.eval() - self.evaluator.reset() - tbar = tqdm(self.test_loader, desc='\r') - predictions = [] - rank = 'cuda:0' - semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( - "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") - semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( - "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) - for i, (sample, image_name) in enumerate(tbar):#self.test_loader: - img = mmcv.imread(image_name[0]) - class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) - pred = class_ids.data.cpu().numpy() - predictions.append(pred) - - output_path = os.environ["OUTPUT_URL"] - id2label = CONFIG_CITYSCAPES_ID2LABEL - self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") - - #print("start validating 120") - return predictions - - def validate_cloud(self): - #print("start validating 55") - self.model.eval() - self.evaluator.reset() - tbar = tqdm(self.test_loader, desc='\r') - predictions = [] - scores = [] - for i, (sample, image_name) in enumerate(tbar):#self.test_loader: - if self.args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - # spec = time.time() - image, target = sample['image'], sample['label'] - #print(self.args.cuda, self.args.gpu_ids) - if self.args.cuda: - image = image.cuda(self.args.gpu_ids) - if self.args.depth: - depth = depth.cuda(self.args.gpu_ids) - - with torch.no_grad(): - if self.args.depth: - output = self.model(image, depth) - else: - output = self.model(image) - - if self.args.cuda: - torch.cuda.synchronize() - - if len(output) == 1: - score = self.confidence(output[0]) - else: - score = self.confidence(output) - scores.append(score) - - pred = output.data.cpu().numpy() - # todo - pred_sam, mask = self.sam_predict(image_name[0], torch.from_numpy(pred[0])) - if pred_sam.ndim < 3: - h, w = pred_sam.shape - pred_sam = pred_sam.reshape(1, h, w) - #print(pred_sam.shape) - - predictions.append(np.array(pred_sam)) - - output_path = os.environ["OUTPUT_URL"] - id2label = CONFIG_CITYSCAPES_ID2LABEL - self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_sam.png") - self.draw_mask(image_name[0], mask, output_path) - - #print("start validating 120") - return predictions, scores - - def vit_validate_cloud(self): - #print("start validating 55") - self.model.eval() - self.evaluator.reset() - tbar = tqdm(self.test_loader, desc='\r') - predictions = [] - rank = 'cuda:0' - semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( - "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") - semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( - "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) - for i, (sample, image_name) in enumerate(tbar):#self.test_loader: - img = mmcv.imread(image_name[0]) - class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) - pred = class_ids.data.cpu().numpy() - pred_sam, mask = self.sam_predict_ssa(image_name[0], torch.from_numpy(pred[0])) - if pred_sam.ndim < 3: - h, w = pred_sam.shape - pred_sam = pred_sam.reshape(1, h, w) - #print(pred_sam.shape) - predictions.append(np.array(pred_sam)) - - output_path = os.environ["OUTPUT_URL"] - id2label = CONFIG_CITYSCAPES_ID2LABEL - self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") - self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_vit_sam.png") - self.draw_mask(image_name[0], mask, output_path) - - #print("start validating 120") - return predictions - - def task_divide(self): - seen_task_samples, unseen_task_samples = [], [] - self.model.eval() - self.evaluator.reset() - tbar = tqdm(self.test_loader, desc='\r') - for i, (sample, image_name) in enumerate(tbar): - - if self.args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - image, target = sample['image'], sample['label'] - if self.args.cuda: - image = image.cuda(self.args.gpu_ids) - if self.args.depth: - depth = depth.cuda(self.args.gpu_ids) - start_time = time.time() - with torch.no_grad(): - if self.args.depth: - output_, output, _ = self.model(image, depth) - else: - output_, output, _ = self.model(image) - if self.args.cuda: - torch.cuda.synchronize() - if i != 0: - fwt = time.time() - start_time - self.time_train.append(fwt) - print("Forward time per img (bath size=%d): %.3f (Mean: %.3f)" % ( - self.args.val_batch_size, fwt / self.args.val_batch_size, - sum(self.time_train) / len(self.time_train) / self.args.val_batch_size)) - time.sleep(0.1) # to avoid overheating the GPU too much - - # pred colorize - pre_colors = Colorize()(torch.max(output, 1)[1].detach().cpu().byte()) - pre_labels = torch.max(output, 1)[1].detach().cpu().byte() - for i in range(pre_colors.shape[0]): - task_sample = dict() - task_sample.update(image=sample["image"][i]) - task_sample.update(label=sample["label"][i]) - if self.args.depth: - task_sample.update(depth=sample["depth"][i]) - - if torch.max(pre_labels) == output.shape[1] - 1: - unseen_task_samples.append((task_sample, image_name[i])) - else: - seen_task_samples.append((task_sample, image_name[i])) - - return seen_task_samples, unseen_task_samples - -def image_merge(image, label, save_name): - image = ToPILImage()(image.detach().cpu().byte()) - # width, height = image.size - left = 140 - top = 30 - right = 2030 - bottom = 900 - # crop - image = image.crop((left, top, right, bottom)) - # resize - image = image.resize(label.size, Image.BILINEAR) - - image = image.convert('RGBA') - label = label.convert('RGBA') - image = Image.blend(image, label, 0.6) - image.save(save_name) - -def load_my_state_dict(model, state_dict): # custom function to load model when not all dict elements - own_state = model.state_dict() - for name, param in state_dict.items(): - if name not in own_state: - print('{} not in model_state'.format(name)) - continue - else: - own_state[name].copy_(param) - - return model diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/replicate.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/replicate.py deleted file mode 100755 index 3734266e..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/replicate.py +++ /dev/null @@ -1,88 +0,0 @@ -# -*- coding: utf-8 -*- -# File : replicate.py -# Author : Jiayuan Mao -# Email : maojiayuan@gmail.com -# Date : 27/01/2018 -# -# This file is part of Synchronized-BatchNorm-PyTorch. -# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch -# Distributed under MIT License. - -import functools - -from torch.nn.parallel.data_parallel import DataParallel - -__all__ = [ - 'CallbackContext', - 'execute_replication_callbacks', - 'DataParallelWithCallback', - 'patch_replication_callback' -] - - -class CallbackContext(object): - pass - - -def execute_replication_callbacks(modules): - """ - Execute an replication callback `__data_parallel_replicate__` on each module created by original replication. - The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` - Note that, as all modules are isomorphism, we assign each sub-module with a context - (shared among multiple copies of this module on different devices). - Through this context, different copies can share some information. - We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback - of any slave copies. - """ - master_copy = modules[0] - nr_modules = len(list(master_copy.modules())) - ctxs = [CallbackContext() for _ in range(nr_modules)] - - for i, module in enumerate(modules): - for j, m in enumerate(module.modules()): - if hasattr(m, '__data_parallel_replicate__'): - m.__data_parallel_replicate__(ctxs[j], i) - - -class DataParallelWithCallback(DataParallel): - """ - Data Parallel with a replication callback. - An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by - original `replicate` function. - The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` - Examples: - > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) - > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) - # sync_bn.__data_parallel_replicate__ will be invoked. - """ - - def replicate(self, module, device_ids): - modules = super(DataParallelWithCallback, self).replicate(module, device_ids) - execute_replication_callbacks(modules) - return modules - - -def patch_replication_callback(data_parallel): - """ - Monkey-patch an existing `DataParallel` object. Add the replication callback. - Useful when you have customized `DataParallel` implementation. - Examples: - > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) - > sync_bn = DataParallel(sync_bn, device_ids=[0, 1]) - > patch_replication_callback(sync_bn) - # this is equivalent to - > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) - > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) - """ - - assert isinstance(data_parallel, DataParallel) - - old_replicate = data_parallel.replicate - - @functools.wraps(old_replicate) - def new_replicate(module, device_ids): - modules = old_replicate(module, device_ids) - execute_replication_callbacks(modules) - return modules - - data_parallel.replicate = new_replicate \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/__init__.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention.py deleted file mode 100755 index 63d81991..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention.py +++ /dev/null @@ -1,391 +0,0 @@ -import torch -import torch.nn as nn -import torch.utils.model_zoo as model_zoo -from itertools import chain -import torch.utils.checkpoint as cp - -from ..util import _Upsample, SpatialPyramidPooling - -__all__ = ['ResNet', 'resnet18'] - -model_urls = { - 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', -} - - -def conv3x3(in_planes, out_planes, stride=1): - """3x3 convolution with padding""" - return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, - padding=1, bias=False) - -def conv1x1(in_planes, out_planes, stride=1): - """3x3 convolution with padding""" - return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, - padding=0, bias=False) - -def _bn_function_factory(conv, norm, relu=None): - """return a conv-bn-relu function""" - def bn_function(x): - x = conv(x) - if norm is not None: - x = norm(x) - if relu is not None: - x = relu(x) - return x - - return bn_function - - -def do_efficient_fwd(block, x, efficient): - if efficient and x.requires_grad: - return cp.checkpoint(block, x) - else: - return block(x) - - -def channel_shuffle(x, groups): - batchsize, num_channels, height, width = x.data.size() - channels_per_group = num_channels // groups - - # reshape - x = x.view(batchsize, groups, - channels_per_group, height, width) - - x = torch.transpose(x, 1, 2).contiguous() - - # flatten - x = x.view(batchsize, -1, height, width) - - return x - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, inplanes, planes, stride=1, downsample=None, efficient=False, use_bn=True): - super(BasicBlock, self).__init__() - self.use_bn = use_bn - self.conv1 = conv3x3(inplanes, planes, stride) - self.bn1 = nn.BatchNorm2d(planes) if self.use_bn else None - self.relu = nn.ReLU(inplace=True) - self.conv2 = conv3x3(planes, planes) - self.bn2 = nn.BatchNorm2d(planes) if self.use_bn else None - self.downsample = downsample - self.stride = stride - self.efficient = efficient - - def forward(self, x): - residual = x - - bn_1 = _bn_function_factory(self.conv1, self.bn1, self.relu) - bn_2 = _bn_function_factory(self.conv2, self.bn2) - - out = do_efficient_fwd(bn_1, x, self.efficient) - out = do_efficient_fwd(bn_2, out, self.efficient) - - if self.downsample is not None: - residual = self.downsample(x) - - out = out + residual - relu = self.relu(out) - - return relu, out - - -class Bottleneck(nn.Module): - expansion = 4 - - def __init__(self, inplanes, planes, stride=1, downsample=None, efficient=True, use_bn=True): - super(Bottleneck, self).__init__() - self.use_bn = use_bn - self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) if self.use_bn else None - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, - padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) if self.use_bn else None - self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) if self.use_bn else None - self.relu = nn.ReLU(inplace=True) - self.downsample = downsample - self.stride = stride - self.efficient = efficient - - def forward(self, x): - residual = x - - bn_1 = _bn_function_factory(self.conv1, self.bn1, self.relu) - bn_2 = _bn_function_factory(self.conv2, self.bn2, self.relu) - bn_3 = _bn_function_factory(self.conv3, self.bn3, self.relu) - - out = do_efficient_fwd(bn_1, x, self.efficient) - out = do_efficient_fwd(bn_2, out, self.efficient) - out = do_efficient_fwd(bn_3, out, self.efficient) - - if self.downsample is not None: - residual = self.downsample(x) - - out = out + residual - relu = self.relu(out) - - return relu, out - - -class ResNet(nn.Module): - def __init__(self, block, layers, *, num_features=128, k_up=3, efficient=True, use_bn=True, - spp_grids=(8, 4, 2, 1), spp_square_grid=False, **kwargs): - super(ResNet, self).__init__() - self.inplanes = 64 - self.efficient = efficient - self.use_bn = use_bn - - # rgb branch - self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) - self.bn1 = nn.BatchNorm2d(64) if self.use_bn else lambda x: x - self.relu = nn.ReLU(inplace=True) - self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - - # depth branch - self.conv1_d = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,bias=False) - self.bn1_d = nn.BatchNorm2d(64) if self.use_bn else lambda x: x - self.relu_d = nn.ReLU(inplace=True) - self.maxpool_d = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - - upsamples = [] - # 修改 _make_layer_rgb _make_layer - self.layer1 = self._make_layer_rgb(block, 64, 64, layers[0]) - self.layer1_d = self._make_layer_d(block, 64, 64, layers[0]) - self.attention_1 = self.attention(64) - self.attention_1_d = self.attention(64) - upsamples += [_Upsample(num_features, self.inplanes, num_features, use_bn=self.use_bn, k=k_up)] # num_maps_in, skip_maps_in, num_maps_out, k: kernel size of blend conv - - self.layer2 = self._make_layer_rgb(block, 64, 128, layers[1], stride=2) - self.layer2_d = self._make_layer_d(block, 64, 128, layers[1], stride=2) - self.attention_2 = self.attention(128) - self.attention_2_d = self.attention(128) - upsamples += [_Upsample(num_features, self.inplanes, num_features, use_bn=self.use_bn, k=k_up)] - - self.layer3 = self._make_layer_rgb(block, 128, 256, layers[2], stride=2) - self.layer3_d = self._make_layer_d(block, 128, 256, layers[2], stride=2) - self.attention_3 = self.attention(256) - self.attention_3_d = self.attention(256) - upsamples += [_Upsample(num_features, self.inplanes, num_features, use_bn=self.use_bn, k=k_up)] - - self.layer4 = self._make_layer_rgb(block, 256, 512, layers[3], stride=2) - self.layer4_d = self._make_layer_d(block, 256, 512, layers[3], stride=2) - self.attention_4 = self.attention(512) - self.attention_4_d = self.attention(512) - - self.fine_tune = [self.conv1, self.maxpool, self.layer1, self.layer2, self.layer3, self.layer4, - self.conv1_d, self.maxpool_d, self.layer1_d, self.layer2_d, self.layer3_d, self.layer4_d] - if self.use_bn: - self.fine_tune += [self.bn1, self.bn1_d, self.attention_1, self.attention_1_d, self.attention_2, self.attention_2_d, - self.attention_3, self.attention_3_d, self.attention_4, self.attention_4_d] - - num_levels = 3 - self.spp_size = num_features - bt_size = self.spp_size - - level_size = self.spp_size // num_levels - - self.spp = SpatialPyramidPooling(self.inplanes, num_levels, bt_size=bt_size, level_size=level_size, - out_size=self.spp_size, grids=spp_grids, square_grid=spp_square_grid, - bn_momentum=0.01 / 2, use_bn=self.use_bn) - self.upsample = nn.ModuleList(list(reversed(upsamples))) - - self.random_init = []#[ self.spp, self.upsample] - self.fine_tune += [self.spp, self.upsample] - - self.num_features = num_features - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') - elif isinstance(m, nn.BatchNorm2d): - nn.init.constant_(m.weight, 1) - nn.init.constant_(m.bias, 0) - - def _make_layer_rgb(self, block, inplanes, planes, blocks, stride=1): - downsample = None - if stride != 1 or inplanes != planes * block.expansion: - layers = [nn.Conv2d(inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False)] - if self.use_bn: - layers += [nn.BatchNorm2d(planes * block.expansion)] - downsample = nn.Sequential(*layers) - layers = [block(inplanes, planes, stride, downsample, efficient=self.efficient, use_bn=self.use_bn)] - inplanes = planes * block.expansion - for i in range(1, blocks): - layers += [block(inplanes, planes, efficient=self.efficient, use_bn=self.use_bn)] - - return nn.Sequential(*layers) - - def _make_layer_d(self, block, inplanes, planes, blocks, stride=1): - downsample = None - if stride != 1 or inplanes != planes * block.expansion: - layers = [nn.Conv2d(inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False)] - if self.use_bn: - layers += [nn.BatchNorm2d(planes * block.expansion)] - downsample = nn.Sequential(*layers) - layers = [block(inplanes, planes, stride, downsample, efficient=self.efficient, use_bn=self.use_bn)] - inplanes = planes * block.expansion - self.inplanes = inplanes - for i in range(1, blocks): - layers += [block(inplanes, planes, efficient=self.efficient, use_bn=self.use_bn)] - - return nn.Sequential(*layers) - - def channel_attention(self, rgb_skip, depth_skip, attention): - assert rgb_skip.shape == depth_skip.shape, 'rgb skip shape:{} != depth skip shape:{}'.format(rgb_skip.shape, depth_skip.shape) - # single_attenton - rgb_attention = attention(rgb_skip) - depth_attention = attention(depth_skip) - rgb_after_attention = torch.mul(rgb_skip, rgb_attention) - depth_after_attention = torch.mul(depth_skip, depth_attention) - skip_after_attention = rgb_after_attention + depth_after_attention - return skip_after_attention - - def attention(self, num_channels): - pool_attention = nn.AdaptiveAvgPool2d(1) - conv_attention = nn.Conv2d(num_channels, num_channels, kernel_size=1) - activate = nn.Sigmoid() - - return nn.Sequential(pool_attention, conv_attention, activate) - - - def random_init_params(self): - return chain(*[f.parameters() for f in self.random_init]) - - def fine_tune_params(self): - return chain(*[f.parameters() for f in self.fine_tune]) - - def forward_resblock(self, x, layers): - skip = None - for l in layers: - x = l(x) - if isinstance(x, tuple): - x, skip = x - return x, skip - - def forward_down(self, rgb): - x = self.conv1(rgb) - x = self.bn1(x) - x = self.relu(x) - x = self.maxpool(x) - - features = [] - x, skip = self.forward_resblock(x, self.layer1) - features += [skip] - x, skip = self.forward_resblock(x, self.layer2) - features += [skip] - x, skip = self.forward_resblock(x, self.layer3) - features += [skip] - x, skip = self.forward_resblock(x.detach(), self.layer4) - features += [self.spp.forward(skip)] - return features - - def forward_down_fusion(self, rgb, depth): - x = self.conv1(rgb) - x = self.bn1(x) - x = self.relu(x) - x = self.maxpool(x) - - depth = depth.unsqueeze(1) - y = self.conv1_d(depth) - y = self.bn1_d(y) - y = self.relu_d(y) - y = self.maxpool_d(y) - - features = [] - # block 1 - x, skip_rgb = self.forward_resblock(x.detach(), self.layer1) - y, skip_depth = self.forward_resblock(y.detach(), self.layer1_d) - x_attention = self.attention_1(x) - y_attention = self.attention_1_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [skip_rgb.detach()] - # block 2 - x, skip_rgb = self.forward_resblock(x.detach(), self.layer2) - y, skip_depth = self.forward_resblock(y.detach(), self.layer2_d) - x_attention = self.attention_2(x) - y_attention = self.attention_2_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [skip_rgb.detach()] - # block 3 - x, skip_rgb = self.forward_resblock(x.detach(), self.layer3) - y, skip_depth = self.forward_resblock(y.detach(), self.layer3_d) - x_attention = self.attention_3(x) - y_attention = self.attention_3_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [skip_rgb.detach()] - # block 4 - x, skip_rgb = self.forward_resblock(x.detach(), self.layer4) - y, skip_depth = self.forward_resblock(y.detach(), self.layer4_d) - x_attention = self.attention_4(x) - y_attention = self.attention_4_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [self.spp.forward(x)] - return features - - - def forward_up(self, features): - features = features[::-1] - - x = features[0] - - upsamples = [] - i = 0 - for skip, up in zip(features[1:], self.upsample): - i += 1 - #print(len(self.upsample)) - if i < len(self.upsample): - x = up(x, skip) - else: - x = up(x, skip) - upsamples += [x] - return x, {'features': features, 'upsamples': upsamples} - - def forward(self, rgb, depth = None): - if depth is None: - return self.forward_up(self.forward_down(rgb)) - else: - return self.forward_up(self.forward_down_fusion(rgb, depth)) - - def _load_resnet_pretrained(self, url): - pretrain_dict = model_zoo.load_url(model_urls[url]) - model_dict = {} - state_dict = self.state_dict() - for k, v in pretrain_dict.items(): - # print('%%%%% ', k) - if k in state_dict: - if k.startswith('conv1'): - model_dict[k] = v - # print('##### ', k) - model_dict[k.replace('conv1', 'conv1_d')] = torch.mean(v, 1).data. \ - view_as(state_dict[k.replace('conv1', 'conv1_d')]) - - elif k.startswith('bn1'): - model_dict[k] = v - model_dict[k.replace('bn1', 'bn1_d')] = v - elif k.startswith('layer'): - model_dict[k] = v - model_dict[k[:6]+'_d'+k[6:]] = v - state_dict.update(model_dict) - self.load_state_dict(state_dict) - - -def resnet18(pretrained=True, **kwargs): - """Constructs a ResNet-18 model. - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet18']), strict=False) - print('pretrained dict loaded sucessfully') - return model \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention_unseen.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention_unseen.py deleted file mode 100755 index 9668734e..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/resnet/resnet_single_scale_single_attention_unseen.py +++ /dev/null @@ -1,396 +0,0 @@ -import torch -import torch.nn as nn -import torch.utils.model_zoo as model_zoo -from itertools import chain -import torch.utils.checkpoint as cp -import cv2 -import numpy as np - -from ..util import _Upsample, SpatialPyramidPooling - -__all__ = ['ResNet', 'resnet18'] - -model_urls = { - 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', -} - - -def conv3x3(in_planes, out_planes, stride=1): - """3x3 convolution with padding""" - return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, - padding=1, bias=False) - -def conv1x1(in_planes, out_planes, stride=1): - """3x3 convolution with padding""" - return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, - padding=0, bias=False) - -def _bn_function_factory(conv, norm, relu=None): - """return a conv-bn-relu function""" - def bn_function(x): - x = conv(x) - if norm is not None: - x = norm(x) - if relu is not None: - x = relu(x) - return x - - return bn_function - - -def do_efficient_fwd(block, x, efficient): - if efficient and x.requires_grad: - return cp.checkpoint(block, x) - else: - return block(x) - - -def channel_shuffle(x, groups): - batchsize, num_channels, height, width = x.data.size() - channels_per_group = num_channels // groups - - # reshape - x = x.view(batchsize, groups, - channels_per_group, height, width) - - x = torch.transpose(x, 1, 2).contiguous() - - # flatten - x = x.view(batchsize, -1, height, width) - - return x - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, inplanes, planes, stride=1, downsample=None, efficient=False, use_bn=True): - super(BasicBlock, self).__init__() - self.use_bn = use_bn - self.conv1 = conv3x3(inplanes, planes, stride) - self.bn1 = nn.BatchNorm2d(planes) if self.use_bn else None - self.relu = nn.ReLU(inplace=True) - self.conv2 = conv3x3(planes, planes) - self.bn2 = nn.BatchNorm2d(planes) if self.use_bn else None - self.downsample = downsample - self.stride = stride - self.efficient = efficient - - def forward(self, x): - residual = x - - bn_1 = _bn_function_factory(self.conv1, self.bn1, self.relu) - bn_2 = _bn_function_factory(self.conv2, self.bn2) - - out = do_efficient_fwd(bn_1, x, self.efficient) - out = do_efficient_fwd(bn_2, out, self.efficient) - - if self.downsample is not None: - residual = self.downsample(x) - - out = out + residual - relu = self.relu(out) - - return relu, out - - -class Bottleneck(nn.Module): - expansion = 4 - - def __init__(self, inplanes, planes, stride=1, downsample=None, efficient=True, use_bn=True): - super(Bottleneck, self).__init__() - self.use_bn = use_bn - self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) if self.use_bn else None - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, - padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) if self.use_bn else None - self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) - self.bn3 = nn.BatchNorm2d(planes * self.expansion) if self.use_bn else None - self.relu = nn.ReLU(inplace=True) - self.downsample = downsample - self.stride = stride - self.efficient = efficient - - def forward(self, x): - residual = x - - bn_1 = _bn_function_factory(self.conv1, self.bn1, self.relu) - bn_2 = _bn_function_factory(self.conv2, self.bn2, self.relu) - bn_3 = _bn_function_factory(self.conv3, self.bn3, self.relu) - - out = do_efficient_fwd(bn_1, x, self.efficient) - out = do_efficient_fwd(bn_2, out, self.efficient) - out = do_efficient_fwd(bn_3, out, self.efficient) - - if self.downsample is not None: - residual = self.downsample(x) - - out = out + residual - relu = self.relu(out) - - return relu, out - - -class ResNet(nn.Module): - def __init__(self, block, layers, *, num_features=128, k_up=3, efficient=True, use_bn=True, - spp_grids=(8, 4, 2, 1), spp_square_grid=False, **kwargs): - super(ResNet, self).__init__() - self.inplanes = 64 - self.efficient = efficient - self.use_bn = use_bn - - # rgb branch - self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) - self.bn1 = nn.BatchNorm2d(64) if self.use_bn else lambda x: x - self.relu = nn.ReLU(inplace=True) - self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - - # depth branch - self.conv1_d = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3,bias=False) - self.bn1_d = nn.BatchNorm2d(64) if self.use_bn else lambda x: x - self.relu_d = nn.ReLU(inplace=True) - self.maxpool_d = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - - upsamples = [] - # 修改 _make_layer_rgb _make_layer - self.layer1 = self._make_layer_rgb(block, 64, 64, layers[0]) - self.layer1_d = self._make_layer_d(block, 64, 64, layers[0]) - self.attention_1 = self.attention(64) - self.attention_1_d = self.attention(64) - upsamples += [_Upsample(num_features, self.inplanes, num_features, use_bn=self.use_bn, k=k_up)] # num_maps_in, skip_maps_in, num_maps_out, k: kernel size of blend conv - - self.layer2 = self._make_layer_rgb(block, 64, 128, layers[1], stride=2) - self.layer2_d = self._make_layer_d(block, 64, 128, layers[1], stride=2) - self.attention_2 = self.attention(128) - self.attention_2_d = self.attention(128) - upsamples += [_Upsample(num_features, self.inplanes, num_features, use_bn=self.use_bn, k=k_up)] - - self.layer3 = self._make_layer_rgb(block, 128, 256, layers[2], stride=2) - self.layer3_d = self._make_layer_d(block, 128, 256, layers[2], stride=2) - self.attention_3 = self.attention(256) - self.attention_3_d = self.attention(256) - upsamples += [_Upsample(num_features, self.inplanes, num_features, use_bn=self.use_bn, k=k_up)] - - self.layer4 = self._make_layer_rgb(block, 256, 512, layers[3], stride=2) - self.layer4_d = self._make_layer_d(block, 256, 512, layers[3], stride=2) - self.attention_4 = self.attention(512) - self.attention_4_d = self.attention(512) - - self.fine_tune = [self.conv1, self.maxpool, self.layer1, self.layer2, self.layer3, self.layer4, - self.conv1_d, self.maxpool_d, self.layer1_d, self.layer2_d, self.layer3_d, self.layer4_d] - if self.use_bn: - self.fine_tune += [self.bn1, self.bn1_d, self.attention_1, self.attention_1_d, self.attention_2, self.attention_2_d, - self.attention_3, self.attention_3_d, self.attention_4, self.attention_4_d] - - num_levels = 3 - self.spp_size = num_features - bt_size = self.spp_size - - level_size = self.spp_size // num_levels - - self.spp = SpatialPyramidPooling(self.inplanes, num_levels, bt_size=bt_size, level_size=level_size, - out_size=self.spp_size, grids=spp_grids, square_grid=spp_square_grid, - bn_momentum=0.01 / 2, use_bn=self.use_bn) - self.upsample = nn.ModuleList(list(reversed(upsamples))) - - self.random_init = [ self.spp, self.upsample] - - self.num_features = num_features - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') - elif isinstance(m, nn.BatchNorm2d): - nn.init.constant_(m.weight, 1) - nn.init.constant_(m.bias, 0) - - def output_num(self): - return self.__in_features - - def _make_layer_rgb(self, block, inplanes, planes, blocks, stride=1): - downsample = None - if stride != 1 or inplanes != planes * block.expansion: - layers = [nn.Conv2d(inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False)] - if self.use_bn: - layers += [nn.BatchNorm2d(planes * block.expansion)] - downsample = nn.Sequential(*layers) - layers = [block(inplanes, planes, stride, downsample, efficient=self.efficient, use_bn=self.use_bn)] - inplanes = planes * block.expansion - for i in range(1, blocks): - layers += [block(inplanes, planes, efficient=self.efficient, use_bn=self.use_bn)] - - return nn.Sequential(*layers) - - def _make_layer_d(self, block, inplanes, planes, blocks, stride=1): - downsample = None - if stride != 1 or inplanes != planes * block.expansion: - layers = [nn.Conv2d(inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False)] - if self.use_bn: - layers += [nn.BatchNorm2d(planes * block.expansion)] - downsample = nn.Sequential(*layers) - layers = [block(inplanes, planes, stride, downsample, efficient=self.efficient, use_bn=self.use_bn)] - inplanes = planes * block.expansion - self.inplanes = inplanes - for i in range(1, blocks): - layers += [block(inplanes, planes, efficient=self.efficient, use_bn=self.use_bn)] - - return nn.Sequential(*layers) - - def channel_attention(self, rgb_skip, depth_skip, attention): - assert rgb_skip.shape == depth_skip.shape, 'rgb skip shape:{} != depth skip shape:{}'.format(rgb_skip.shape, depth_skip.shape) - # single_attenton - rgb_attention = attention(rgb_skip) - depth_attention = attention(depth_skip) - rgb_after_attention = torch.mul(rgb_skip, rgb_attention) - depth_after_attention = torch.mul(depth_skip, depth_attention) - skip_after_attention = rgb_after_attention + depth_after_attention - return skip_after_attention - - def attention(self, num_channels): - pool_attention = nn.AdaptiveAvgPool2d(1) - conv_attention = nn.Conv2d(num_channels, num_channels, kernel_size=1) - activate = nn.Sigmoid() - - return nn.Sequential(pool_attention, conv_attention, activate) - - - def random_init_params(self): - return chain(*[f.parameters() for f in self.random_init]) - - def fine_tune_params(self): - return chain(*[f.parameters() for f in self.fine_tune]) - - def forward_resblock(self, x, layers): - skip = None - for l in layers: - x = l(x) - if isinstance(x, tuple): - x, skip = x - return x, skip - - def forward_down(self, rgb): - x = self.conv1(rgb) - x = self.bn1(x) - x = self.relu(x) - x = self.maxpool(x) - - features = [] - x, skip = self.forward_resblock(x, self.layer1) - features += [skip] - x, skip = self.forward_resblock(x, self.layer2) - features += [skip] - x, skip = self.forward_resblock(x, self.layer3) - features += [skip] - x, skip = self.forward_resblock(x, self.layer4) - features += [self.spp.forward(skip)] - features_da = self.spp.forward(skip) - return features, features_da - - def forward_down_fusion(self, rgb, depth): - x = self.conv1(rgb) - x = self.bn1(x) - x = self.relu(x) - x = self.maxpool(x) - - depth = depth.unsqueeze(1) - y = self.conv1_d(depth) - y = self.bn1_d(y) - y = self.relu_d(y) - y = self.maxpool_d(y) - - features = [] - # block 1 - x, skip_rgb = self.forward_resblock(x, self.layer1) - y, skip_depth = self.forward_resblock(y, self.layer1_d) - x_attention = self.attention_1(x) - y_attention = self.attention_1_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [skip_rgb] - # block 2 - x, skip_rgb = self.forward_resblock(x, self.layer2) - y, skip_depth = self.forward_resblock(y, self.layer2_d) - x_attention = self.attention_2(x) - y_attention = self.attention_2_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [skip_rgb] - # block 3 - x, skip_rgb = self.forward_resblock(x, self.layer3) - y, skip_depth = self.forward_resblock(y, self.layer3_d) - x_attention = self.attention_3(x) - y_attention = self.attention_3_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [skip_rgb] - # block 4 - x, skip_rgb = self.forward_resblock(x, self.layer4) - y, skip_depth = self.forward_resblock(y, self.layer4_d) - x_attention = self.attention_4(x) - y_attention = self.attention_4_d(y) - x = torch.mul(x, x_attention) - y = torch.mul(y, y_attention) - x = x + y - features += [self.spp.forward(x)] - features_da = self.spp.forward(x) - return features, features_da - - - def forward_up(self, features): - features = features[::-1] - - x = features[0] - - upsamples = [] - for skip, up in zip(features[1:], self.upsample): - x = up(x, skip) - upsamples += [x] - return x, {'features': features, 'upsamples': upsamples} - - def forward(self, rgb, depth = None): - if depth is None: - down_features, da_features = self.forward_down(rgb) - x, additional = self.forward_up(down_features) - return x, additional, da_features#self.forward_up(self.forward_down(rgb)), self.forward_down(rgb) - else: - down_features, da_features = self.forward_down_fusion(rgb, depth) - x, additional = self.forward_up(down_features) - #print(down_features.shape) - return x, additional, da_features#self.forward_up(self.forward_down_fusion(rgb, depth)), self.forward_down_fusion(rgb, depth) - - def _load_resnet_pretrained(self, url): - pretrain_dict = model_zoo.load_url(model_urls[url]) - model_dict = {} - state_dict = self.state_dict() - for k, v in pretrain_dict.items(): - # print('%%%%% ', k) - if k in state_dict: - if k.startswith('conv1'): - model_dict[k] = v - # print('##### ', k) - model_dict[k.replace('conv1', 'conv1_d')] = torch.mean(v, 1).data. \ - view_as(state_dict[k.replace('conv1', 'conv1_d')]) - - elif k.startswith('bn1'): - model_dict[k] = v - model_dict[k.replace('bn1', 'bn1_d')] = v - elif k.startswith('layer'): - model_dict[k] = v - model_dict[k[:6]+'_d'+k[6:]] = v - state_dict.update(model_dict) - self.load_state_dict(state_dict) - - -def resnet18(pretrained=True, **kwargs): - """Constructs a ResNet-18 model. - Args: - pretrained (bool): If True, returns a model pre-trained on ImageNet - """ - model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) - if pretrained: - model.load_state_dict(model_zoo.load_url(model_urls['resnet18']), strict=False) - print('pretrained dict loaded sucessfully') - return model \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet.py deleted file mode 100755 index 87f02863..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet.py +++ /dev/null @@ -1,27 +0,0 @@ -import torch.nn as nn -from itertools import chain # 串联多个迭代对象 - -from .util import _BNReluConv, upsample - - -class RFNet(nn.Module): - def __init__(self, backbone, num_classes, use_bn=True): - super(RFNet, self).__init__() - self.backbone = backbone - self.num_classes = num_classes - print(self.backbone.num_features) - self.logits = _BNReluConv(self.backbone.num_features, self.num_classes, batch_norm=use_bn) - - def forward(self, rgb_inputs, depth_inputs = None): - x, additional = self.backbone(rgb_inputs, depth_inputs) - logits = self.logits.forward(x) - logits_upsample = upsample(logits, rgb_inputs.shape[2:]) - #print(logits_upsample.size) - return logits_upsample - - - def random_init_params(self): - return chain(*([self.logits.parameters(), self.backbone.random_init_params()])) - - def fine_tune_params(self): - return self.backbone.fine_tune_params() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet_for_unseen.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet_for_unseen.py deleted file mode 100755 index f61eb1ce..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/rfnet_for_unseen.py +++ /dev/null @@ -1,33 +0,0 @@ -import torch.nn as nn -from itertools import chain # 串联多个迭代对象 - -from .util import _BNReluConv, upsample - - -class RFNet(nn.Module): - def __init__(self, backbone, num_classes, use_bn=True): - super(RFNet, self).__init__() - self.backbone = backbone - self.num_classes = num_classes - #self.bottleneck = _BNReluConv(self.backbone.num_features, 128, k = 3, batch_norm=use_bn) - #self.logits = _BNReluConv(128, self.num_classes+1, k = 1, batch_norm=use_bn) - self.logits = _BNReluConv(self.backbone.num_features, self.num_classes, batch_norm=use_bn) - #self.logits_target = _BNReluConv(self.backbone.num_features, self.num_classes, batch_norm=use_bn) - self.logits_aux = _BNReluConv(self.backbone.num_features, self.num_classes, batch_norm=use_bn) - - def forward(self, rgb_inputs, depth_inputs = None): - x, additional, da_features = self.backbone(rgb_inputs, depth_inputs) - #print(additional['features'][0].shape) - #bottleneck = self.bottleneck(x) - logits = self.logits.forward(x) - logits_aux = self.logits_aux.forward(x) - #print(logits_aux.shape) - logits_upsample = upsample(logits, rgb_inputs.shape[2:]) - logits_aux_upsample = upsample(logits_aux, rgb_inputs.shape[2:]) - return logits_upsample, logits_aux_upsample, da_features - - def random_init_params(self): - return chain(*([self.logits.parameters(), self.logits_aux.parameters(), self.backbone.random_init_params()])) - - def fine_tune_params(self): - return self.backbone.fine_tune_params() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/util.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/util.py deleted file mode 100755 index 5c86e759..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/models/util.py +++ /dev/null @@ -1,99 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F - -upsample = lambda x, size: F.interpolate(x, size, mode='bilinear', align_corners=False) -batchnorm_momentum = 0.01 / 2 - - -def get_n_params(parameters): - pp = 0 - for p in parameters: - nn = 1 - for s in list(p.size()): - nn = nn * s - pp += nn - return pp - - -class _BNReluConv(nn.Sequential): - def __init__(self, num_maps_in, num_maps_out, k=3, batch_norm=True, bn_momentum=0.1, bias=False, dilation=1): - super(_BNReluConv, self).__init__() - if batch_norm: - self.add_module('norm', nn.BatchNorm2d(num_maps_in, momentum=bn_momentum)) - self.add_module('relu', nn.ReLU(inplace=batch_norm is True)) - padding = k // 2 # same conv - self.add_module('conv', nn.Conv2d(num_maps_in, num_maps_out, - kernel_size=k, padding=padding, bias=bias, dilation=dilation)) - - -class _Upsample(nn.Module): - def __init__(self, num_maps_in, skip_maps_in, num_maps_out, use_bn=True, k=3): - super(_Upsample, self).__init__() - print(f'Upsample layer: in = {num_maps_in}, skip = {skip_maps_in}, out = {num_maps_out}') - self.bottleneck = _BNReluConv(skip_maps_in, num_maps_in, k=1, batch_norm=use_bn) - self.blend_conv = _BNReluConv(num_maps_in, num_maps_out, k=k, batch_norm=use_bn) - - def forward(self, x, skip): - skip = self.bottleneck.forward(skip) - skip_size = skip.size()[2:4] - x = upsample(x, skip_size) - x = x + skip - x = self.blend_conv.forward(x) - return x - - -class SpatialPyramidPooling(nn.Module): - def __init__(self, num_maps_in, num_levels, bt_size=512, level_size=128, out_size=128, - grids=(6, 3, 2, 1), square_grid=False, bn_momentum=0.1, use_bn=True): - super(SpatialPyramidPooling, self).__init__() - self.grids = grids - self.square_grid = square_grid - self.spp = nn.Sequential() - self.spp.add_module('spp_bn', - _BNReluConv(num_maps_in, bt_size, k=1, bn_momentum=bn_momentum, batch_norm=use_bn)) - num_features = bt_size - final_size = num_features - for i in range(num_levels): - final_size += level_size - self.spp.add_module('spp' + str(i), - _BNReluConv(num_features, level_size, k=1, bn_momentum=bn_momentum, batch_norm=use_bn)) - self.spp.add_module('spp_fuse', - _BNReluConv(final_size, out_size, k=1, bn_momentum=bn_momentum, batch_norm=use_bn)) - - def forward(self, x): - levels = [] - target_size = x.size()[2:4] - - ar = target_size[1] / target_size[0] - - x = self.spp[0].forward(x) - levels.append(x) - num = len(self.spp) - 1 - - for i in range(1, num): - if not self.square_grid: - grid_size = (self.grids[i - 1], max(1, round(ar * self.grids[i - 1]))) - x_pooled = F.adaptive_avg_pool2d(x, grid_size) - else: - x_pooled = F.adaptive_avg_pool2d(x, self.grids[i - 1]) - level = self.spp[i].forward(x_pooled) - - level = upsample(level, target_size) - levels.append(level) - x = torch.cat(levels, 1) - x = self.spp[-1].forward(x) - return x - - -class _UpsampleBlend(nn.Module): - def __init__(self, num_features, use_bn=True): - super(_UpsampleBlend, self).__init__() - self.blend_conv = _BNReluConv(num_features, num_features, k=3, batch_norm=use_bn) - - def forward(self, x, skip): - skip_size = skip.size()[2:4] - x = upsample(x, skip_size) - x = x + skip - x = self.blend_conv.forward(x) - return x diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/mypath.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/mypath.py deleted file mode 100755 index 640544e7..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/mypath.py +++ /dev/null @@ -1,20 +0,0 @@ -class Path(object): - @staticmethod - def db_root_dir(dataset): - if dataset == 'cityscapes': - return '/home/robo/m0063/project/RFNet-master/Data/cityscapes/' # folder that contains leftImg8bit/ - elif dataset == 'citylostfound': - return '/home/robo/m0063/project/RFNet-master/Data/cityscapesandlostandfound/' # folder that mixes Cityscapes and Lost and Found - elif dataset == 'cityrand': - return '/home/robo/m0063/project/RFNet-master/Data/cityrand/' - elif dataset == 'target': - return '/home/robo/m0063/project/RFNet-master/Data/target/' - elif dataset == 'xrlab': - return '/home/robo/m0063/project/RFNet-master/Data/xrlab/' - elif dataset == 'e1': - return '/home/robo/m0063/project/RFNet-master/Data/e1/' - elif dataset == 'mapillary': - return '/home/robo/m0063/project/RFNet-master/Data/mapillary/' - else: - print('Dataset {} not available.'.format(dataset)) - raise NotImplementedError diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/predict.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/predict.py deleted file mode 100755 index ed56fffd..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/predict.py +++ /dev/null @@ -1,100 +0,0 @@ -import os -os.environ['BACKEND_TYPE'] = 'PYTORCH' -# set at yaml -# os.environ["PREDICT_RESULT_DIR"] = "./inference_results" -# os.environ["EDGE_OUTPUT_URL"] = "./edge_kb" -# os.environ["video_url"] = "./video/radio.mp4" -# os.environ["MODEL_URLS"] = "./cloud_next_kb/index.pkl" - - -import cv2 -cv2.setNumThreads(0) -cv2.ocl.setUseOpenCL(False) -import time -import torch -import numpy as np -from PIL import Image -import base64 -import tempfile -import warnings -from io import BytesIO - -from sedna.datasources import BaseDataSource -from sedna.core.lifelong_learning import LifelongLearning -from sedna.common.config import Context - -from dataloaders import custom_transforms as tr -from torchvision import transforms - -from accuracy import accuracy -from basemodel import preprocess, val_args, Model - -def preprocess(samples): - composed_transforms = transforms.Compose([ - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - data = BaseDataSource(data_type="test") - data.x = [(composed_transforms(samples), "")] - return data - -def init_ll_job(): - estimator = Model() - - task_allocation = { - "method": "TaskAllocationByOrigin", - "param": { - "origins": ["real", "sim"], - "default": "real" - } - } - unseen_task_allocation = { - "method": "UnseenTaskAllocationDefault" - } - - ll_job = LifelongLearning( - estimator, - task_definition=None, - task_relationship_discovery=None, - task_allocation=task_allocation, - task_remodeling=None, - inference_integrate=None, - task_update_decision=None, - unseen_task_allocation=unseen_task_allocation, - unseen_sample_recognition=None, - unseen_sample_re_recognition=None) - - return ll_job - -def predict(): - ll_job = init_ll_job() - - camera_address = Context.get_parameters('video_url') - # use video streams for testing - camera = cv2.VideoCapture(camera_address) - fps = 10 - nframe = 0 - while 1: - ret, input_yuv = camera.read() - if not ret: - time.sleep(5) - camera = cv2.VideoCapture(camera_address) - continue - - if nframe % fps: - nframe += 1 - continue - - img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_BGR2RGB) - nframe += 1 - if nframe % 1000 == 1: # logs every 1000 frames - warnings.warn(f"camera is open, current frame index is {nframe}") - - img_rgb = cv2.resize(np.array(img_rgb), (2048, 1024), interpolation=cv2.INTER_CUBIC) - img_rgb = Image.fromarray(img_rgb) - sample = {'image': img_rgb, "depth": img_rgb, "label": img_rgb} - data = preprocess(sample) - print("Inference results:", ll_job.inference(data=data)) - -if __name__ == '__main__': - predict() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/run_server.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/run_server.py deleted file mode 100755 index 0f0b2c88..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/run_server.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2021 The KubeEdge Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from io import BytesIO -from typing import Optional, Any - -import cv2 -cv2.setNumThreads(0) -cv2.ocl.setUseOpenCL(False) -import numpy as np -from PIL import Image -import uvicorn -import time -from pydantic import BaseModel -from fastapi import FastAPI, UploadFile, File -from fastapi.routing import APIRoute -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse -import sedna_predict -from sedna.common.utils import get_host_ip -from dataloaders.datasets.cityscapes import CityscapesSegmentation - - -class ImagePayload(BaseModel): - image: UploadFile = File(...) - depth: Optional[UploadFile] = None - - -class ResultModel(BaseModel): - type: int = 0 - box: Any = None - curr: str = None - future: str = None - img: str = None - - -class ResultResponse(BaseModel): - msg: str = "" - result: Optional[ResultModel] = None - code: int - - -class BaseServer: - # pylint: disable=too-many-instance-attributes,too-many-arguments - DEBUG = True - WAIT_TIME = 15 - - def __init__( - self, - servername: str, - host: str, - http_port: int = 8080, - grpc_port: int = 8081, - workers: int = 1, - ws_size: int = 16 * 1024 * 1024, - ssl_key=None, - ssl_cert=None, - timeout=300): - self.server_name = servername - self.app = None - self.host = host or '0.0.0.0' - self.http_port = http_port or 80 - self.grpc_port = grpc_port - self.workers = workers - self.keyfile = ssl_key - self.certfile = ssl_cert - self.ws_size = int(ws_size) - self.timeout = int(timeout) - protocal = "https" if self.certfile else "http" - self.url = f"{protocal}://{self.host}:{self.http_port}" - - def run(self, app, **kwargs): - if hasattr(app, "add_middleware"): - app.add_middleware( - CORSMiddleware, allow_origins=["*"], allow_credentials=True, - allow_methods=["*"], allow_headers=["*"], - ) - - uvicorn.run( - app, - host=self.host, - port=self.http_port, - ssl_keyfile=self.keyfile, - ssl_certfile=self.certfile, - workers=self.workers, - timeout_keep_alive=self.timeout, - **kwargs) - - def get_all_urls(self): - url_list = [{"path": route.path, "name": route.name} - for route in getattr(self.app, 'routes', [])] - return url_list - - -class InferenceServer(BaseServer): # pylint: disable=too-many-arguments - """ - rest api server for inference - """ - - def __init__( - self, - servername, - host: str, - http_port: int = 5000, - max_buffer_size: int = 104857600, - workers: int = 1): - super( - InferenceServer, - self).__init__( - servername=servername, - host=host, - http_port=http_port, - workers=workers) - - self.job, self.detection_validator = sedna_predict.init_ll_job() - - self.max_buffer_size = max_buffer_size - self.app = FastAPI( - routes=[ - APIRoute( - f"/{servername}", - self.model_info, - methods=["GET"], - ), - APIRoute( - f"/{servername}/predict", - self.predict, - methods=["POST"], - response_model=ResultResponse - ), - ], - log_level="trace", - timeout=600, - ) - self.index_frame = 0 - - def start(self): - return self.run(self.app) - - @staticmethod - def model_info(): - return HTMLResponse( - """

Welcome to the RestNet API!

-

To use this service, send a POST HTTP request to {this-url}/predict

-

The JSON payload has the following format: {"image": "BASE64_STRING_OF_IMAGE", - "depth": "BASE64_STRING_OF_DEPTH"}

- """) - - async def predict(self, image: UploadFile = File(...), depth: Optional[UploadFile] = None) -> ResultResponse: - contents = await image.read() - recieve_img_time = time.time() - print("Recieve image from the robo:", recieve_img_time) - - image = Image.open(BytesIO(contents)).convert('RGB') - - img_dep = None - self.index_frame = self.index_frame + 1 - - if depth: - depth_contents = await depth.read() - depth = Image.open(BytesIO(depth_contents)).convert('RGB') - img_dep = cv2.resize(np.array(depth), (2048, 1024), interpolation=cv2.INTER_CUBIC) - img_dep = Image.fromarray(img_dep) - - img_rgb = cv2.resize(np.array(image), (2048, 1024), interpolation=cv2.INTER_CUBIC) - img_rgb = Image.fromarray(img_rgb) - - sample = {'image': img_rgb, "depth": img_dep, "label": img_rgb} - results = sedna_predict.predict(self.job, data=sample, validator=self.detection_validator) - - predict_finish_time = time.time() - print(f"Prediction costs {predict_finish_time - recieve_img_time} seconds") - - post_process = True - if results["result"]["box"] is None: - results["result"]["curr"] = None - results["result"]["future"] = None - elif post_process: - curr, future = get_curb(results["result"]["box"]) - results["result"]["curr"] = curr - results["result"]["future"] = future - results["result"]["box"] = None - print("Post process cost at worker:", (time.time()-predict_finish_time)) - else: - results["result"]["curr"] = None - results["result"]["future"] = None - - print("Result transmit to robo time:", time.time()) - return results - -def parse_result(label, count): - label_map = ['road', 'sidewalk', ] - count_d = dict(zip(label, count)) - curb_count = count_d.get(19, 0) - if curb_count / np.sum(count) > 0.3: - return "curb" - r = sorted(label, key=count_d.get, reverse=True)[0] - try: - c = label_map[r] - except: - c = "other" - - return c - -def get_curb(results): - results = np.array(results[0]) - input_height, input_width = results.shape - - closest = np.array([ - [0, int(input_height)], - [int(input_width), - int(input_height)], - [int(0.118 * input_width + .5), - int(.8 * input_height + .5)], - [int(0.882 * input_width + .5), - int(.8 * input_height + .5)], - ]) - - future = np.array([ - [int(0.118 * input_width + .5), - int(.8 * input_height + .5)], - [int(0.882 * input_width + .5), - int(.8 * input_height + .5)], - [int(.765 * input_width + .5), - int(.66 * input_height + .5)], - [int(.235 * input_width + .5), - int(.66 * input_height + .5)] - ]) - - mask = np.zeros((input_height, input_width), dtype=np.uint8) - mask = cv2.fillPoly(mask, [closest], 1) - mask = cv2.fillPoly(mask, [future], 2) - d1, c1 = np.unique(results[mask == 1], return_counts=True) - d2, c2 = np.unique(results[mask == 2], return_counts=True) - c = parse_result(d1, c1) - f = parse_result(d2, c2) - - return c, f - -if __name__ == '__main__': - web_app = InferenceServer("lifelong-learning-robo", host=get_host_ip()) - web_app.start() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_evaluate.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_evaluate.py deleted file mode 100755 index 56633347..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_evaluate.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -os.environ['BACKEND_TYPE'] = 'PYTORCH' -# os.environ["KB_SERVER"] = "http://0.0.0.0:9020" -# os.environ["test_dataset_url"] = "./data_txt/sedna_data.txt" -# os.environ["MODEL_URLS"] = "./cloud_next_kb/index.pkl" -# os.environ["operator"] = "<" -# os.environ["model_threshold"] = "0" - -from sedna.core.lifelong_learning import LifelongLearning -from sedna.datasources import IndexDataParse -from sedna.common.config import Context - -from accuracy import accuracy -from basemodel import Model - -def _load_txt_dataset(dataset_url): - # use original dataset url - original_dataset_url = Context.get_parameters('original_dataset_url') - return os.path.join(os.path.dirname(original_dataset_url), dataset_url) - -def eval(): - estimator = Model() - eval_dataset_url = Context.get_parameters("test_dataset_url") - eval_data = IndexDataParse(data_type="eval", func=_load_txt_dataset) - eval_data.parse(eval_dataset_url, use_raw=False) - - task_allocation = { - "method": "TaskAllocationByOrigin", - "param": { - "origins": ["real", "sim"] - } - } - - ll_job = LifelongLearning(estimator, - task_definition=None, - task_relationship_discovery=None, - task_allocation=task_allocation, - task_remodeling=None, - inference_integrate=None, - task_update_decision=None, - unseen_task_allocation=None, - unseen_sample_recognition=None, - unseen_sample_re_recognition=None - ) - - ll_job.evaluate(eval_data, metrics=accuracy) - - -if __name__ == '__main__': - print(eval()) diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_predict.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_predict.py deleted file mode 100755 index b32c01d2..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_predict.py +++ /dev/null @@ -1,132 +0,0 @@ -import os - -os.environ['BACKEND_TYPE'] = 'PYTORCH' -# os.environ["UNSEEN_SAVE_URL"] = "s3://kubeedge/sedna-robo/unseen_samples/" -# set at yaml -# os.environ["PREDICT_RESULT_DIR"] = "./inference_results" -os.environ["TEST_DATASET_URL"] = "./data_txt/door_test.txt" -os.environ["EDGE_OUTPUT_URL"] = "./edge_kb" -os.environ["ORIGINAL_DATASET_URL"] = "/tmp" - -import torch -import numpy as np -from PIL import Image -import base64 -import tempfile -from io import BytesIO -from torchvision.transforms import ToPILImage -from torchvision import transforms -from torch.utils.data import DataLoader - -from sedna.datasources import IndexDataParse -from sedna.core.lifelong_learning import LifelongLearning -from sedna.common.config import Context - -from eval import Validator -from accuracy import accuracy -from basemodel import preprocess, val_args, Model -from dataloaders.utils import Colorize -from dataloaders import custom_transforms as tr -from dataloaders.datasets.cityscapes import CityscapesSegmentation - -def _load_txt_dataset(dataset_url): - # use original dataset url, - # see https://github.com/kubeedge/sedna/issues/35 - original_dataset_url = Context.get_parameters('original_dataset_url') - return os.path.join(os.path.dirname(original_dataset_url), dataset_url) - -def fetch_data(): - test_dataset_url = Context.get_parameters("test_dataset_url") - test_data = IndexDataParse(data_type="test", func=_load_txt_dataset) - test_data.parse(test_dataset_url, use_raw=False) - return test_data - -def pre_data_process(samples): - composed_transforms = transforms.Compose([ - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - data = BaseDataSource(data_type="test") - data.x = [(composed_transforms(samples), "")] - return data - -def post_process(res, is_unseen_task): - if is_unseen_task: - res, base64_string = None, None - else: - res = res[0].tolist() - - type = 0 if not is_unseen_task else 1 - mesg = { - "msg": "", - "result": { - "type": type, - "box": res - }, - "code": 0 - } - return mesg - -def image_merge(raw_img, result): - raw_img = ToPILImage()(raw_img) - - pre_colors = Colorize()(torch.from_numpy(result)) - pre_color_image = ToPILImage()(pre_colors[0]) # pre_colors.dtype = float64 - - image = raw_img.resize(pre_color_image.size, Image.BILINEAR) - image = image.convert('RGBA') - label = pre_color_image.convert('RGBA') - image = Image.blend(image, label, 0.6) - with tempfile.NamedTemporaryFile(suffix='.png') as f: - image.save(f.name) - - with open(f.name, 'rb') as open_file: - byte_content = open_file.read() - base64_bytes = base64.b64encode(byte_content) - base64_string = base64_bytes.decode('utf-8') - return base64_string - -def init_ll_job(): - estimator = Model() - inference_integrate = { - "method": "BBoxInferenceIntegrate" - } - unseen_task_allocation = { - "method": "UnseenTaskAllocationDefault" - } - unseen_sample_recognition = { - "method": "SampleRegonitionByRFNet" - } - - ll_job = LifelongLearning( - estimator, - task_definition=None, - task_relationship_discovery=None, - task_allocation=None, - task_remodeling=None, - inference_integrate=inference_integrate, - task_update_decision=None, - unseen_task_allocation=unseen_task_allocation, - unseen_sample_recognition=unseen_sample_recognition, - unseen_sample_re_recognition=None) - - args = val_args() - args.weight_path = "./models/detection_model.pth" - args.num_class = 31 - - return ll_job, Validator(args, unseen_detection=True) - -def predict(ll_job, data=None, validator=None): - if data: - data = pre_data_process(data) - else: - data = fetch_data() - data.x = preprocess(data.x) - - res, is_unseen_task, _ = ll_job.inference( - data, validator=validator, initial=False) - return post_process(res, is_unseen_task) - -if __name__ == '__main__': - ll_job, validator = init_ll_job() - print("Inference result:", predict(ll_job, validator=validator)) diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_train.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_train.py deleted file mode 100755 index 1c99361a..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/sedna_train.py +++ /dev/null @@ -1,78 +0,0 @@ -import os -os.environ['BACKEND_TYPE'] = 'PYTORCH' -os.environ["OUTPUT_URL"] = "./cloud_kb/" -# os.environ['CLOUD_KB_INDEX'] = "./cloud_kb/index.pkl" -os.environ["TRAIN_DATASET_URL"] = "./data_txt/sedna_data.txt" -os.environ["KB_SERVER"] = "http://0.0.0.0:9020" -os.environ["HAS_COMPLETED_INITIAL_TRAINING"] = "false" - -from sedna.common.file_ops import FileOps -from sedna.datasources import IndexDataParse -from sedna.common.config import Context, BaseConfig -from sedna.core.lifelong_learning import LifelongLearning - -from basemodel import Model - -def _load_txt_dataset(dataset_url): - # use original dataset url - original_dataset_url = Context.get_parameters('original_dataset_url') - return os.path.join(os.path.dirname(original_dataset_url), dataset_url) - -def train(estimator, train_data): - task_definition = { - "method": "TaskDefinitionByOrigin", - "param": { - "origins": ["real", "sim"] - } - } - - task_allocation = { - "method": "TaskAllocationByOrigin", - "param": { - "origins": ["real", "sim"] - } - } - - ll_job = LifelongLearning(estimator, - task_definition=task_definition, - task_relationship_discovery=None, - task_allocation=task_allocation, - task_remodeling=None, - inference_integrate=None, - task_update_decision=None, - unseen_task_allocation=None, - unseen_sample_recognition=None, - unseen_sample_re_recognition=None - ) - - ll_job.train(train_data) - -def update(estimator, train_data): - ll_job = LifelongLearning(estimator, - task_definition=None, - task_relationship_discovery=None, - task_allocation=None, - task_remodeling=None, - inference_integrate=None, - task_update_decision=None, - unseen_task_allocation=None, - unseen_sample_recognition=None, - unseen_sample_re_recognition=None - ) - - ll_job.update(train_data) - -def run(): - estimator = Model() - train_dataset_url = BaseConfig.train_dataset_url - train_data = IndexDataParse(data_type="train") - train_data.parse(train_dataset_url, use_raw=False) - - is_completed_initilization = str(Context.get_parameters("HAS_COMPLETED_INITIAL_TRAINING", "false")).lower() - if is_completed_initilization == "false": - train(estimator, train_data) - else: - update(estimator, train_data) - -if __name__ == '__main__': - run() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/test.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/test.py deleted file mode 100755 index fd9cd657..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/test.py +++ /dev/null @@ -1,52 +0,0 @@ -import numpy as np -import seaborn as sns -import pandas as pd -import matplotlib.pyplot as plt - -CPA_results = np.load("./cpa_results.npy").T -ratios = [0.3, 0.5, 0.6, 0.7, 0.8, 0.9] -ratio_counts = np.zeros((len(CPA_results), len(ratios)), dtype=float) - -for i in range(len(CPA_results)): - for j in range(len(ratios)): - result = CPA_results[i] - result = result[result <= ratios[j]] - - ratio_counts[i][j] = len(result) / 275 - -plt.figure(figsize=(45, 10)) -ratio_counts = pd.DataFrame(data=ratio_counts.T, index=ratios) -sns.heatmap(data=ratio_counts, annot=True, cmap="YlGnBu", annot_kws={'fontsize': 15}) -plt.xticks(fontsize=20) -plt.yticks(fontsize=25) -plt.xlabel("Test images", fontsize=25) -plt.ylabel("Ratio of PA ranges", fontsize=25) -plt.savefig("./figs/ratio_count.png") -plt.show() - - -# data = pd.DataFrame(CPA_results.T) -# -# plt.figure(figsize=(30, 15)) -# cpa_result = pd.DataFrame(data=data) -# sns.heatmap(data=cpa_result) -# plt.savefig("./figs/heatmap_pa.png") -# plt.show() -# -# plt.figure(figsize=(30, 15)) -# cpa_result = pd.DataFrame(data=data[:15]) -# sns.heatmap(data=cpa_result) -# plt.savefig("./figs/door_heatmap_pa.png") -# plt.show() -# -# plt.figure(figsize=(30, 15)) -# cpa_result = pd.DataFrame(data=data[15:31]) -# sns.heatmap(data=cpa_result) -# plt.savefig("./figs/garden1_heatmap_pa.png") -# plt.show() -# -# plt.figure(figsize=(30, 15)) -# cpa_result = pd.DataFrame(data=data[31:]) -# sns.heatmap(data=cpa_result) -# plt.savefig("./figs/garden2_heatmap_pa.png") -# plt.show() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/train.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/train.py deleted file mode 100755 index 4e673f85..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/train.py +++ /dev/null @@ -1,310 +0,0 @@ -import argparse -import os -import numpy as np -from tqdm import tqdm -import torch -import copy - -from mypath import Path -from dataloaders import make_data_loader -from models.rfnet import RFNet -from models.resnet.resnet_single_scale_single_attention import * -from utils.loss import SegmentationLosses -from models.replicate import patch_replication_callback -from utils.calculate_weights import calculate_weigths_labels -from utils.lr_scheduler import LR_Scheduler -from utils.saver import Saver -from utils.summaries import TensorboardSummary -from utils.metrics import Evaluator -from sedna.datasources import BaseDataSource - -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" - -class Trainer(object): - def __init__(self, args, train_data=None, valid_data=None): - self.args = args - # Define Saver - self.saver = Saver(args) - self.saver.save_experiment_config() - # Define Tensorboard Summary - self.summary = TensorboardSummary(self.saver.experiment_dir) - self.writer = self.summary.create_summary() - # denormalize for detph image - self.mean_depth = torch.as_tensor(0.12176, dtype=torch.float32, device='cpu') - self.std_depth = torch.as_tensor(0.09752, dtype=torch.float32, device='cpu') - self.nclass = args.num_class - # Define Dataloader - kwargs = {'num_workers': args.workers, 'pin_memory': True} - self.train_loader, self.val_loader, self.test_loader, _ = make_data_loader(args, train_data=train_data, - valid_data=valid_data, **kwargs) - - # Define network - resnet = resnet18(pretrained=True, efficient=False, use_bn=True) - model = RFNet(resnet, num_classes=self.nclass, use_bn=True) - train_params = [{'params': model.random_init_params(), 'lr': args.lr}, - {'params': model.fine_tune_params(), 'lr': 0.1*args.lr, 'weight_decay':args.weight_decay}] - # Define Optimizer - optimizer = torch.optim.Adam(train_params, lr=args.lr, - weight_decay=args.weight_decay) - # Define Criterion - # whether to use class balanced weights - if args.use_balanced_weights: - classes_weights_path = os.path.join(Path.db_root_dir(args.dataset), args.dataset+'_classes_weights.npy') - if os.path.isfile(classes_weights_path): - weight = np.load(classes_weights_path) - else: - weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass) - weight = torch.from_numpy(weight.astype(np.float32)) - else: - weight = None - # Define loss function - self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda, gpu_ids=args.gpu_ids).build_loss(mode=args.loss_type) - self.model, self.optimizer = model, optimizer - # Define Evaluator - self.evaluator = Evaluator(self.nclass) - # # Define lr scheduler - self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr, args.epochs, len(self.train_loader)) - # Using cuda - if args.cuda: - #self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids) - #patch_replication_callback(self.model) - self.model = self.model.cuda(args.gpu_ids) - - # Resuming checkpoint - self.best_pred = 0.0 - if args.resume is not None: - if not os.path.isfile(args.resume): - raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume)) - print(f"Training: load model from {args.resume}") - checkpoint = torch.load(args.resume, map_location=torch.device("cpu")) - args.start_epoch = checkpoint['epoch'] - # if args.cuda: - # self.model.load_state_dict(checkpoint['state_dict']) - # else: - # self.model.load_state_dict(checkpoint['state_dict']) - self.model.load_state_dict(checkpoint['state_dict']) - if not args.ft: - self.optimizer.load_state_dict(checkpoint['optimizer']) - self.best_pred = checkpoint['best_pred'] - print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch'])) - - # Clear start epoch if fine-tuning - if args.ft: - args.start_epoch = 0 - - def get_weight(self): - print("get weight") - current_model = copy.deepcopy(self.model) - return current_model.parameters() - - def set_weight(self, weights): - length = len(weights) - print("set weight", length) - print("model:", self.args.resume) - tau = 0.2 - if length == 1: - for param, target_param in zip(weights[0], self.model.parameters()): - target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data) - elif length == 2: - for param1, param2, target_param in zip(weights[0], weights[1], self.model.parameters()): - target_param.data.copy_(0.5 * tau * param1.data + 0.5 * tau * param2.data + (1 - tau) * target_param.data) - - - - def my_training(self, epoch): - train_loss = 0.0 - print(self.optimizer.state_dict()['param_groups'][0]['lr']) - current_model = copy.deepcopy(self.model) - self.model.train() - tbar = tqdm(self.train_loader) - num_img_tr = len(self.train_loader) - - for i, sample in enumerate(tbar): - if self.args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - #print(target.shape) - else: - image, target = sample['image'], sample['label'] - print(image.shape) - if self.args.cuda: - image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids) - if self.args.depth: - depth = depth.cuda(self.args.gpu_ids) - self.scheduler(self.optimizer, i, epoch, self.best_pred) - self.optimizer.zero_grad() - - if self.args.depth: - output = self.model(image, depth) - else: - output = self.model(image) - #print(target.max()) - #print(output.shape) - target[target > self.nclass-1] = 255 - loss = self.criterion(output, target) - loss.backward() - self.optimizer.step() - #print(self.optimizer.state_dict()['param_groups'][0]['lr']) - train_loss += loss.item() - tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1))) - self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch) - # Show 10 * 3 inference results each epoch - if i % (num_img_tr // 10 + 1) == 0: - global_step = i + num_img_tr * epoch - if self.args.depth: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) - - depth_display = depth[0].cpu().unsqueeze(0) - depth_display = depth_display.mul_(self.std_depth).add_(self.mean_depth) - depth_display = depth_display.numpy() - depth_display = depth_display*255 - depth_display = depth_display.astype(np.uint8) - self.writer.add_image('Depth', depth_display, global_step) - - else: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) - - self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch) - print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) - print('Loss: %.3f' % train_loss) - tau = 0.3 - flag = True - for param, target_param in zip(current_model.parameters(), self.model.parameters()): - if flag: - #print(param[0]-target_param[0]) - flag = False - target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data) - - del current_model - # if self.args.no_val: - # # save checkpoint every epoch - # is_best = False - # checkpoint_path = self.saver.save_checkpoint({ - # 'epoch': epoch + 1, - # 'state_dict': self.model.state_dict(), - # 'optimizer': self.optimizer.state_dict(), - # 'best_pred': self.best_pred, - # }, is_best) - return train_loss - - def training(self, epoch): - train_loss = 0.0 - print(self.optimizer.state_dict()['param_groups'][0]['lr']) - self.model.train() - tbar = tqdm(self.train_loader) - num_img_tr = len(self.train_loader) - - for i, sample in enumerate(tbar): - if self.args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - #print(target.shape) - else: - image, target = sample['image'], sample['label'] - print(image.shape) - if self.args.cuda: - image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids) - if self.args.depth: - depth = depth.cuda(self.args.gpu_ids) - self.scheduler(self.optimizer, i, epoch, self.best_pred) - self.optimizer.zero_grad() - - if self.args.depth: - output = self.model(image, depth) - else: - output = self.model(image) - #print(target.max()) - #print(output.shape) - target[target > self.nclass-1] = 255 - loss = self.criterion(output, target) - loss.backward() - self.optimizer.step() - #print(self.optimizer.state_dict()['param_groups'][0]['lr']) - train_loss += loss.item() - tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1))) - self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch) - # Show 10 * 3 inference results each epoch - if i % (num_img_tr // 10 + 1) == 0: - global_step = i + num_img_tr * epoch - if self.args.depth: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) - - depth_display = depth[0].cpu().unsqueeze(0) - depth_display = depth_display.mul_(self.std_depth).add_(self.mean_depth) - depth_display = depth_display.numpy() - depth_display = depth_display*255 - depth_display = depth_display.astype(np.uint8) - self.writer.add_image('Depth', depth_display, global_step) - - else: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) - - self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch) - print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) - print('Loss: %.3f' % train_loss) - - # if self.args.no_val: - # # save checkpoint every epoch - # is_best = False - # checkpoint_path = self.saver.save_checkpoint({ - # 'epoch': epoch + 1, - # 'state_dict': self.model.state_dict(), - # 'optimizer': self.optimizer.state_dict(), - # 'best_pred': self.best_pred, - # }, is_best) - return train_loss - - def validation(self, epoch): - self.model.eval() - self.evaluator.reset() - tbar = tqdm(self.val_loader, desc='\r') - test_loss = 0.0 - for i, (sample, img_path) in enumerate(tbar): - if self.args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - image, target = sample['image'], sample['label'] - # print(f"val image is {image}") - if self.args.cuda: - image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids) - if self.args.depth: - depth = depth.cuda(self.args.gpu_ids) - with torch.no_grad(): - if self.args.depth: - output = self.model(image, depth) - else: - output = self.model(image) - target[target > self.nclass-1] = 255 - loss = self.criterion(output, target) - test_loss += loss.item() - tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1))) - pred = output.data.cpu().numpy() - target = target.cpu().numpy() - pred = np.argmax(pred, axis=1) - # Add batch sample into evaluator - self.evaluator.add_batch(target, pred) - - # Fast test during the training - Acc = self.evaluator.Pixel_Accuracy() - Acc_class = self.evaluator.Pixel_Accuracy_Class() - mIoU = self.evaluator.Mean_Intersection_over_Union() - FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union() - self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch) - self.writer.add_scalar('val/mIoU', mIoU, epoch) - self.writer.add_scalar('val/Acc', Acc, epoch) - self.writer.add_scalar('val/Acc_class', Acc_class, epoch) - self.writer.add_scalar('val/fwIoU', FWIoU, epoch) - print('Validation:') - print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) - print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU)) - print('Loss: %.3f' % test_loss) - - new_pred = mIoU - if new_pred > self.best_pred: - is_best = True - self.best_pred = new_pred - self.saver.save_checkpoint({ - 'epoch': epoch + 1, - 'state_dict': self.model.state_dict(), - 'optimizer': self.optimizer.state_dict(), - 'best_pred': self.best_pred, - }, is_best) \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/__init__.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py deleted file mode 100755 index 3691a1f3..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py +++ /dev/null @@ -1,52 +0,0 @@ -class TrainArgs: - def __init__(self, **kwargs): - self.depth = False - self.dataset = 'cityscapes' - self.workers = 4 - self.base_size = 1024 - self.crop_size = 768 - self.loss_type = 'ce' - self.epochs = kwargs.get("epochs", 2) - self.start_epoch = 0 - - self.batch_size = 4 - self.val_batch_size = 1 - self.use_balanced_weights = False - self.num_class = 30 - self.lr = kwargs.get("learning_rate", 1e-4) - self.lr_scheduler = 'cos' - self.momentum = 0.9 - self.weight_decay = 2.5e-5 - self.no_cuda = False - self.gpu_ids = 0 - - self.seed = 1 - self.resume = '/home/hsj/ianvs/project/pretrain/pretrain_model.pth' - self.checkname = 'RFNet' - self.ft = True - self.eval_interval = kwargs.get("eval_interval", 50) - self.no_val = kwargs.get("no_val", True) - self.cuda = True - - -class ValArgs: - def __init__(self, **kwargs): - self.dataset = 'cityscapes' - self.workers = 0 - self.base_size = 1024 - self.crop_size = 768 - self.batch_size = 6 - self.val_batch_size = 1 - self.test_batch_size = 1 - self.num_class = 30 - self.no_cuda = False - self.gpu_ids = 0 - self.checkname = None - self.weight_path = "./models/530_exp3_2.pth" - self.save_predicted_image = False - self.color_label_save_path = './test/color' - self.merge_label_save_path = './test/merge' - self.label_save_path = './test/label' - self.merge = True - self.depth = False - self.cuda = True diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/calculate_weights.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/calculate_weights.py deleted file mode 100755 index 2c2c9821..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/calculate_weights.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -from tqdm import tqdm -import numpy as np -from mypath import Path - -def calculate_weigths_labels(dataset, dataloader, num_classes): - # Create an instance from the data loader - z = np.zeros((num_classes,)) - # Initialize tqdm - tqdm_batch = tqdm(dataloader) - print('Calculating classes weights') - for sample in tqdm_batch: - y = sample['label'] - y = y.detach().cpu().numpy() - mask = (y >= 0) & (y < num_classes) - labels = y[mask].astype(np.uint8) - count_l = np.bincount(labels, minlength=num_classes) - z += count_l - tqdm_batch.close() - total_frequency = np.sum(z) - class_weights = [] - for frequency in z: - class_weight = 1 / (np.log(1.02 + (frequency / total_frequency))) - class_weights.append(class_weight) - ret = np.array(class_weights) - classes_weights_path = os.path.join(Path.db_root_dir(dataset), dataset+'_classes_weights.npy') - np.save(classes_weights_path, ret) - - return ret \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/iouEval.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/iouEval.py deleted file mode 100755 index d3801a9e..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/iouEval.py +++ /dev/null @@ -1,141 +0,0 @@ -import torch - -class iouEval: - - def __init__(self, nClasses, ignoreIndex=20): - - self.nClasses = nClasses - self.ignoreIndex = ignoreIndex if nClasses > ignoreIndex else -1 # if ignoreIndex is larger than nClasses, consider no ignoreIndex - self.reset() - - def reset(self): - classes = self.nClasses if self.ignoreIndex == -1 else self.nClasses - 1 - self.tp = torch.zeros(classes).double() - self.fp = torch.zeros(classes).double() - self.fn = torch.zeros(classes).double() - self.cdp_obstacle = torch.zeros(1).double() - self.tp_obstacle = torch.zeros(1).double() - self.idp_obstacle = torch.zeros(1).double() - self.tp_nonobstacle = torch.zeros(1).double() - # self.cdi = torch.zeros(1).double() - - def addBatch(self, x, y): # x=preds, y=targets - # sizes should be "batch_size x nClasses x H x W" - # cdi = 0 - - # print ("X is cuda: ", x.is_cuda) - # print ("Y is cuda: ", y.is_cuda) - - if (x.is_cuda or y.is_cuda): - x = x.cuda() - y = y.cuda() - - # if size is "batch_size x 1 x H x W" scatter to onehot - if (x.size(1) == 1): - x_onehot = torch.zeros(x.size(0), self.nClasses, x.size(2), x.size(3)) - if x.is_cuda: - x_onehot = x_onehot.cuda() - x_onehot.scatter_(1, x, 1).float() # dim index src 按照列用1替换0，索引为x - else: - x_onehot = x.float() - - if (y.size(1) == 1): - y_onehot = torch.zeros(y.size(0), self.nClasses, y.size(2), y.size(3)) - if y.is_cuda: - y_onehot = y_onehot.cuda() - y_onehot.scatter_(1, y, 1).float() - else: - y_onehot = y.float() - - if (self.ignoreIndex != -1): - ignores = y_onehot[:, self.ignoreIndex].unsqueeze(1) # 加一维 - x_onehot = x_onehot[:, :self.ignoreIndex] # ignoreIndex后的都不要 - y_onehot = y_onehot[:, :self.ignoreIndex] - else: - ignores = 0 - - - tpmult = x_onehot * y_onehot # times prediction and gt coincide is 1 - tp = torch.sum(torch.sum(torch.sum(tpmult, dim=0, keepdim=True), dim=2, keepdim=True), dim=3, - keepdim=True).squeeze() - fpmult = x_onehot * ( - 1 - y_onehot - ignores) # times prediction says its that class and gt says its not (subtracting cases when its ignore label!) - fp = torch.sum(torch.sum(torch.sum(fpmult, dim=0, keepdim=True), dim=2, keepdim=True), dim=3, - keepdim=True).squeeze() - fnmult = (1 - x_onehot) * (y_onehot) # times prediction says its not that class and gt says it is - fn = torch.sum(torch.sum(torch.sum(fnmult, dim=0, keepdim=True), dim=2, keepdim=True), dim=3, - keepdim=True).squeeze() - - self.tp += tp.double().cpu() - self.fp += fp.double().cpu() - self.fn += fn.double().cpu() - - cdp_obstacle = tpmult[:, 19].sum() # obstacle index 19 - tp_obstacle = y_onehot[:, 19].sum() - - idp_obstacle = (x_onehot[:, 19] - tpmult[:, 19]).sum() - tp_nonobstacle = (-1*y_onehot+1).sum() - - # for i in range(0, x.size(0)): - # if tpmult[i].sum()/(y_onehot[i].sum() + 1e-15) >= 0.5: - # cdi += 1 - - - self.cdp_obstacle += cdp_obstacle.double().cpu() - self.tp_obstacle += tp_obstacle.double().cpu() - self.idp_obstacle += idp_obstacle.double().cpu() - self.tp_nonobstacle += tp_nonobstacle.double().cpu() - # self.cdi += cdi.double().cpu() - - - - def getIoU(self): - num = self.tp - den = self.tp + self.fp + self.fn + 1e-15 - iou = num / den - iou_not_zero = list(filter(lambda x: x != 0, iou)) - # print(len(iou_not_zero)) - iou_mean = sum(iou_not_zero) / len(iou_not_zero) - tfp = self.tp + self.fp + 1e-15 - acc = num / tfp - acc_not_zero = list(filter(lambda x: x != 0, acc)) - acc_mean = sum(acc_not_zero) / len(acc_not_zero) - - return iou_mean, iou, acc_mean, acc # returns "iou mean", "iou per class" - - def getObstacleEval(self): - - pdr_obstacle = self.cdp_obstacle / (self.tp_obstacle+1e-15) - pfp_obstacle = self.idp_obstacle / (self.tp_nonobstacle+1e-15) - - return pdr_obstacle, pfp_obstacle - - -# Class for colors -class colors: - RED = '\033[31;1m' - GREEN = '\033[32;1m' - YELLOW = '\033[33;1m' - BLUE = '\033[34;1m' - MAGENTA = '\033[35;1m' - CYAN = '\033[36;1m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - ENDC = '\033[0m' - - -# Colored value output if colorized flag is activated. -def getColorEntry(val): - if not isinstance(val, float): - return colors.ENDC - if (val < .20): - return colors.RED - elif (val < .40): - return colors.YELLOW - elif (val < .60): - return colors.BLUE - elif (val < .80): - return colors.CYAN - else: - return colors.GREEN - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/loss.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/loss.py deleted file mode 100755 index 33f19042..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/loss.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import torch.nn as nn - -class SegmentationLosses(object): - def __init__(self, weight=None, size_average=True, batch_average=True, ignore_index=255, cuda=False, gpu_ids=0): # ignore_index=255 - self.ignore_index = ignore_index - self.weight = weight - self.size_average = size_average - self.batch_average = batch_average - self.cuda = cuda - self.gpu_ids = gpu_ids - - def build_loss(self, mode='ce'): - """Choices: ['ce' or 'focal']""" - if mode == 'ce': - return self.CrossEntropyLoss - elif mode == 'focal': - return self.FocalLoss - else: - raise NotImplementedError - - def CrossEntropyLoss(self, logit, target): - n, c, h, w = logit.size() - #criterion = nn.CrossEntropyLoss(weight=self.weight, ignore_index=self.ignore_index, - #size_average=self.size_average) - criterion = nn.CrossEntropyLoss(reduction='mean', ignore_index=self.ignore_index) - if self.cuda: - criterion = criterion.cuda(self.gpu_ids) - - loss = criterion(logit, target.long()) - - if self.batch_average: - loss /= n - - return loss - - def FocalLoss(self, logit, target, gamma=2, alpha=0.5): - n, c, h, w = logit.size() - criterion = nn.CrossEntropyLoss(weight=self.weight, ignore_index=self.ignore_index, - size_average=self.size_average) - if self.cuda: - criterion = criterion.cuda(self.gpu_ids) - - logpt = -criterion(logit, target.long()) - pt = torch.exp(logpt) - if alpha is not None: - logpt *= alpha - loss = -((1 - pt) ** gamma) * logpt - - if self.batch_average: - loss /= n - - return loss - -if __name__ == "__main__": - loss = SegmentationLosses(cuda=True) - a = torch.rand(1, 3, 7, 7).cuda() - b = torch.rand(1, 7, 7).cuda() - print(loss.CrossEntropyLoss(a, b).item()) - print(loss.FocalLoss(a, b, gamma=0, alpha=None).item()) - print(loss.FocalLoss(a, b, gamma=2, alpha=0.5).item()) - - - - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/lr_scheduler.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/lr_scheduler.py deleted file mode 100755 index 47124028..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/lr_scheduler.py +++ /dev/null @@ -1,70 +0,0 @@ -##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -## Created by: Hang Zhang -## ECE Department, Rutgers University -## Email: zhang.hang@rutgers.edu -## Copyright (c) 2017 -## -## This source code is licensed under the MIT-style license found in the -## LICENSE file in the root directory of this source tree -##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -import math - -class LR_Scheduler(object): - """Learning Rate Scheduler - - Step mode: ``lr = baselr * 0.1 ^ {floor(epoch-1 / lr_step)}`` - - Cosine mode: ``lr = baselr * 0.5 * (1 + cos(iter/maxiter))`` - - Poly mode: ``lr = baselr * (1 - iter/maxiter) ^ 0.9`` - - Args: - args: - :attr:`args.lr_scheduler` lr scheduler mode (`cos`, `poly`), - :attr:`args.lr` base learning rate, :attr:`args.epochs` number of epochs, - :attr:`args.lr_step` - - iters_per_epoch: number of iterations per epoch - """ - def __init__(self, mode, base_lr, num_epochs, iters_per_epoch=0, - lr_step=0, warmup_epochs=0): - self.mode = mode - print('Using {} LR Scheduler!'.format(self.mode)) - self.lr = base_lr - if mode == 'step': - assert lr_step - self.lr_step = lr_step - self.iters_per_epoch = iters_per_epoch - self.N = num_epochs * iters_per_epoch - self.epoch = -1 - self.warmup_iters = warmup_epochs * iters_per_epoch - - def __call__(self, optimizer, i, epoch, best_pred): - T = epoch * self.iters_per_epoch + i - if self.mode == 'cos': - lr = 0.5 * self.lr * (1 + math.cos(1.0 * T / self.N * math.pi)) - elif self.mode == 'poly': - lr = self.lr * pow((1 - 1.0 * T / self.N), 2) - elif self.mode == 'step': - lr = self.lr * (0.1 ** (epoch // self.lr_step)) - else: - raise NotImplemented - # warm up lr schedule - if self.warmup_iters > 0 and T < self.warmup_iters: - lr = lr * 1.0 * T / self.warmup_iters - if epoch > self.epoch: - print('\n=>Epoches %i, learning rate = %.4f, \ - previous best = %.4f' % (epoch, lr, best_pred)) - self.epoch = epoch - assert lr >= 0 - self._adjust_learning_rate(optimizer, lr) - - def _adjust_learning_rate(self, optimizer, lr): - if len(optimizer.param_groups) == 1: - optimizer.param_groups[0]['lr'] = lr * 4 - else: - # enlarge the lr at the head - optimizer.param_groups[0]['lr'] = lr * 4 - for i in range(1, len(optimizer.param_groups)): - optimizer.param_groups[i]['lr'] = lr diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py deleted file mode 100755 index 0acef918..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py +++ /dev/null @@ -1,203 +0,0 @@ -import numpy as np - - -class Evaluator(object): - def __init__(self, num_class): - self.num_class = num_class - self.confusion_matrix = np.zeros((self.num_class,)*2) # shape:(num_class, num_class) - - def Pixel_Accuracy(self): - Acc = np.diag(self.confusion_matrix).sum() / self.confusion_matrix.sum() - return Acc - - def Pixel_Accuracy_Class_Curb(self): - Acc = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1) - print('-----------Acc of each classes-----------') - print("road : %.6f" % (Acc[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (Acc[1] * 100.0), "%\t") - Acc = np.nanmean(Acc[:2]) - return Acc - - - def Pixel_Accuracy_Class(self): - Acc = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1) - print('-----------Acc of each classes-----------') - print("road : %.6f" % (Acc[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (Acc[1] * 100.0), "%\t") - print("building : %.6f" % (Acc[2] * 100.0), "%\t") - print("wall : %.6f" % (Acc[3] * 100.0), "%\t") - print("fence : %.6f" % (Acc[4] * 100.0), "%\t") - print("pole : %.6f" % (Acc[5] * 100.0), "%\t") - print("traffic light: %.6f" % (Acc[6] * 100.0), "%\t") - print("traffic sign : %.6f" % (Acc[7] * 100.0), "%\t") - print("vegetation : %.6f" % (Acc[8] * 100.0), "%\t") - print("terrain : %.6f" % (Acc[9] * 100.0), "%\t") - print("sky : %.6f" % (Acc[10] * 100.0), "%\t") - print("person : %.6f" % (Acc[11] * 100.0), "%\t") - print("rider : %.6f" % (Acc[12] * 100.0), "%\t") - print("car : %.6f" % (Acc[13] * 100.0), "%\t") - print("truck : %.6f" % (Acc[14] * 100.0), "%\t") - print("bus : %.6f" % (Acc[15] * 100.0), "%\t") - print("train : %.6f" % (Acc[16] * 100.0), "%\t") - print("motorcycle : %.6f" % (Acc[17] * 100.0), "%\t") - print("bicycle : %.6f" % (Acc[18] * 100.0), "%\t") - print("stair : %.6f" % (Acc[19] * 100.0), "%\t") - print("curb : %.6f" % (Acc[20] * 100.0), "%\t") - print("ramp : %.6f" % (Acc[21] * 100.0), "%\t") - print("runway : %.6f" % (Acc[22] * 100.0), "%\t") - print("flowerbed : %.6f" % (Acc[23] * 100.0), "%\t") - print("door : %.6f" % (Acc[24] * 100.0), "%\t") - print("CCTV camera : %.6f" % (Acc[25] * 100.0), "%\t") - print("Manhole : %.6f" % (Acc[26] * 100.0), "%\t") - print("hydrant : %.6f" % (Acc[27] * 100.0), "%\t") - print("belt : %.6f" % (Acc[28] * 100.0), "%\t") - print("dustbin : %.6f" % (Acc[29] * 100.0), "%\t") - if self.num_class == 20: - print("small obstacles: %.6f" % (Acc[19] * 100.0), "%\t") - Acc = np.nanmean(Acc) - return Acc - - def Mean_Intersection_over_Union(self): - MIoU = np.diag(self.confusion_matrix) / ( - np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) - - np.diag(self.confusion_matrix)) - - # print MIoU of each class - print('-----------IoU of each classes-----------') - print("road : %.6f" % (MIoU[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (MIoU[1] * 100.0), "%\t") - print("building : %.6f" % (MIoU[2] * 100.0), "%\t") - print("wall : %.6f" % (MIoU[3] * 100.0), "%\t") - print("fence : %.6f" % (MIoU[4] * 100.0), "%\t") - print("pole : %.6f" % (MIoU[5] * 100.0), "%\t") - print("traffic light: %.6f" % (MIoU[6] * 100.0), "%\t") - print("traffic sign : %.6f" % (MIoU[7] * 100.0), "%\t") - print("vegetation : %.6f" % (MIoU[8] * 100.0), "%\t") - print("terrain : %.6f" % (MIoU[9] * 100.0), "%\t") - print("sky : %.6f" % (MIoU[10] * 100.0), "%\t") - print("person : %.6f" % (MIoU[11] * 100.0), "%\t") - print("rider : %.6f" % (MIoU[12] * 100.0), "%\t") - print("car : %.6f" % (MIoU[13] * 100.0), "%\t") - print("truck : %.6f" % (MIoU[14] * 100.0), "%\t") - print("bus : %.6f" % (MIoU[15] * 100.0), "%\t") - print("train : %.6f" % (MIoU[16] * 100.0), "%\t") - print("motorcycle : %.6f" % (MIoU[17] * 100.0), "%\t") - print("bicycle : %.6f" % (MIoU[18] * 100.0), "%\t") - print("stair : %.6f" % (MIoU[19] * 100.0), "%\t") - print("curb : %.6f" % (MIoU[20] * 100.0), "%\t") - print("ramp : %.6f" % (MIoU[21] * 100.0), "%\t") - print("runway : %.6f" % (MIoU[22] * 100.0), "%\t") - print("flowerbed : %.6f" % (MIoU[23] * 100.0), "%\t") - print("door : %.6f" % (MIoU[24] * 100.0), "%\t") - print("CCTV camera : %.6f" % (MIoU[25] * 100.0), "%\t") - print("Manhole : %.6f" % (MIoU[26] * 100.0), "%\t") - print("hydrant : %.6f" % (MIoU[27] * 100.0), "%\t") - print("belt : %.6f" % (MIoU[28] * 100.0), "%\t") - print("dustbin : %.6f" % (MIoU[29] * 100.0), "%\t") - if self.num_class == 20: - print("small obstacles: %.6f" % (MIoU[19] * 100.0), "%\t") - - MIoU = np.nanmean(MIoU) - return MIoU - - def Mean_Intersection_over_Union_Curb(self): - MIoU = np.diag(self.confusion_matrix) / ( - np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) - - np.diag(self.confusion_matrix)) - - # print MIoU of each class - print('-----------IoU of each classes-----------') - print("road : %.6f" % (MIoU[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (MIoU[1] * 100.0), "%\t") - - if self.num_class == 20: - print("small obstacles: %.6f" % (MIoU[19] * 100.0), "%\t") - - MIoU = np.nanmean(MIoU[:2]) - return MIoU - - def Frequency_Weighted_Intersection_over_Union(self): - freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix) - iu = np.diag(self.confusion_matrix) / ( - np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) - - np.diag(self.confusion_matrix)) - - FWIoU = (freq[freq > 0] * iu[freq > 0]).sum() - CFWIoU = freq[freq > 0] * iu[freq > 0] - if len(CFWIoU) > 1: - print('-----------FWIoU of each classes-----------') - print("road : %.6f" % (CFWIoU[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (CFWIoU[1] * 100.0), "%\t") - - print('-----------freq of each classes-----------') - print("road : %.6f" % (freq[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (freq[1] * 100.0), "%\t") - print("building : %.6f" % (freq[2] * 100.0), "%\t") - print("wall : %.6f" % (freq[3] * 100.0), "%\t") - print("fence : %.6f" % (freq[4] * 100.0), "%\t") - print("pole : %.6f" % (freq[5] * 100.0), "%\t") - print("traffic light: %.6f" % (freq[6] * 100.0), "%\t") - print("traffic sign : %.6f" % (freq[7] * 100.0), "%\t") - print("vegetation : %.6f" % (freq[8] * 100.0), "%\t") - print("terrain : %.6f" % (freq[9] * 100.0), "%\t") - print("sky : %.6f" % (freq[10] * 100.0), "%\t") - print("person : %.6f" % (freq[11] * 100.0), "%\t") - print("rider : %.6f" % (freq[12] * 100.0), "%\t") - print("car : %.6f" % (freq[13] * 100.0), "%\t") - print("truck : %.6f" % (freq[14] * 100.0), "%\t") - print("bus : %.6f" % (freq[15] * 100.0), "%\t") - print("train : %.6f" % (freq[16] * 100.0), "%\t") - print("motorcycle : %.6f" % (freq[17] * 100.0), "%\t") - print("bicycle : %.6f" % (freq[18] * 100.0), "%\t") - print("stair : %.6f" % (freq[19] * 100.0), "%\t") - print("curb : %.6f" % (freq[20] * 100.0), "%\t") - print("ramp : %.6f" % (freq[21] * 100.0), "%\t") - print("runway : %.6f" % (freq[22] * 100.0), "%\t") - print("flowerbed : %.6f" % (freq[23] * 100.0), "%\t") - print("door : %.6f" % (freq[24] * 100.0), "%\t") - print("CCTV camera : %.6f" % (freq[25] * 100.0), "%\t") - print("Manhole : %.6f" % (freq[26] * 100.0), "%\t") - print("hydrant : %.6f" % (freq[27] * 100.0), "%\t") - print("belt : %.6f" % (freq[28] * 100.0), "%\t") - print("dustbin : %.6f" % (freq[29] * 100.0), "%\t") - - return FWIoU - - def Frequency_Weighted_Intersection_over_Union_Curb(self): - freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix) - iu = np.diag(self.confusion_matrix) / ( - np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) - - np.diag(self.confusion_matrix)) - - # FWIoU = (freq[freq > 0] * iu[freq > 0]).sum() - CFWIoU = freq[freq > 0] * iu[freq > 0] - print('-----------FWIoU of each classes-----------') - print("road : %.6f" % (CFWIoU[0] * 100.0), "%\t") - print("sidewalk : %.6f" % (CFWIoU[1] * 100.0), "%\t") - - return np.nanmean(CFWIoU[:2]) - - def _generate_matrix(self, gt_image, pre_image): - mask = (gt_image >= 0) & (gt_image < self.num_class) - label = self.num_class * gt_image[mask].astype('int') + pre_image[mask] - count = np.bincount(label, minlength=self.num_class**2) - confusion_matrix = count.reshape(self.num_class, self.num_class) - return confusion_matrix - - def add_batch(self, gt_image, pre_image): - gt_image = np.array(gt_image) - pre_image = np.array(pre_image) - print(gt_image.shape, pre_image.shape) - if gt_image.shape != pre_image.shape: - pre_image = pre_image[0] - if gt_image.shape != pre_image.shape: - pre_image = pre_image[0] - assert gt_image.shape == pre_image.shape - self.confusion_matrix += self._generate_matrix(gt_image, pre_image) - - def reset(self): - self.confusion_matrix = np.zeros((self.num_class,) * 2) - - - - diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/saver.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/saver.py deleted file mode 100755 index 03866432..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/saver.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import time -import shutil -import tempfile -import torch -from collections import OrderedDict -import glob - -class Saver(object): - - def __init__(self, args): - self.args = args - self.directory = os.path.join('/tmp', args.dataset, args.checkname) - self.runs = sorted(glob.glob(os.path.join(self.directory, 'experiment_*'))) - run_id = int(self.runs[-1].split('_')[-1]) + 1 if self.runs else 0 - - self.experiment_dir = os.path.join(self.directory, 'experiment_{}'.format(str(run_id))) - if not os.path.exists(self.experiment_dir): - os.makedirs(self.experiment_dir) - - def save_checkpoint(self, state, is_best): # filename from .pth.tar change to .pth? - """Saves checkpoint to disk""" - filename = f'checkpoint_{time.time()}.pth' - checkpoint_path = os.path.join(self.experiment_dir, filename) - torch.save(state, checkpoint_path) - if is_best: - best_pred = state['best_pred'] - with open(os.path.join(self.experiment_dir, 'best_pred.txt'), 'w') as f: - f.write(str(best_pred)) - if self.runs: - previous_miou = [0.0] - for run in self.runs: - run_id = run.split('_')[-1] - path = os.path.join(self.directory, 'experiment_{}'.format(str(run_id)), 'best_pred.txt') - if os.path.exists(path): - with open(path, 'r') as f: - miou = float(f.readline()) - previous_miou.append(miou) - else: - continue - max_miou = max(previous_miou) - if best_pred > max_miou: - checkpoint_path_best = os.path.join(self.directory, 'model_best.pth') - shutil.copyfile(checkpoint_path, checkpoint_path_best) - checkpoint_path = checkpoint_path_best - else: - checkpoint_path_best = os.path.join(self.directory, 'model_best.pth') - shutil.copyfile(checkpoint_path, checkpoint_path_best) - checkpoint_path = checkpoint_path_best - - return checkpoint_path - - def save_experiment_config(self): - logfile = os.path.join(self.experiment_dir, 'parameters.txt') - log_file = open(logfile, 'w') - p = OrderedDict() - p['datset'] = self.args.dataset - # p['out_stride'] = self.args.out_stride - p['lr'] = self.args.lr - p['lr_scheduler'] = self.args.lr_scheduler - p['loss_type'] = self.args.loss_type - p['epoch'] = self.args.epochs - p['base_size'] = self.args.base_size - p['crop_size'] = self.args.crop_size - - for key, val in p.items(): - log_file.write(key + ':' + str(val) + '\n') - log_file.close() \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/summaries.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/summaries.py deleted file mode 100755 index 04bcdb82..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/summaries.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import torch -from torchvision.utils import make_grid -# from tensorboardX import SummaryWriter -from torch.utils.tensorboard import SummaryWriter -from dataloaders.utils import decode_seg_map_sequence - -class TensorboardSummary(object): - def __init__(self, directory): - self.directory = directory - - def create_summary(self): - writer = SummaryWriter(log_dir=os.path.join(self.directory)) - return writer - - def visualize_image(self, writer, dataset, image, target, output, global_step, depth=None): - if depth is None: - grid_image = make_grid(image[:3].clone().cpu().data, 3, normalize=True) - writer.add_image('Image', grid_image, global_step) - - grid_image = make_grid(decode_seg_map_sequence(torch.max(output[:3], 1)[1].detach().cpu().numpy(), - dataset=dataset), 3, normalize=False, range=(0, 255)) - writer.add_image('Predicted label', grid_image, global_step) - grid_image = make_grid(decode_seg_map_sequence(torch.squeeze(target[:3], 1).detach().cpu().numpy(), - dataset=dataset), 3, normalize=False, range=(0, 255)) - writer.add_image('Groundtruth label', grid_image, global_step) - else: - grid_image = make_grid(image[:3].clone().cpu().data, 4, normalize=True) - writer.add_image('Image', grid_image, global_step) - - grid_image = make_grid(depth[:3].clone().cpu().data, 4, normalize=True) # normalize=False? - writer.add_image('Depth', grid_image, global_step) - - grid_image = make_grid(decode_seg_map_sequence(torch.max(output[:3], 1)[1].detach().cpu().numpy(), - dataset=dataset), 4, normalize=False, range=(0, 255)) - writer.add_image('Predicted label', grid_image, global_step) - grid_image = make_grid(decode_seg_map_sequence(torch.squeeze(target[:3], 1).detach().cpu().numpy(), - dataset=dataset), 4, normalize=False, range=(0, 255)) - writer.add_image('Groundtruth label', grid_image, global_step) \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py deleted file mode 100755 index 5563a82c..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py +++ /dev/null @@ -1,200 +0,0 @@ -import os -import gc -import numpy as np -import torch -from torch.utils.data import DataLoader -from sedna.common.class_factory import ClassType, ClassFactory -from sedna.common.config import Context -from sedna.common.file_ops import FileOps -from sedna.common.log import LOGGER -from PIL import Image -from torchvision import transforms - -from RFNet.train import Trainer -from RFNet.eval import Validator, load_my_state_dict -from RFNet.dataloaders import custom_transforms as tr -from RFNet.dataloaders import make_data_loader -from RFNet.utils.args import TrainArgs, ValArgs - -# set backend -os.environ['BACKEND_TYPE'] = 'PYTORCH' - -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" - -@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") -class BaseModel: - def __init__(self, **kwargs): - self.train_args = TrainArgs(**kwargs) - self.trainer = None - - self.val_args = ValArgs(**kwargs) - label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") - self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") - self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") - self.val_args.label_save_path = os.path.join(label_save_dir, "label") - self.validator = Validator(self.val_args) - - def get_weights(self): - return self.trainer.get_weight() - - def set_weights(self, weights): - self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close - - def train(self, train_data, valid_data=None, **kwargs): - self.trainer = Trainer(self.train_args, train_data=train_data) - print("Total epoches:", self.trainer.args.epochs) - loss_all = [] - for epoch in range( - self.trainer.args.start_epoch, - self.trainer.args.epochs): - if epoch == 0 and self.trainer.val_loader: - self.trainer.validation(epoch) - loss = self.trainer.training(epoch) - loss_all.append(loss) - if self.trainer.args.no_val and ( - epoch % - self.trainer.args.eval_interval == ( - self.trainer.args.eval_interval - - 1) or epoch == self.trainer.args.epochs - - 1): - # save checkpoint when it meets eval_interval or the training - # finished - is_best = False - self.train_model_url = self.trainer.saver.save_checkpoint({ - 'epoch': epoch + 1, - 'state_dict': self.trainer.model.state_dict(), - 'optimizer': self.trainer.optimizer.state_dict(), - 'best_pred': self.trainer.best_pred, - }, is_best) - - self.trainer.writer.close() - return self.train_model_url - - def predict(self, data, **kwargs): - if len(data) > 10: - print("predict start for big data") - my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} - _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) - else: - print("predict start for small data") - if not isinstance(data[0][0], dict): - data = self._preprocess(data) - #print("predict starting 69") - if type(data) is np.ndarray: - data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") - self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, - pin_memory=True) - - #print("predict starting 75") - predictions, scores = self.validator.validate() - return predictions - - def predict_cloud(self, data, **kwargs): - if len(data) > 10: - print("predict start for big data") - my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} - _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) - else: - print("predict start for small data") - if not isinstance(data[0][0], dict): - data = self._preprocess(data) - #print("predict starting 69") - if type(data) is np.ndarray: - data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") - self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, - pin_memory=True) - - #print("predict starting 75") - predictions, scores = self.validator.validate() - return predictions - - def predict_score(self, data, **kwargs): - if len(data) > 10: - print("predict start for big data") - my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} - _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) - else: - print("predict start for small data") - if not isinstance(data[0][0], dict): - data = self._preprocess(data) - #print("predict starting 69") - if type(data) is np.ndarray: - data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") - self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, - pin_memory=True) - - #print("predict starting 75") - predictions, scores = self.validator.validate() - return scores - - def evaluate(self, data, **kwargs): - #print("evaluate starting 77") - self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) - samples = self._preprocess(data.x) - predictions = self.predict(samples) - metric_name, metric_func = kwargs.get("metric") - if callable(metric_func): - return metric_func(data.y, predictions) - else: - raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") - - def load(self, model_url, **kwargs): - if model_url: - print("load model url: ",model_url) - self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) - self.train_args.resume = model_url - else: - raise Exception("model url does not exist.") - self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) - - def save(self, model_path=None): - # TODO: save unstructured data model - if not model_path: - LOGGER.warning(f"Not specify model path.") - return self.train_model_url - - return FileOps.upload(self.train_model_url, model_path) - - def _preprocess(self, image_urls): - transformed_images = [] - for paths in image_urls: - if len(paths) == 2: - img_path, depth_path = paths - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(depth_path) - #print(img_path, depth_path) - else: - img_path = paths[0] - #print(img_path) - _img = Image.open(img_path).convert('RGB') - _depth = _img - - sample = {'image': _img, 'depth': _depth, 'label': _img} - del _img - gc.collect() - composed_transforms = transforms.Compose([ - # tr.CropBlackArea(), - # tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - transformed_images.append((composed_transforms(sample), img_path)) - - return transformed_images diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py deleted file mode 100755 index 37e3fbd9..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py +++ /dev/null @@ -1,164 +0,0 @@ -import os -import gc -import numpy as np -import torch -from torch.utils.data import DataLoader -from sedna.common.class_factory import ClassType, ClassFactory -from sedna.common.config import Context -from sedna.common.file_ops import FileOps -from sedna.common.log import LOGGER -from PIL import Image -from torchvision import transforms - -from RFNet.train import Trainer -from RFNet.eval import Validator, load_my_state_dict -from RFNet.dataloaders import custom_transforms as tr -from RFNet.dataloaders import make_data_loader -from RFNet.utils.args import TrainArgs, ValArgs - -# set backend -os.environ['BACKEND_TYPE'] = 'PYTORCH' - - -@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") -class BaseModel: - def __init__(self, **kwargs): - self.train_args = TrainArgs(**kwargs) - self.trainer = None - - self.val_args = ValArgs(**kwargs) - label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") - self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") - self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") - self.val_args.label_save_path = os.path.join(label_save_dir, "label") - self.validator = Validator(self.val_args) - - def get_weights(self): - return self.trainer.get_weight() - - def set_weights(self, weights): - self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - #train_loss = self.trainer.training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close - - def train(self, train_data, valid_data=None, **kwargs): - self.trainer = Trainer(self.train_args, train_data=train_data) - print("Total epoches:", self.trainer.args.epochs) - loss_all = [] - for epoch in range( - self.trainer.args.start_epoch, - self.trainer.args.epochs): - if epoch == 0 and self.trainer.val_loader: - self.trainer.validation(epoch) - if self.train_args.resume: - print("my_training:", self.train_args.resume) - loss = self.trainer.my_training(epoch) - #loss = self.trainer.training(epoch) - else: - loss = self.trainer.training(epoch) - loss_all.append(loss) - if self.trainer.args.no_val and ( - epoch % - self.trainer.args.eval_interval == ( - self.trainer.args.eval_interval - - 1) or epoch == self.trainer.args.epochs - - 1): - # save checkpoint when it meets eval_interval or the training - # finished - is_best = False - self.train_model_url = self.trainer.saver.save_checkpoint({ - 'epoch': epoch + 1, - 'state_dict': self.trainer.model.state_dict(), - 'optimizer': self.trainer.optimizer.state_dict(), - 'best_pred': self.trainer.best_pred, - }, is_best) - - self.trainer.writer.close() - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close - return self.train_model_url - - def predict(self, data, **kwargs): - if len(data) > 10: - print("predict start for big data") - my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} - _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) - else: - print("predict start for small data") - if not isinstance(data[0][0], dict): - data = self._preprocess(data) - #print("predict starting 69") - if type(data) is np.ndarray: - data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") - self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, - pin_memory=True) - - #print("predict starting 75") - return self.validator.validate() - - def evaluate(self, data, **kwargs): - print("evaluate starting 77") - self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) - samples = self._preprocess(data.x) - predictions = self.predict(samples) - metric_name, metric_func = kwargs.get("metric") - if callable(metric_func): - return metric_func(data.y, predictions) - else: - raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") - - def load(self, model_url, **kwargs): - if model_url: - print("load model url: ",model_url) - self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) - self.train_args.resume = model_url - else: - raise Exception("model url does not exist.") - self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) - - def save(self, model_path=None): - # TODO: save unstructured data model - if not model_path: - LOGGER.warning(f"Not specify model path.") - return self.train_model_url - - return FileOps.upload(self.train_model_url, model_path) - - def _preprocess(self, image_urls): - transformed_images = [] - for paths in image_urls: - if len(paths) == 2: - img_path, depth_path = paths - _img = Image.open(img_path).convert('RGB') - _depth = Image.open(depth_path) - #print(img_path, depth_path) - else: - img_path = paths[0] - #print(img_path) - _img = Image.open(img_path).convert('RGB') - _depth = _img - - sample = {'image': _img, 'depth': _depth, 'label': _img} - del _img - gc.collect() - composed_transforms = transforms.Compose([ - # tr.CropBlackArea(), - # tr.FixedResize(size=self.args.crop_size), - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), - tr.ToTensor()]) - - transformed_images.append((composed_transforms(sample), img_path)) - - return transformed_images diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm-simple.yaml b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm-simple.yaml deleted file mode 100755 index 6eaafad3..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm-simple.yaml +++ /dev/null @@ -1,63 +0,0 @@ -algorithm: - # paradigm type; string type; - # currently the options of value are as follows: - # 1> "singletasklearning" - # 2> "incrementallearning" - # 3> "lifelonglearning" - paradigm_type: "lifelonglearning" - lifelong_learning_data_setting: - # ratio of training dataset; float type; - # the default value is 0.8. - train_ratio: 0.9 - # the method of splitting dataset; string type; optional; - # currently the options of value are as follows: - # 1> "default": the dataset is evenly divided based train_ratio; - #splitting_method: "default" - splitting_method: "fwt_splitting" - - # algorithm module configuration in the paradigm; list type; - modules: - # type of algorithm module; string type; - # currently the options of value are as follows: - # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; - - type: "basemodel" - # name of python module; string type; - # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; - name: "BaseModel" - # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py" - # hyperparameters configuration for the python module; list type; - hyperparameters: - # name of the hyperparameter; string type; - - learning_rate: - values: - - 0.0001 - - epochs: - values: - - 1 - # 2> "task_definition": define lifelong task ; optional module; - - type: "task_definition" - # name of python module; string type; - name: "TaskDefinitionByOrigin" - # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" - # hyperparameters configuration for the python module; list type; - hyperparameters: - # name of the hyperparameter; string type; - # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - - origins: - values: - - [ "front", "garden" ] - # 3> "task_allocation": allocate lifelong task ; optional module; - - type: "task_allocation" - # name of python module; string type; - name: "TaskAllocationByOrigin" - # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" - # hyperparameters configuration for the python module; list type; - hyperparameters: - # name of the hyperparameter; string type; - # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - - origins: - values: - - [ "front", "garden" ] \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml deleted file mode 100755 index 8777c34b..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml +++ /dev/null @@ -1,74 +0,0 @@ -algorithm: - # paradigm type; string type; - # currently the options of value are as follows: - # 1> "singletasklearning" - # 2> "incrementallearning" - # 3> "lifelonglearning" - paradigm_type: "lifelonglearning" - lifelong_learning_data_setting: - # ratio of training dataset; float type; - # the default value is 0.8. - train_ratio: 0.9 - # the method of splitting dataset; string type; optional; - # currently the options of value are as follows: - # 1> "default": the dataset is evenly divided based train_ratio; - #splitting_method: "default" - #splitting_method: "fwt_splitting" - splitting_method: "hard-example_splitting" - - # algorithm module configuration in the paradigm; list type; - modules: - # type of algorithm module; string type; - # currently the options of value are as follows: - # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; - - type: "basemodel" - # name of python module; string type; - # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; - name: "BaseModel" - # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py" - # hyperparameters configuration for the python module; list type; - hyperparameters: - # name of the hyperparameter; string type; - - learning_rate: - values: - - 0.0001 - - epochs: - values: - - 10 - # 2> "task_definition": define lifelong task ; optional module; - - type: "task_definition" - # name of python module; string type; - name: "TaskDefinitionByOrigin" - # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" - # hyperparameters configuration for the python module; list type; - hyperparameters: - # name of the hyperparameter; string type; - # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - - origins: - values: - - [ "front", "garden" ] - # 3> "task_allocation": allocate lifelong task ; optional module; - - type: "task_allocation" - # name of python module; string type; - name: "TaskAllocationByOrigin" - # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" - # hyperparameters configuration for the python module; list type; - hyperparameters: - # name of the hyperparameter; string type; - # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - - origins: - values: - - [ "front", "garden" ] - - - type: "unseen_sample_recognition" - # name of python module; string type; - name: "HardSampleMining" - # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" - hyperparameters: - - threhold: - values: - - 0.95 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py deleted file mode 100755 index 4fa0f136..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py +++ /dev/null @@ -1,49 +0,0 @@ -from sedna.datasources import BaseDataSource -from sedna.common.class_factory import ClassFactory, ClassType - -__all__ = ('TaskAllocationByOrigin',) - - -@ClassFactory.register(ClassType.STP, alias="TaskAllocationByOrigin") -class TaskAllocationByOrigin: - """ - Corresponding to `TaskDefinitionByOrigin` - - Parameters - ---------- - task_extractor : Dict - used to match target tasks - origins: List[Metadata] - metadata is usually a class feature - label with a finite values. - """ - - def __init__(self, **kwargs): - self.default_origin = kwargs.get("default", None) - - def __call__(self, task_extractor, samples: BaseDataSource): - #self.task_extractor = task_extractor - self.task_extractor = {"front": 0, "garden": 1} - if self.default_origin: - return samples, [int(self.task_extractor.get( - self.default_origin))] * len(samples.x) - - cities = [ - "front" - ] - - sample_origins = [] - for _x in samples.x: - is_real = False - for city in cities: - if city in _x[0]: - is_real = True - sample_origins.append("front") - break - if not is_real: - sample_origins.append("garden") - - allocations = [int(self.task_extractor.get(sample_origin)) - for sample_origin in sample_origins] - - return samples, allocations diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py deleted file mode 100755 index 9ce030b1..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py +++ /dev/null @@ -1,49 +0,0 @@ -from sedna.datasources import BaseDataSource -from sedna.common.class_factory import ClassFactory, ClassType - -__all__ = ('TaskAllocationByOrigin',) - - -@ClassFactory.register(ClassType.STP, alias="TaskAllocationByOrigin") -class TaskAllocationByOrigin: - """ - Corresponding to `TaskDefinitionByOrigin` - - Parameters - ---------- - task_extractor : Dict - used to match target tasks - origins: List[Metadata] - metadata is usually a class feature - label with a finite values. - """ - - def __init__(self, **kwargs): - self.default_origin = kwargs.get("default", None) - - def __call__(self, task_extractor, samples: BaseDataSource): - self.task_extractor = task_extractor - if self.default_origin: - return samples, [int(self.task_extractor.get( - self.default_origin))] * len(samples.x) - - cities = [ - "front", - "garden" - ] - - sample_origins = [] - for _x in samples.x: - is_real = False - for city in cities: - if city in _x[0]: - is_real = True - sample_origins.append("real") - break - if not is_real: - sample_origins.append("sim") - - allocations = [int(self.task_extractor.get(sample_origin)) - for sample_origin in sample_origins] - - return samples, allocations diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py deleted file mode 100755 index 8aaa4c35..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py +++ /dev/null @@ -1,64 +0,0 @@ -from typing import List, Any, Tuple - -from sedna.datasources import BaseDataSource -from sedna.common.class_factory import ClassType, ClassFactory -from sedna.algorithms.seen_task_learning.artifact import Task - -__all__ = ('TaskDefinitionByOrigin',) - - -@ClassFactory.register(ClassType.STP, alias="TaskDefinitionByOrigin") -class TaskDefinitionByOrigin: - """ - Dividing datasets based on the their origins. - - Parameters - ---------- - origins： List[Metadata] - metadata is usually a class feature label with a finite values. - """ - - def __init__(self, **kwargs): - self.origins = kwargs.get("origins", ["front", "garden"]) - - def __call__(self, - samples: BaseDataSource, **kwargs) -> Tuple[List[Task], - Any, - BaseDataSource]: - cities = [ - "front" - ] - - tasks = [] - d_type = samples.data_type - x_data = samples.x - y_data = samples.y - - task_index = dict(zip(self.origins, range(len(self.origins)))) - - real_df = BaseDataSource(data_type=d_type) - real_df.x, real_df.y = [], [] - sim_df = BaseDataSource(data_type=d_type) - sim_df.x, sim_df.y = [], [] - - for i in range(samples.num_examples()): - is_real = False - for city in cities: - if city in x_data[i][0]: - is_real = True - real_df.x.append(x_data[i]) - real_df.y.append(y_data[i]) - break - if not is_real: - sim_df.x.append(x_data[i]) - sim_df.y.append(y_data[i]) - - g_attr = "front_semantic_segamentation_model" - task_obj = Task(entry=g_attr, samples=real_df, meta_attr="front") - tasks.append(task_obj) - - g_attr = "garden_semantic_segamentation_model" - task_obj = Task(entry=g_attr, samples=sim_df, meta_attr="garden") - tasks.append(task_obj) - - return tasks, task_index, samples diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py deleted file mode 100755 index 6c59cecb..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import List, Any, Tuple - -from sedna.datasources import BaseDataSource -from sedna.common.class_factory import ClassType, ClassFactory -from sedna.algorithms.seen_task_learning.artifact import Task - -__all__ = ('TaskDefinitionByOrigin',) - - -@ClassFactory.register(ClassType.STP, alias="TaskDefinitionByOrigin") -class TaskDefinitionByOrigin: - """ - Dividing datasets based on the their origins. - - Parameters - ---------- - origins： List[Metadata] - metadata is usually a class feature label with a finite values. - """ - - def __init__(self, **kwargs): - self.origins = kwargs.get("origins", ["real", "sim"]) - - def __call__(self, - samples: BaseDataSource, **kwargs) -> Tuple[List[Task], - Any, - BaseDataSource]: - cities = [ - "front", - "garden" - ] - - tasks = [] - d_type = samples.data_type - x_data = samples.x - y_data = samples.y - - task_index = dict(zip(self.origins, range(len(self.origins)))) - - real_df = BaseDataSource(data_type=d_type) - real_df.x, real_df.y = [], [] - sim_df = BaseDataSource(data_type=d_type) - sim_df.x, sim_df.y = [], [] - - for i in range(samples.num_examples()): - is_real = False - for city in cities: - if city in x_data[i][0]: - is_real = True - real_df.x.append(x_data[i]) - real_df.y.append(y_data[i]) - break - if not is_real: - sim_df.x.append(x_data[i]) - sim_df.y.append(y_data[i]) - - g_attr = "real_semantic_segamentation_model" - task_obj = Task(entry=g_attr, samples=real_df, meta_attr="real") - tasks.append(task_obj) - - g_attr = "sim_semantic_segamentation_model" - task_obj = Task(entry=g_attr, samples=sim_df, meta_attr="sim") - tasks.append(task_obj) - - return tasks, task_index, samples diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/accuracy.py b/examples/robot-sam/lifelong_learning_bench/testenv/accuracy.py deleted file mode 100755 index 7e6fcb2a..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testenv/accuracy.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2022 The KubeEdge Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tqdm import tqdm - -from sedna.common.class_factory import ClassType, ClassFactory - -from RFNet.dataloaders import make_data_loader -from RFNet.utils.metrics import Evaluator -from RFNet.utils.args import ValArgs - -__all__ = ('accuracy') - - -@ClassFactory.register(ClassType.GENERAL, alias="accuracy") -def accuracy(y_true, y_pred, **kwargs): - args = ValArgs() - _, _, test_loader, num_class = make_data_loader(args, test_data=y_true) - evaluator = Evaluator(num_class) - #print(y_true) - tbar = tqdm(test_loader, desc='\r') - for i, (sample, img_path) in enumerate(tbar): - if args.depth: - image, depth, target = sample['image'], sample['depth'], sample['label'] - else: - image, target = sample['image'], sample['label'] - if args.cuda: - image, target = image.cuda(args.gpu_ids), target.cuda(args.gpu_ids) - if args.depth: - depth = depth.cuda(args.gpu_ids) - - target[target > evaluator.num_class-1] = 255 - target = target.cpu().numpy() - #if i% 20 == 0: - # print(img_path) - # print(image, target, y_pred[i]) - # Add batch sample into evaluator - evaluator.add_batch(target, y_pred[i]) - - # Test during the training - # Acc = evaluator.Pixel_Accuracy() - CPA = evaluator.Pixel_Accuracy_Class() - mIoU = evaluator.Mean_Intersection_over_Union() - FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union() - - print("CPA:{}, mIoU:{}, fwIoU: {}".format(CPA, mIoU, FWIoU)) - return mIoU diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-all.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-all.yaml deleted file mode 100755 index 2a7021f0..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-all.yaml +++ /dev/null @@ -1,35 +0,0 @@ -testenv: - # dataset configuration - dataset: - # the url address of train dataset index; string type; - train_url: "/home/shijing.hu/ianvs/dataset/all-train-index.txt" - # the url address of test dataset index; string type; - test_url: "/home/shijing.hu/ianvs/dataset/all-test-index.txt" - - # model eval configuration of incremental learning; - model_eval: - # metric used for model evaluation - model_metric: - # metric name; string type; - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" - - # condition of triggering inference model to update - # threshold of the condition; types are float/int - threshold: 0 - # operator of the condition; string type; - # values are ">=", ">", "<=", "<" and "="; - operator: "<" - - # metrics configuration for test case's evaluation; list type; - metrics: - # metric name; string type; - - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - - name: "samples_transfer_ratio" - - # incremental rounds setting; int type; default value is 2; - incremental_rounds: 5 \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-city.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-city.yaml deleted file mode 100755 index 7c118f0f..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-city.yaml +++ /dev/null @@ -1,35 +0,0 @@ -testenv: - # dataset configuration - dataset: - # the url address of train dataset index; string type; - train_url: "/home/shijing.hu/ianvs/dataset/semantic_segmentation_dataset/city-train-index-2.txt" - # the url address of test dataset index; string type; - test_url: "/home/shijing.hu/ianvs/dataset/semantic_segmentation_dataset/city-test-index-2.txt" - - # model eval configuration of incremental learning; - model_eval: - # metric used for model evaluation - model_metric: - # metric name; string type; - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" - - # condition of triggering inference model to update - # threshold of the condition; types are float/int - threshold: 0 - # operator of the condition; string type; - # values are ">=", ">", "<=", "<" and "="; - operator: "<" - - # metrics configuration for test case's evaluation; list type; - metrics: - # metric name; string type; - - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - - name: "samples_transfer_ratio" - - # incremental rounds setting; int type; default value is 2; - incremental_rounds: 5 \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml deleted file mode 100755 index cde00454..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml +++ /dev/null @@ -1,40 +0,0 @@ -testenv: - # dataset configuration - dataset: - # the url address of train dataset index; string type; - train_url: "/data/datasets/robot_dataset/train-index-small.txt" - # the url address of test dataset index; string type; - test_url: "/data/datasets/robot_dataset/test-index-small.txt" - - # model eval configuration of incremental learning; - model_eval: - # metric used for model evaluation - model_metric: - # metric name; string type; - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - #mode: "no-inference" - mode: "hard-example-mining" - - # condition of triggering inference model to update - # threshold of the condition; types are float/int - threshold: 0 - # operator of the condition; string type; - # values are ">=", ">", "<=", "<" and "="; - operator: "<" - - # metrics configuration for test case's evaluation; list type; - metrics: - # metric name; string type; - - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - - name: "samples_transfer_ratio" - - name: "Task_Avg_Acc" - - name: "BWT" - - name: "FWT" - - name: "Matrix" - - # incremental rounds setting; int type; default value is 2; - incremental_rounds: 3 \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml deleted file mode 100755 index e8645110..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml +++ /dev/null @@ -1,40 +0,0 @@ -testenv: - # dataset configuration - dataset: - # the url address of train dataset index; string type; - train_url: "/data/datasets/robot_dataset/train-index-mix.txt" - # the url address of test dataset index; string type; - test_url: "/data/datasets/robot_dataset/test-index.txt" - - # model eval configuration of incremental learning; - model_eval: - # metric used for model evaluation - model_metric: - # metric name; string type; - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - #mode: "no-inference" - mode: "hard-example-mining" - - # condition of triggering inference model to update - # threshold of the condition; types are float/int - threshold: 0 - # operator of the condition; string type; - # values are ">=", ">", "<=", "<" and "="; - operator: "<" - - # metrics configuration for test case's evaluation; list type; - metrics: - # metric name; string type; - - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - - name: "samples_transfer_ratio" - - name: "Task_Avg_Acc" - - name: "BWT" - - name: "FWT" - - name: "Matrix" - - # incremental rounds setting; int type; default value is 2; - incremental_rounds: 5 \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv.yaml deleted file mode 100755 index a6151b99..00000000 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv.yaml +++ /dev/null @@ -1,35 +0,0 @@ -testenv: - # dataset configuration - dataset: - # the url address of train dataset index; string type; - train_url: "/home/shijing.hu/ianvs/dataset/robot_dataset/train-index.txt" - # the url address of test dataset index; string type; - test_url: "/home/shijing.hu/ianvs/dataset/robot_dataset/test-index.txt" - - # model eval configuration of incremental learning; - model_eval: - # metric used for model evaluation - model_metric: - # metric name; string type; - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" - - # condition of triggering inference model to update - # threshold of the condition; types are float/int - threshold: 0 - # operator of the condition; string type; - # values are ">=", ">", "<=", "<" and "="; - operator: "<" - - # metrics configuration for test case's evaluation; list type; - metrics: - # metric name; string type; - - name: "accuracy" - # the url address of python file - url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - - name: "samples_transfer_ratio" - - # incremental rounds setting; int type; default value is 2; - incremental_rounds: 5 \ No newline at end of file diff --git a/examples/robot/lifelong_learning_bench/README.md b/examples/robot/lifelong_learning_bench/README.md index c113c14d..f8e9d6c2 100755 --- a/examples/robot/lifelong_learning_bench/README.md +++ b/examples/robot/lifelong_learning_bench/README.md @@ -1,27 +1,26 @@ # Quick Start -Welcome to Ianvs! Ianvs aims to test the performance of distributed synergy AI solutions following recognized standards, -in order to facilitate more efficient and effective development. Quick start helps you to test your algorithm on Ianvs -with a simple example of industrial defect detection. You can reduce manual procedures to just a few steps so that you can -build and start your distributed synergy AI solution development within minutes. +Welcome to Ianvs! Ianvs aims to test the performance of distributed synergy AI solutions following recognized standards, +in order to facilitate more efficient and effective development. Quick start helps you to test your algorithm on Ianvs +with a simple example of semantic segmentation based on lifelong learning. You can reduce manual procedures to just a few steps so that you can build and start your distributed synergy AI solution development within minutes. + +Before using Ianvs, you might want to have the device ready: -Before using Ianvs, you might want to have the device ready: - One machine is all you need, i.e., a laptop or a virtual machine is sufficient and a cluster is not necessary - 2 CPUs or more - 4GB+ free memory, depends on algorithm and simulation setting - 10GB+ free disk space - Internet connection for GitHub and pip, etc - Python 3.6+ installed - -In this example, we are using the Linux platform with Python 3.8.5. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. +In this example, we are using the Linux platform with Python 3.9. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. ## Step 1. Ianvs Preparation First, we download the code of Ianvs. Assuming that we are using `/ianvs` as workspace, Ianvs can be cloned with `Git` as: -``` shell +```shell mkdir /ianvs cd /ianvs #One might use another path preferred @@ -30,9 +29,9 @@ cd project git clone https://github.com/kubeedge/ianvs.git ``` +Then, we install third-party dependencies for ianvs. -Then, we install third-party dependencies for ianvs. -``` shell +```shell sudo apt-get update sudo apt-get install libgl1-mesa-glx -y python -m pip install --upgrade pip @@ -42,8 +41,9 @@ python -m pip install ./examples/resources/third_party/* python -m pip install -r requirements.txt ``` -We are now ready to install Ianvs. -``` shell +We are now ready to install Ianvs. + +```shell python setup.py install ``` @@ -52,14 +52,12 @@ python setup.py install Datasets and models can be large. To avoid over-size projects in the Github repository of Ianvs, the Ianvs code base does not include origin datasets. Then developers do not need to download non-necessary datasets for a quick start. -``` shell +```shell mkdir /data cd /data mkdir datasets cd datasets -python -m pip install kaggle -kaggle datasets download -d hsj576/cloud-robotics -unzip cloud-robotics.zip +download datasets in https://kubeedge-ianvs.github.io/download.html ``` The URL address of this dataset then should be filled in the configuration file ``testenv.yaml``. In this quick start, @@ -67,22 +65,70 @@ we have done that for you and the interested readers can refer to [testenv.yaml] +Related algorithm is also ready in this quick start. -Related algorithm is also ready in this quick start. -``` shell +```shell export PYTHONPATH=$PYTHONPATH:/ianvs/project/ianvs/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet ``` The URL address of this algorithm then should be filled in the configuration file ``algorithm.yaml``. In this quick start, we have done that for you and the interested readers can refer to [algorithm.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. +## Step 2.5. Large Vision Model Preparation(Optional) + +If you want to run the large vision model based cloud-edge collaboration process, then you need to follow the steps below to install the large vision model additionally. If you only want to run the basic lifelong learning process, you can ignore the steps below. + +In this example, we use [SAM model](https://segment-anything.com/) as the cloud large vision model. So, we need to install SAM by the following instructions: + +```bash +cd /ianvs/project +git clone https://github.com/facebookresearch/segment-anything.git +cd segment-anything +python -m pip install -e . +``` + +Then, we need to download the pretrained SAM model: + +```bash +wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth +``` + +In order to save the inference result, we need to install mmcv and mmdetection by the following instructions: + +```bash +python -m pip install https://download.openmmlab.com/mmcv/dist/cu118/torch2.0.0/mmcv-2.0.0-cp39-cp39-manylinux1_x86_64.whl +cd /ianvs/project +git clone https://github.com/hsj576/mmdetection.git +cd mmdetection +python -m pip install -v -e . +``` + +In case that your computer couldn't run SAM model, we prepare a cache for all the SAM inference results in Cloud-Robotics dataset. You could download the cache from [this link](https://pan.baidu.com/s/1oGGBa8TjZn0ccbznQsl48g?pwd=wpp1) and put the cache file in "/ianvs/project/": + +```bash +cp cache.pickle /ianvs/project +``` + + By using the cache, you could simulate the edge-cloud joint inference without installing SAM model. + +Besides that, we also provided you a pretrained RFNet model in [this link](https://pan.baidu.com/s/1h8JnUgr1hfx5QnaFLLkMAg?pwd=jts4), you could use it if you don't want to train the RFNet model from zero. This instruction is optional: + +```bash +cd /ianvs/project +mkdir pretrain +cp pretrain_model.pth /ianvs/project/pretrain +in /ianvs/project/ianvs/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py set self.resume = '/ianvs/project/pretrain/pretrain_model.pth' +``` + ## Step 3. Ianvs Execution and Presentation -We are now ready to run the ianvs for benchmarking. +We are now ready to run the ianvs for benchmarking. + +To run the basic lifelong learning process: -``` shell +```shell cd /ianvs/project/ianvs -ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob.yaml +ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml ``` Finally, the user can check the result of benchmarking on the console and also in the output path( @@ -90,19 +136,34 @@ e.g. `/ianvs/lifelong_learning_bench/workspace`) defined in the benchmarking con e.g. `benchmarkingjob.yaml`). In this quick start, we have done all configurations for you and the interested readers can refer to [benchmarkingJob.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. -The final output might look like this: +The final output might look like this: + +| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | url | +| :--: | :---------------------: | :----------------: | :-----------------: | :-----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :-----------------: | :-----------------------------------------------------------------------------------------------------------------------------: | +| 1 | rfnet_lifelong_learning | 0.2970033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | /ianvs/lifelong_learning_bench/robot-workspace-bwt/benchmarkingjob/rfnet_lifelong_learning/efdc47a2-f9fb-11ed-8f8b-0242ac110007 | +To run the large vision model based cloud-edge collaboration process: -| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | url | -|:------:|:-------------------------:|:--------------------:|:---------------------:|:---------------------:|:------------------:|:-----------:|:------------------------:|:------------------------:|:-------------------------:|:------------------:|:-------------------------:|:-------------------------:|:---------------------:|:---------------------------------------------------------------------------------------------------------------------------------:| -| 1 | rfnet_lifelong_learning | 0.2970033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | /ianvs/lifelong_learning_bench/robot-workspace-bwt/benchmarkingjob/rfnet_lifelong_learning/efdc47a2-f9fb-11ed-8f8b-0242ac110007 | +```shell +cd /ianvs/project/ianvs +ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml +``` + +Finally, the user can check the result of benchmarking on the console and also in the output path( +e.g. `/ianvs/lifelong_learning_bench/workspace`) defined in the benchmarking config file ( +e.g. `benchmarkingjob.yaml`). In this quick start, we have done all configurations for you and the interested readers +can refer to [benchmarkingJob.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. +The final output might look like this: +| rank | algorithm | accuracy | Task_Avg_Acc | paradigm | basemodel | task_definition | task_allocation | unseen_sample_recognition | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | unseen_sample_recognition-threhold | time | url | +| :--: | :-------------------------: | :----------------: | :----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :-----------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :--------------------------------: | ------------------- | :---------------------------------------------------------------------------------------------------------------: | +| 1 | sam_rfnet_lifelong_learning | 0.7052917006987501 | 0.6258875117354328 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | HardSampleMining | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 0.95 | 2023-08-24 12:43:19 | /ianvs/sam_bench/robot-workspace/benchmarkingjob/sam_rfnet_lifelong_learning/9465c47a-4235-11ee-8519-ec2a724ccd3e | This ends the quick start experiment. # What is next -If any problems happen, the user can refer to [the issue page on Github](https://github.com/kubeedge/ianvs/issues) for help and are also welcome to raise any new issue. +If any problems happen, the user can refer to [the issue page on Github](https://github.com/kubeedge/ianvs/issues) for help and are also welcome to raise any new issue. -Enjoy your journey on Ianvs! \ No newline at end of file +Enjoy your journey on Ianvs! diff --git a/examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml b/examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml old mode 100755 new mode 100644 similarity index 85% rename from examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml rename to examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml index f352988a..01cb80c2 --- a/examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml +++ b/examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml @@ -7,8 +7,8 @@ benchmarkingjob: # the url address of test environment configuration file; string type; # the file format supports yaml/yml; - testenv: "./examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml" - #testenv: "./examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml" + testenv: "./examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml" + #testenv: "./examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml" # the configuration of test object test_object: @@ -24,10 +24,10 @@ benchmarkingjob: #- name: "sam_vit_lifelong_learning" # the url address of test algorithm configuration file; string type; # the file format supports yaml/yml - #url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml" - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml" - #url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml" - #url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml" + #url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml" + #url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml" + #url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml" # the url address of test algorithm configuration file; string type; # the file format supports yaml/yml diff --git a/examples/robot/lifelong_learning_bench/benchmarkingjob.yaml b/examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml old mode 100755 new mode 100644 similarity index 97% rename from examples/robot/lifelong_learning_bench/benchmarkingjob.yaml rename to examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml index 3e6b5a49..24f795a3 --- a/examples/robot/lifelong_learning_bench/benchmarkingjob.yaml +++ b/examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml @@ -2,7 +2,7 @@ benchmarkingjob: # job name of bechmarking; string type; name: "benchmarkingjob" # the url address of job workspace that will reserve the output of tests; string type; - workspace: "/home/vipuser/ianvs/lifelong_learning_bench/robot-workspace-test" + workspace: "/ianvs/lifelong_learning_bench/robot-workspace-test" # the url address of test environment configuration file; string type; # the file format supports yaml/yml; diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py similarity index 100% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py index 2d303429..da3538b8 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py @@ -7,6 +7,12 @@ from torchvision.transforms import ToPILImage from PIL import Image +from segment_anything import sam_model_registry, SamAutomaticMaskGenerator +import mmcv +import pycocotools.mask as maskUtils +from mmdet.visualization.image import imshow_det_bboxes +import pickle + from dataloaders import make_data_loader from dataloaders.utils import decode_seg_map_sequence, Colorize from utils.metrics import Evaluator @@ -15,6 +21,9 @@ from models.resnet.resnet_single_scale_single_attention import * from models.resnet import resnet_single_scale_single_attention_unseen import torch.backends.cudnn as cudnn +from cityscapes_id2label import CONFIG as CONFIG_CITYSCAPES_ID2LABEL +import torch.nn.functional as F +from transformers import SegformerFeatureExtractor, SegformerForSemanticSegmentation os.environ["OMP_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" @@ -54,12 +63,257 @@ def __init__(self, args, data=None, unseen_detection=False): # self.new_state_dict = torch.load(args.weight_path, map_location=torch.device("cpu")) # self.model = load_my_state_dict(self.model, self.new_state_dict['state_dict']) + def segformer_segmentation(self, image, processor, model, rank): + h, w, _ = image.shape + inputs = processor(images=image, return_tensors="pt").to(rank) + outputs = model(**inputs) + logits = outputs.logits + logits = F.interpolate(logits, size=(h, w), mode='bilinear', align_corners=True) + predicted_semantic_map = logits.argmax(dim=1) + return predicted_semantic_map + + def draw_mask(self, image_name, mask, output_path): + img = mmcv.imread(image_name) + anns = {'annotations': mask} + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + semantc_mask = torch.zeros(1024, 2048) + i = 0 + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + semantc_mask[valid_mask] = i + i += 1 + sematic_class_in_img = torch.unique(semantc_mask) + semantic_bitmasks, semantic_class_names = [], [] + + # semantic prediction + for i in range(len(sematic_class_in_img)): + class_name = str(i) + class_mask = semantc_mask == sematic_class_in_img[i] + class_mask = class_mask.cpu().numpy().astype(np.uint8) + semantic_class_names.append(class_name) + semantic_bitmasks.append(class_mask) + + length = len(image_name) + for i in range(length): + if image_name[length-i-1] == '_': + break + filename = image_name[length-i:] + imshow_det_bboxes(img, + bboxes=None, + labels=np.arange(len(sematic_class_in_img)), + segms=np.stack(semantic_bitmasks), + class_names=semantic_class_names, + font_size=25, + show=False, + out_file=os.path.join(output_path, filename + '_mask.png')) + print('[Save] save mask: ', os.path.join(output_path, filename + '_mask.png')) + semantc_mask = semantc_mask.unsqueeze(0).numpy() + del img + del semantic_bitmasks + del semantic_class_names + + def draw_picture(self, image_name, semantc_mask, id2label, output_path, suffix): + img = mmcv.imread(image_name) + sematic_class_in_img = torch.unique(semantc_mask) + semantic_bitmasks, semantic_class_names = [], [] + + # semantic prediction + for i in range(len(sematic_class_in_img)): + class_name = id2label['id2label'][str(sematic_class_in_img[i].item())] + class_mask = semantc_mask == sematic_class_in_img[i] + class_mask = class_mask.cpu().numpy().astype(np.uint8) + semantic_class_names.append(class_name) + semantic_bitmasks.append(class_mask) + + #print(os.environ["OUTPUT_URL"]) + length = len(image_name) + for i in range(length): + if image_name[length-i-1] == '_': + break + filename = image_name[length-i:] + imshow_det_bboxes(img, + bboxes=None, + labels=np.arange(len(sematic_class_in_img)), + segms=np.stack(semantic_bitmasks), + class_names=semantic_class_names, + font_size=25, + show=False, + out_file=os.path.join(output_path, filename + suffix)) + print('[Save] save rfnet prediction: ', os.path.join(output_path, filename + suffix)) + #semantc_mask = semantc_mask.unsqueeze(0).numpy() + del img + del semantic_bitmasks + del semantic_class_names + + def confidence(self, input_output): + output = torch.softmax(input_output, dim=0) + highth = len(output[0]) + width = len(output[0][0]) + sum_1 = 0.0 + sum_2 = 0.0 + values, _ = torch.topk(output, k=2, dim=0) + sum_1 = torch.sum(values[0]) + value_2 = torch.sub(values[0],values[1]) + sum_2 = torch.sum(value_2) + sum_1 = sum_1/(highth*width) + sum_2 = sum_2/(highth*width) + count = (values[0] > 0.9).sum().item() + sum_3 = count/(highth*width) + return sum_3 + + def sam_predict_ssa(self, image_name, pred): + with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: + cache = pickle.load(file) + img = mmcv.imread(image_name) + if image_name in cache.keys(): + mask = cache[image_name] + print("load cache") + else: + sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') + mask_branch_model = SamAutomaticMaskGenerator( + model=sam, + #points_per_side=64, + # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side + #pred_iou_thresh=0.86, + #stability_score_thresh=0.92, + #crop_n_layers=1, + #crop_n_points_downscale_factor=2, + #min_mask_region_area=100, # Requires open-cv to run post-processing + output_mode='coco_rle', + ) + print('[Model loaded] Mask branch (SAM) is loaded.') + mask = mask_branch_model.generate(img) + cache[image_name] = mask + with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: + pickle.dump(cache, file) + print("save cache") + + anns = {'annotations': mask} + #print(len(anns['annotations']), len(anns['annotations'][0])) + #print(pred.shape) + #print(pred[0]) + class_names = [] + semantc_mask = pred.clone() + id2label = CONFIG_CITYSCAPES_ID2LABEL + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + #print(valid_mask) + propose_classes_ids = pred[valid_mask] + num_class_proposals = len(torch.unique(propose_classes_ids)) + if num_class_proposals == 1: + semantc_mask[valid_mask] = propose_classes_ids[0] + ann['class_name'] = id2label['id2label'][str(propose_classes_ids[0].item())] + ann['class_proposals'] = id2label['id2label'][str(propose_classes_ids[0].item())] + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + continue + top_1_propose_class_ids = torch.bincount(propose_classes_ids.flatten()).topk(1).indices + top_1_propose_class_names = [id2label['id2label'][str(class_id.item())] for class_id in top_1_propose_class_ids] + #print("top_1_propose_class_ids: ", top_1_propose_class_ids) + semantc_mask[valid_mask] = top_1_propose_class_ids + ann['class_name'] = top_1_propose_class_names[0] + ann['class_proposals'] = top_1_propose_class_names[0] + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + + del valid_mask + del propose_classes_ids + del num_class_proposals + del top_1_propose_class_ids + del top_1_propose_class_names + + #print(semantc_mask.shape) + #print(semantc_mask) + + del img + del anns + #del semantc_mask + # del bitmasks + del class_names + return semantc_mask, mask + + def sam_predict(self, image_name, pred): + with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: + cache = pickle.load(file) + img = mmcv.imread(image_name) + if image_name in cache.keys(): + mask = cache[image_name] + print("load cache") + else: + sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') + mask_branch_model = SamAutomaticMaskGenerator( + model=sam, + #points_per_side=64, + # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side + #pred_iou_thresh=0.86, + #stability_score_thresh=0.92, + #crop_n_layers=1, + #crop_n_points_downscale_factor=2, + #min_mask_region_area=100, # Requires open-cv to run post-processing + output_mode='coco_rle', + ) + print('[Model loaded] Mask branch (SAM) is loaded.') + mask = mask_branch_model.generate(img) + cache[image_name] = mask + with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: + pickle.dump(cache, file) + print("save cache") + + anns = {'annotations': mask} + #print(len(anns['annotations']), len(anns['annotations'][0])) + #print(pred.shape) + #print(pred[0]) + class_names = [] + pred_2 = np.argmax(pred, axis=0) + semantc_mask = pred_2.clone() + id2label = CONFIG_CITYSCAPES_ID2LABEL + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + #print(valid_mask) + length = len(pred) + all_scores = [0 for i in range(length)] + for i in range(length): + propose_classes_ids = pred[i][valid_mask] + #print(propose_classes_ids.shape) + all_scores[i] = torch.sum(propose_classes_ids) + #print(all_scores[i]) + top_1_propose_class_ids = np.argmax(all_scores) + #print(top_1_propose_class_ids) + top_1_propose_class_names = id2label['id2label'][str(top_1_propose_class_ids)] + + semantc_mask[valid_mask] = top_1_propose_class_ids + ann['class_name'] = top_1_propose_class_names + ann['class_proposals'] = top_1_propose_class_names + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + + del valid_mask + del propose_classes_ids + del top_1_propose_class_ids + del top_1_propose_class_names + + #print(semantc_mask.shape) + #print(semantc_mask) + + #self.draw_picture(img, image_name, pred_2, id2label, output_path, "_origin.png") + #self.draw_picture(img, image_name, semantc_mask, id2label, output_path, "_sam.png") + + del img + del anns + #del semantc_mask + # del bitmasks + del class_names + return semantc_mask, mask + def validate(self): #print("start validating 55") self.model.eval() self.evaluator.reset() tbar = tqdm(self.test_loader, desc='\r') predictions = [] + scores = [] for i, (sample, image_name) in enumerate(tbar):#self.test_loader: if self.args.depth: image, depth, target = sample['image'], sample['depth'], sample['label'] @@ -81,46 +335,131 @@ def validate(self): if self.args.cuda: torch.cuda.synchronize() + if len(output) == 1: + score = self.confidence(output[0]) + else: + score = self.confidence(output) + scores.append(score) + pred = output.data.cpu().numpy() # todo pred = np.argmax(pred, axis=1) predictions.append(pred) - if not self.args.save_predicted_image: - continue + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_origin.png") - pre_colors = Colorize()(torch.max(output, 1)[1].detach().cpu().byte()) - pre_labels = torch.max(output, 1)[1].detach().cpu().byte() - print(pre_labels.shape) - # save - for i in range(pre_colors.shape[0]): - print(image_name[0]) + #print("start validating 120") + return predictions, scores + + def vit_validate(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + rank = 'cuda:0' + semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") + semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + img = mmcv.imread(image_name[0]) + class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) + pred = class_ids.data.cpu().numpy() + predictions.append(pred) + + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") + + #print("start validating 120") + return predictions - if not image_name[0]: - img_name = "test.png" + def validate_cloud(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + scores = [] + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + if self.args.depth: + image, depth, target = sample['image'], sample['depth'], sample['label'] + else: + # spec = time.time() + image, target = sample['image'], sample['label'] + #print(self.args.cuda, self.args.gpu_ids) + if self.args.cuda: + image = image.cuda(self.args.gpu_ids) + if self.args.depth: + depth = depth.cuda(self.args.gpu_ids) + + with torch.no_grad(): + if self.args.depth: + output = self.model(image, depth) else: - img_name = os.path.basename(image_name[0]) + output = self.model(image) + + if self.args.cuda: + torch.cuda.synchronize() - color_label_name = os.path.join(self.args.color_label_save_path, img_name) - label_name = os.path.join(self.args.label_save_path, img_name) - merge_label_name = os.path.join(self.args.merge_label_save_path, img_name) + if len(output) == 1: + score = self.confidence(output[0]) + else: + score = self.confidence(output) + scores.append(score) + + pred = output.data.cpu().numpy() + # todo + pred_sam, mask = self.sam_predict(image_name[0], torch.from_numpy(pred[0])) + if pred_sam.ndim < 3: + h, w = pred_sam.shape + pred_sam = pred_sam.reshape(1, h, w) + #print(pred_sam.shape) - os.makedirs(os.path.dirname(color_label_name), exist_ok=True) - os.makedirs(os.path.dirname(merge_label_name), exist_ok=True) - os.makedirs(os.path.dirname(label_name), exist_ok=True) + predictions.append(np.array(pred_sam)) - pre_color_image = ToPILImage()(pre_colors[i]) # pre_colors.dtype = float64 - pre_color_image.save(color_label_name) + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_sam.png") + self.draw_mask(image_name[0], mask, output_path) + + #print("start validating 120") + return predictions, scores - pre_label_image = ToPILImage()(pre_labels[i]) - pre_label_image.save(label_name) + def vit_validate_cloud(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + rank = 'cuda:0' + semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") + semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + img = mmcv.imread(image_name[0]) + class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) + pred = class_ids.data.cpu().numpy() + pred_sam, mask = self.sam_predict_ssa(image_name[0], torch.from_numpy(pred[0])) + if pred_sam.ndim < 3: + h, w = pred_sam.shape + pred_sam = pred_sam.reshape(1, h, w) + #print(pred_sam.shape) + predictions.append(np.array(pred_sam)) - if (self.args.merge): - image_merge(image[i], pre_color_image, merge_label_name) - print('save image: {}'.format(merge_label_name)) + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") + self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_vit_sam.png") + self.draw_mask(image_name[0], mask, output_path) + #print("start validating 120") return predictions - + def task_divide(self): seen_task_samples, unseen_task_samples = [], [] self.model.eval() diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py index b2e93683..0acef918 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py @@ -188,6 +188,8 @@ def add_batch(self, gt_image, pre_image): gt_image = np.array(gt_image) pre_image = np.array(pre_image) print(gt_image.shape, pre_image.shape) + if gt_image.shape != pre_image.shape: + pre_image = pre_image[0] if gt_image.shape != pre_image.shape: pre_image = pre_image[0] assert gt_image.shape == pre_image.shape diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py old mode 100755 new mode 100644 similarity index 85% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py index 8b0d2c39..056627b8 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py @@ -40,16 +40,6 @@ def get_weights(self): def set_weights(self, weights): self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close def train(self, train_data, valid_data=None, **kwargs): self.trainer = Trainer(self.train_args, train_data=train_data) @@ -82,70 +72,60 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the RFNet model to predict at the edge + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions, scores = self.validator.validate() return predictions def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions = self.validator.validate_cloud() return predictions def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions, scores = self.validator.validate() return scores def evaluate(self, data, **kwargs): - #print("evaluate starting 77") self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) samples = self._preprocess(data.x) predictions = self.predict(samples) diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py index c2ad091e..4684d567 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py @@ -40,16 +40,6 @@ def get_weights(self): def set_weights(self, weights): self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close def train(self, train_data, valid_data=None, **kwargs): self.trainer = Trainer(self.train_args, train_data=train_data) @@ -82,27 +72,60 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the RFNet model to predict at the edge + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return predictions + + def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return predictions + + def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") - return self.validator.validate() + predictions, scores = self.validator.validate() + return scores def evaluate(self, data, **kwargs): - #print("evaluate starting 77") self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) samples = self._preprocess(data.x) predictions = self.predict(samples) diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py old mode 100755 new mode 100644 similarity index 85% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py index bb339951..cfde33e6 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py @@ -40,16 +40,6 @@ def get_weights(self): def set_weights(self, weights): self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close def train(self, train_data, valid_data=None, **kwargs): self.trainer = Trainer(self.train_args, train_data=train_data) @@ -82,70 +72,60 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the Segformer model to predict at the edge + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions = self.validator.vit_validate() return predictions def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions = self.validator.vit_validate_cloud() return predictions def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions, scores = self.validator.validate() return scores def evaluate(self, data, **kwargs): - #print("evaluate starting 77") self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) samples = self._preprocess(data.x) predictions = self.predict(samples) diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py old mode 100755 new mode 100644 similarity index 85% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py index e89200ce..c83b7b1e --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py @@ -40,16 +40,6 @@ def get_weights(self): def set_weights(self, weights): self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close def train(self, train_data, valid_data=None, **kwargs): self.trainer = Trainer(self.train_args, train_data=train_data) @@ -82,70 +72,60 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the Segformer model to predict at the edge + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions = self.validator.vit_validate() return predictions def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions = self.validator.vit_validate() return predictions def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") predictions, scores = self.validator.validate() return scores def evaluate(self, data, **kwargs): - #print("evaluate starting 77") self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) samples = self._preprocess(data.x) predictions = self.predict(samples) diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py index 37e3fbd9..b93e2f48 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py @@ -89,6 +89,9 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the RFNet model to predict at the edge + """ if len(data) > 10: print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py similarity index 100% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml index 628ceb9a..6d7de7f7 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml @@ -12,8 +12,9 @@ algorithm: # the method of splitting dataset; string type; optional; # currently the options of value are as follows: # 1> "default": the dataset is evenly divided based train_ratio; - splitting_method: "default" - #splitting_method: "my_splitting" + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" # algorithm module configuration in the paradigm; list type; modules: @@ -25,7 +26,7 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "BaseModel" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -34,30 +35,40 @@ algorithm: - 0.0001 - epochs: values: - - 1 + - 10 # 2> "task_definition": define lifelong task ; optional module; - type: "task_definition" # name of python module; string type; name: "TaskDefinitionByOrigin" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - origins: values: - - [ "real", "sim" ] + - [ "front", "garden" ] # 3> "task_allocation": allocate lifelong task ; optional module; - type: "task_allocation" # name of python module; string type; name: "TaskAllocationByOrigin" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - origins: values: - - [ "real", "sim" ] \ No newline at end of file + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml old mode 100755 new mode 100644 similarity index 86% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml index 3b60b680..89715d9f --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml @@ -26,7 +26,7 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "BaseModel" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -41,7 +41,7 @@ algorithm: # name of python module; string type; name: "TaskDefinitionByOrigin" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -54,7 +54,7 @@ algorithm: # name of python module; string type; name: "TaskAllocationByOrigin" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -67,7 +67,7 @@ algorithm: # name of python module; string type; name: "HardSampleMining" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" hyperparameters: - threhold: values: diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml old mode 100755 new mode 100644 similarity index 86% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml index ada7d004..9fec28de --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml @@ -26,7 +26,7 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "BaseModel" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -41,7 +41,7 @@ algorithm: # name of python module; string type; name: "TaskDefinitionByOrigin" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -54,7 +54,7 @@ algorithm: # name of python module; string type; name: "TaskAllocationByOrigin" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -67,7 +67,7 @@ algorithm: # name of python module; string type; name: "HardSampleMining" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" hyperparameters: - threhold: values: diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml old mode 100755 new mode 100644 similarity index 86% rename from examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml rename to examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml index 223a41db..adff5710 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml @@ -26,7 +26,7 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "BaseModel" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -41,7 +41,7 @@ algorithm: # name of python module; string type; name: "TaskDefinitionByOrigin" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -54,7 +54,7 @@ algorithm: # name of python module; string type; name: "TaskAllocationByOrigin" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -67,7 +67,7 @@ algorithm: # name of python module; string type; name: "HardSampleMining" # the url address of python module; string type; - url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" hyperparameters: - threhold: values: diff --git a/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml b/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml index 092abcb8..cde00454 100755 --- a/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml +++ b/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml @@ -14,7 +14,8 @@ testenv: name: "accuracy" # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" + #mode: "no-inference" + mode: "hard-example-mining" # condition of triggering inference model to update # threshold of the condition; types are float/int diff --git a/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml b/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml index d224c3e6..e8645110 100755 --- a/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml +++ b/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml @@ -14,7 +14,8 @@ testenv: name: "accuracy" # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" + #mode: "no-inference" + mode: "hard-example-mining" # condition of triggering inference model to update # threshold of the condition; types are float/int