diff --git a/core/storymanager/rank/rank.py b/core/storymanager/rank/rank.py index 403fa79e..342cae4a 100644 --- a/core/storymanager/rank/rank.py +++ b/core/storymanager/rank/rank.py @@ -196,7 +196,7 @@ def _get_selected(self, test_cases, test_results) -> pd.DataFrame: all_df = copy.deepcopy(self.all_df) selected_df = pd.DataFrame(all_df, columns=header) selected_df = selected_df.drop_duplicates(header[:-2]) - + # pylint: disable=E1136 paradigms = self.selected_dataitem.get("paradigms") if paradigms != ["all"]: selected_df = selected_df.loc[selected_df["paradigm"].isin(paradigms)] diff --git a/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py b/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py index 6e226788..2e50ba7d 100644 --- a/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py +++ b/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py @@ -24,7 +24,7 @@ from core.testcasecontroller.metrics import get_metric_func from core.common.utils import get_file_format, is_local_dir -os.environ['CUDA_VISIBLE_DEVICES'] = '0' +os.environ['CUDA_VISIBLE_DEVICES'] = '1' class LifelongLearning(ParadigmBase): # pylint: disable=too-many-locals @@ -162,6 +162,102 @@ def run(self): #BWT, FWT = self.compute(key, matrix) self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix + elif mode == 'hard-example-mining': + dataset_files = self._split_dataset(splitting_dataset_times=rounds) + # pylint: disable=C0103 + # pylint: disable=C0206 + # pylint: disable=C0201 + # pylint: disable=W1203 + my_dict = {} + for r in range(rounds + 1): + train_dataset_file, eval_dataset_file = dataset_files[r] + if r == 0: + self.cloud_task_index = self._train(self.cloud_task_index, + train_dataset_file, + r) + + self.edge_task_index, tasks_detail, res = self.my_eval( + self.cloud_task_index, + eval_dataset_file, + r) + + else: + infer_dataset_file, eval_dataset_file = dataset_files[r] + inference_results, unseen_task_train_samples = self._inference( + self.cloud_task_index, + infer_dataset_file, + r) + samples_transfer_ratio_info.append((inference_results, + unseen_task_train_samples.x)) + + # If no unseen task samples in the this round, starting the next round + if len(unseen_task_train_samples.x) <= 0: + continue + + self.cloud_task_index = self._train(self.cloud_task_index, + unseen_task_train_samples, + r) + + tmp_dict = {} + for j in range(1, rounds+1): + _, eval_dataset_file = dataset_files[j] + self.edge_task_index, tasks_detail, res = self.my_eval( + self.cloud_task_index, + eval_dataset_file, + r) + LOGGER.info(f"train from round {r}") + LOGGER.info(f"test round {j}") + LOGGER.info(f"all scores: {res}") + score_list = tmp_dict.get("all", ['' for i in range(rounds)]) + score_list[j-1] = res + tmp_dict["all"] = score_list + task_avg_score = {'accuracy':0.0} + i = 0 + for detail in tasks_detail: + i += 1 + scores = detail.scores + entry = detail.entry + LOGGER.info(f"{entry} scores: {scores}") + task_avg_score['accuracy'] += scores['accuracy'] + score_list = tmp_dict.get(entry, ['' for i in range(rounds)]) + score_list[j-1] = scores + tmp_dict[entry] = score_list + task_avg_score['accuracy'] = task_avg_score['accuracy']/i + score_list = tmp_dict.get("task_avg", [{'accuracy':0.0} for i in range(rounds)]) + score_list[j-1] = task_avg_score + tmp_dict["task_avg"] = score_list + + for key in tmp_dict.keys(): + scores_list = my_dict.get(key, []) + scores_list.append(tmp_dict[key]) + my_dict[key] = scores_list + LOGGER.info(f"{key} scores: {scores_list}") + + + self.edge_task_index, tasks_detail, res = self.my_eval(self.cloud_task_index, + self.dataset.test_url, + rounds + 1) + task_avg_score = {'accuracy':0.0} + i = 0 + for detail in tasks_detail: + i += 1 + scores = detail.scores + entry = detail.entry + LOGGER.info(f"{entry} scores: {scores}") + task_avg_score['accuracy'] += scores['accuracy'] + task_avg_score['accuracy'] = task_avg_score['accuracy']/i + self.system_metric_info[SystemMetricType.Task_Avg_Acc.value] = task_avg_score + LOGGER.info(task_avg_score) + test_res, unseen_task_train_samples = self._inference(self.edge_task_index, + self.dataset.test_url, + "test") + for key in my_dict.keys(): + LOGGER.info(f"{key} scores: {my_dict[key]}") + for key in my_dict.keys(): + matrix = my_dict[key] + #BWT, FWT = self.compute(key, matrix) + self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix + elif mode != 'multi-inference': dataset_files = self._split_dataset(splitting_dataset_times=rounds) # pylint: disable=C0103 @@ -214,6 +310,7 @@ def _inference(self, edge_task_index, data_index_file, rounds): os.makedirs(unseen_task_saved_dir) os.environ["INFERENCE_RESULT_DIR"] = output_dir + os.environ["OUTPUT_URL"] = output_dir os.environ["MODEL_URLS"] = f"{edge_task_index}" inference_dataset = self.dataset.load_data(data_index_file, "eval", @@ -234,7 +331,7 @@ def _inference(self, edge_task_index, data_index_file, rounds): for i, _ in enumerate(inference_dataset.x): data = BaseDataSource(data_type="test") data.x = inference_dataset.x[i:(i + 1)] - res, is_unseen_task, _ = job.inference(data, **kwargs) + res, is_unseen_task, _ = job.inference_2(data, **kwargs) inference_results.append(res) if is_unseen_task: unseen_tasks.append(inference_dataset.x[i]) @@ -257,7 +354,7 @@ def _train(self, cloud_task_index, train_dataset, rounds): os.environ["CLOUD_KB_INDEX"] = cloud_task_index os.environ["OUTPUT_URL"] = train_output_dir - if rounds <= 1: + if rounds < 1: os.environ["HAS_COMPLETED_INITIAL_TRAINING"] = 'False' else: os.environ["HAS_COMPLETED_INITIAL_TRAINING"] = 'True' diff --git a/core/testenvmanager/dataset/dataset.py b/core/testenvmanager/dataset/dataset.py index 06843374..16bd038f 100644 --- a/core/testenvmanager/dataset/dataset.py +++ b/core/testenvmanager/dataset/dataset.py @@ -168,6 +168,12 @@ def split_dataset(self, dataset_url, dataset_format, ratio, method="default", output_dir=output_dir, times=times) + if method == "hard-example_splitting": + return self._hard_example_splitting(dataset_url, dataset_format, ratio, + data_types=dataset_types, + output_dir=output_dir, + times=times) + raise ValueError(f"dataset splitting method({method}) is not supported," f"currently, method supports 'default'.") @@ -320,6 +326,44 @@ def _city_splitting(self, data_file, data_format, ratio, return data_files + def _hard_example_splitting(self, data_file, data_format, ratio, + data_types=None, output_dir=None, times=1): + if not data_types: + data_types = ("train", "eval") + + if not output_dir: + output_dir = tempfile.mkdtemp() + + all_data = self._read_data_file(data_file, data_format) + + data_files = [] + + all_num = len(all_data) + step = int(all_num / (times*2)) + data_files.append(( + self._get_dataset_file(all_data[:int((all_num * ratio)/2)], output_dir, + data_types[0], 0, data_format), + self._get_dataset_file(all_data[int((all_num * ratio)/2):int(all_num/2)], output_dir, + data_types[1], 0, data_format))) + index = 1 + while index <= times: + if index == times: + new_dataset = all_data[int(all_num/2)+step*(index-1):] + else: + new_dataset = all_data[int(all_num/2)+step*(index-1): int(all_num/2)+step*index] + + new_num = len(new_dataset) + + data_files.append(( + self._get_dataset_file(new_dataset[:int(new_num * ratio)], output_dir, + data_types[0], index, data_format), + self._get_dataset_file(new_dataset[int(new_num * ratio):], output_dir, + data_types[1], index, data_format))) + + index += 1 + + return data_files + @classmethod def load_data(cls, file: str, data_type: str, label=None, use_raw=False, feature_process=None): """ diff --git a/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md b/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md index 51dd1bc3..02444f66 100644 --- a/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md +++ b/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md @@ -63,7 +63,7 @@ The overall workflow of the system is as follows: ![plugin-workflow](images/plugin-workflow.jpg) -The Hard Example Mining Module will be implemented in the unknown sample recognition module of Ianvs. The Edge Inference Module will be implemented in the known sample inference module of Ianvs' edge-side knowledge management. The Cloud Inference Module will be implemented in the unknown task processing module of Ianvs' cloud-side knowledge management. +The Hard Example Mining Module will be implemented in the unknown sample recognition module of Ianvs. The Edge Inference Module will be implemented in the known sample inference module of Ianvs' edge-side knowledge management. The Cloud Inference Module will be implemented in the unknown sample inference module. The Liflelong Training module will be implemented in the unknown task processing module of Ianvs' cloud-side knowledge management. ### 3.2 Implementation of SAM-based Semantic Segmentation diff --git a/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg b/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg index 0638501d..8ffc28a9 100644 Binary files a/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg and b/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg differ diff --git a/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg b/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg index 3ea7b40a..2241c5e4 100644 Binary files a/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg and b/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg differ diff --git a/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl b/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl index 5fb53301..a1c7cf66 100644 Binary files a/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl and b/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl differ diff --git a/examples/robot-sam/lifelong_learning_bench/README.md b/examples/robot-sam/lifelong_learning_bench/README.md index c113c14d..399d651f 100755 --- a/examples/robot-sam/lifelong_learning_bench/README.md +++ b/examples/robot-sam/lifelong_learning_bench/README.md @@ -14,7 +14,7 @@ Before using Ianvs, you might want to have the device ready: - Python 3.6+ installed -In this example, we are using the Linux platform with Python 3.8.5. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. +In this example, we are using the Linux platform with Python 3.9. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. ## Step 1. Ianvs Preparation @@ -70,19 +70,60 @@ we have done that for you and the interested readers can refer to [testenv.yaml] Related algorithm is also ready in this quick start. ``` shell -export PYTHONPATH=$PYTHONPATH:/ianvs/project/ianvs/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet +export PYTHONPATH=$PYTHONPATH:/ianvs/project/ianvs/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet ``` The URL address of this algorithm then should be filled in the configuration file ``algorithm.yaml``. In this quick start, we have done that for you and the interested readers can refer to [algorithm.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. +In this example, we use [SAM model](https://segment-anything.com/) as the cloud large model. So, we need to install SAM by the following instructions: + +~~~bash +cd /ianvs/project +git clone https://github.com/facebookresearch/segment-anything.git +cd segment-anything +python -m pip install -e . +~~~ + +Then, we need to download the pretrained SAM model: + +~~~bash +wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth +~~~ + +In order to save the inference result, we need to install mmcv and mmdetection by the following instructions: + +~~~bash +python -m pip install https://download.openmmlab.com/mmcv/dist/cu118/torch2.0.0/mmcv-2.0.0-cp39-cp39-manylinux1_x86_64.whl +cd /ianvs/project +git clone https://github.com/hsj576/mmdetection.git +cd mmdetection +python -m pip install -v -e . +~~~ + +In case that your computer couldn't run SAM model, we prepare a cache for all the SAM inference results in Cloud-Robotics dataset. You could download the cache from [this link](https://pan.baidu.com/s/1oGGBa8TjZn0ccbznQsl48g?pwd=wpp1) and put the cache file in "/ianvs/project/": + +~~~bash +cp cache.pickle /ianvs/project +~~~ + + By using the cache, you could simulate the edge-cloud joint inference without installing SAM model. + +Besides that, we also provided you a pretrained RFNet model in [this link](https://pan.baidu.com/s/1h8JnUgr1hfx5QnaFLLkMAg?pwd=jts4), you could use it if you don't want to train the RFNet model from zero. This instruction is optional: + +~~~bash +cd /ianvs/project +mkdir pretrain +cp pretrain_model.pth /ianvs/project/pretrain +~~~ + ## Step 3. Ianvs Execution and Presentation We are now ready to run the ianvs for benchmarking. ``` shell cd /ianvs/project/ianvs -ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob.yaml +ianvs -f examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml ``` Finally, the user can check the result of benchmarking on the console and also in the output path( @@ -93,9 +134,9 @@ can refer to [benchmarkingJob.yaml](https://ianvs.readthedocs.io/en/latest/guide The final output might look like this: -| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | url | -|:------:|:-------------------------:|:--------------------:|:---------------------:|:---------------------:|:------------------:|:-----------:|:------------------------:|:------------------------:|:-------------------------:|:------------------:|:-------------------------:|:-------------------------:|:---------------------:|:---------------------------------------------------------------------------------------------------------------------------------:| -| 1 | rfnet_lifelong_learning | 0.2970033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | /ianvs/lifelong_learning_bench/robot-workspace-bwt/benchmarkingjob/rfnet_lifelong_learning/efdc47a2-f9fb-11ed-8f8b-0242ac110007 | +| rank | algorithm | accuracy | Task_Avg_Acc | paradigm | basemodel | task_definition | task_allocation | unseen_sample_recognition | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | unseen_sample_recognition-threhold | time | url | +|:------:|:-------------------------:|:--------------------:|:---------------------:|:------------------:|:-----------:|:------------------------:|:------------------------:|:-------------------------:|:------------------:|:-------------------------:|:-------------------------:|:---------------------:|:---------------------------------------------------------------------------------------------------------------------------------:|-----------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------:| +| 1 | sam_rfnet_lifelong_learning | 0.7052917006987501 | 0.6258875117354328 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | HardSampleMining | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 0.95 | 2023-08-24 12:43:19 | /ianvs/sam_bench/robot-workspace/benchmarkingjob/sam_rfnet_lifelong_learning/9465c47a-4235-11ee-8519-ec2a724ccd3e | diff --git a/examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml b/examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml index 3e6b5a49..f352988a 100755 --- a/examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml +++ b/examples/robot-sam/lifelong_learning_bench/benchmarkingjob.yaml @@ -2,11 +2,13 @@ benchmarkingjob: # job name of bechmarking; string type; name: "benchmarkingjob" # the url address of job workspace that will reserve the output of tests; string type; - workspace: "/home/vipuser/ianvs/lifelong_learning_bench/robot-workspace-test" + workspace: "/ianvs/sam_bench/robot-workspace" + #workspace: "/home/hsj/ianvs/sam_bench/cloud-robot-workspace" # the url address of test environment configuration file; string type; # the file format supports yaml/yml; - testenv: "./examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml" + testenv: "./examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml" + #testenv: "./examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml" # the configuration of test object test_object: @@ -16,16 +18,24 @@ benchmarkingjob: # test algorithm configuration files; list type; algorithms: # algorithm name; string type; - - name: "rfnet_lifelong_learning" + #- name: "rfnet_lifelong_learning" + - name: "sam_rfnet_lifelong_learning" + #- name: "vit_lifelong_learning" + #- name: "sam_vit_lifelong_learning" + # the url address of test algorithm configuration file; string type; + # the file format supports yaml/yml + #url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml" + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml" + #url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml" + #url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml" # the url address of test algorithm configuration file; string type; # the file format supports yaml/yml - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm-simple.yaml" # the configuration of ranking leaderboard rank: # rank leaderboard with metric of test case's evaluation and order ; list type; # the sorting priority is based on the sequence of metrics in the list from front to back; - sort_by: [ { "accuracy": "descend" }, { "BWT": "descend" } ] + sort_by: [ { "accuracy": "descend" } ] # visualization configuration visualization: @@ -57,7 +67,7 @@ benchmarkingjob: # currently the options of value are as follows: # 1> "all": select all metrics in the leaderboard; # 2> metrics in the leaderboard, e.g., "F1_SCORE" - metrics: [ "accuracy", "Task_Avg_Acc", "BWT", "FWT"] + metrics: [ "accuracy", "Task_Avg_Acc"] # model of save selected and all dataitems in workspace `./rank` ; string type; # currently the options of value are as follows: diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py new file mode 100644 index 00000000..c8ad1d27 --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py @@ -0,0 +1,34 @@ +CONFIG = { +"id2label": { + "0": "road", + "1": "sidewalk", + "2": "building", + "3": "wall", + "4": "fence", + "5": "pole", + "6": "traffic light", + "7": "traffic sign", + "8": "vegetation", + "9": "terrain", + "10": "sky", + "11": "person", + "12": "rider", + "13": "car", + "14": "truck", + "15": "bus", + "16": "train", + "17": "motorcycle", + "18": "bicycle", + "19": "stair", + "20": "curb", + "21": "ramp", + "22": "runway", + "23": "flowerbed", + "24": "door", + "25": "CCTV camera", + "26": "Manhole", + "27": "hydrant", + "28": "belt", + "29": "dustbin" + }, +} \ No newline at end of file diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py index 2d303429..da3538b8 100755 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py @@ -7,6 +7,12 @@ from torchvision.transforms import ToPILImage from PIL import Image +from segment_anything import sam_model_registry, SamAutomaticMaskGenerator +import mmcv +import pycocotools.mask as maskUtils +from mmdet.visualization.image import imshow_det_bboxes +import pickle + from dataloaders import make_data_loader from dataloaders.utils import decode_seg_map_sequence, Colorize from utils.metrics import Evaluator @@ -15,6 +21,9 @@ from models.resnet.resnet_single_scale_single_attention import * from models.resnet import resnet_single_scale_single_attention_unseen import torch.backends.cudnn as cudnn +from cityscapes_id2label import CONFIG as CONFIG_CITYSCAPES_ID2LABEL +import torch.nn.functional as F +from transformers import SegformerFeatureExtractor, SegformerForSemanticSegmentation os.environ["OMP_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" @@ -54,12 +63,257 @@ def __init__(self, args, data=None, unseen_detection=False): # self.new_state_dict = torch.load(args.weight_path, map_location=torch.device("cpu")) # self.model = load_my_state_dict(self.model, self.new_state_dict['state_dict']) + def segformer_segmentation(self, image, processor, model, rank): + h, w, _ = image.shape + inputs = processor(images=image, return_tensors="pt").to(rank) + outputs = model(**inputs) + logits = outputs.logits + logits = F.interpolate(logits, size=(h, w), mode='bilinear', align_corners=True) + predicted_semantic_map = logits.argmax(dim=1) + return predicted_semantic_map + + def draw_mask(self, image_name, mask, output_path): + img = mmcv.imread(image_name) + anns = {'annotations': mask} + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + semantc_mask = torch.zeros(1024, 2048) + i = 0 + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + semantc_mask[valid_mask] = i + i += 1 + sematic_class_in_img = torch.unique(semantc_mask) + semantic_bitmasks, semantic_class_names = [], [] + + # semantic prediction + for i in range(len(sematic_class_in_img)): + class_name = str(i) + class_mask = semantc_mask == sematic_class_in_img[i] + class_mask = class_mask.cpu().numpy().astype(np.uint8) + semantic_class_names.append(class_name) + semantic_bitmasks.append(class_mask) + + length = len(image_name) + for i in range(length): + if image_name[length-i-1] == '_': + break + filename = image_name[length-i:] + imshow_det_bboxes(img, + bboxes=None, + labels=np.arange(len(sematic_class_in_img)), + segms=np.stack(semantic_bitmasks), + class_names=semantic_class_names, + font_size=25, + show=False, + out_file=os.path.join(output_path, filename + '_mask.png')) + print('[Save] save mask: ', os.path.join(output_path, filename + '_mask.png')) + semantc_mask = semantc_mask.unsqueeze(0).numpy() + del img + del semantic_bitmasks + del semantic_class_names + + def draw_picture(self, image_name, semantc_mask, id2label, output_path, suffix): + img = mmcv.imread(image_name) + sematic_class_in_img = torch.unique(semantc_mask) + semantic_bitmasks, semantic_class_names = [], [] + + # semantic prediction + for i in range(len(sematic_class_in_img)): + class_name = id2label['id2label'][str(sematic_class_in_img[i].item())] + class_mask = semantc_mask == sematic_class_in_img[i] + class_mask = class_mask.cpu().numpy().astype(np.uint8) + semantic_class_names.append(class_name) + semantic_bitmasks.append(class_mask) + + #print(os.environ["OUTPUT_URL"]) + length = len(image_name) + for i in range(length): + if image_name[length-i-1] == '_': + break + filename = image_name[length-i:] + imshow_det_bboxes(img, + bboxes=None, + labels=np.arange(len(sematic_class_in_img)), + segms=np.stack(semantic_bitmasks), + class_names=semantic_class_names, + font_size=25, + show=False, + out_file=os.path.join(output_path, filename + suffix)) + print('[Save] save rfnet prediction: ', os.path.join(output_path, filename + suffix)) + #semantc_mask = semantc_mask.unsqueeze(0).numpy() + del img + del semantic_bitmasks + del semantic_class_names + + def confidence(self, input_output): + output = torch.softmax(input_output, dim=0) + highth = len(output[0]) + width = len(output[0][0]) + sum_1 = 0.0 + sum_2 = 0.0 + values, _ = torch.topk(output, k=2, dim=0) + sum_1 = torch.sum(values[0]) + value_2 = torch.sub(values[0],values[1]) + sum_2 = torch.sum(value_2) + sum_1 = sum_1/(highth*width) + sum_2 = sum_2/(highth*width) + count = (values[0] > 0.9).sum().item() + sum_3 = count/(highth*width) + return sum_3 + + def sam_predict_ssa(self, image_name, pred): + with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: + cache = pickle.load(file) + img = mmcv.imread(image_name) + if image_name in cache.keys(): + mask = cache[image_name] + print("load cache") + else: + sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') + mask_branch_model = SamAutomaticMaskGenerator( + model=sam, + #points_per_side=64, + # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side + #pred_iou_thresh=0.86, + #stability_score_thresh=0.92, + #crop_n_layers=1, + #crop_n_points_downscale_factor=2, + #min_mask_region_area=100, # Requires open-cv to run post-processing + output_mode='coco_rle', + ) + print('[Model loaded] Mask branch (SAM) is loaded.') + mask = mask_branch_model.generate(img) + cache[image_name] = mask + with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: + pickle.dump(cache, file) + print("save cache") + + anns = {'annotations': mask} + #print(len(anns['annotations']), len(anns['annotations'][0])) + #print(pred.shape) + #print(pred[0]) + class_names = [] + semantc_mask = pred.clone() + id2label = CONFIG_CITYSCAPES_ID2LABEL + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + #print(valid_mask) + propose_classes_ids = pred[valid_mask] + num_class_proposals = len(torch.unique(propose_classes_ids)) + if num_class_proposals == 1: + semantc_mask[valid_mask] = propose_classes_ids[0] + ann['class_name'] = id2label['id2label'][str(propose_classes_ids[0].item())] + ann['class_proposals'] = id2label['id2label'][str(propose_classes_ids[0].item())] + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + continue + top_1_propose_class_ids = torch.bincount(propose_classes_ids.flatten()).topk(1).indices + top_1_propose_class_names = [id2label['id2label'][str(class_id.item())] for class_id in top_1_propose_class_ids] + #print("top_1_propose_class_ids: ", top_1_propose_class_ids) + semantc_mask[valid_mask] = top_1_propose_class_ids + ann['class_name'] = top_1_propose_class_names[0] + ann['class_proposals'] = top_1_propose_class_names[0] + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + + del valid_mask + del propose_classes_ids + del num_class_proposals + del top_1_propose_class_ids + del top_1_propose_class_names + + #print(semantc_mask.shape) + #print(semantc_mask) + + del img + del anns + #del semantc_mask + # del bitmasks + del class_names + return semantc_mask, mask + + def sam_predict(self, image_name, pred): + with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: + cache = pickle.load(file) + img = mmcv.imread(image_name) + if image_name in cache.keys(): + mask = cache[image_name] + print("load cache") + else: + sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') + mask_branch_model = SamAutomaticMaskGenerator( + model=sam, + #points_per_side=64, + # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side + #pred_iou_thresh=0.86, + #stability_score_thresh=0.92, + #crop_n_layers=1, + #crop_n_points_downscale_factor=2, + #min_mask_region_area=100, # Requires open-cv to run post-processing + output_mode='coco_rle', + ) + print('[Model loaded] Mask branch (SAM) is loaded.') + mask = mask_branch_model.generate(img) + cache[image_name] = mask + with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: + pickle.dump(cache, file) + print("save cache") + + anns = {'annotations': mask} + #print(len(anns['annotations']), len(anns['annotations'][0])) + #print(pred.shape) + #print(pred[0]) + class_names = [] + pred_2 = np.argmax(pred, axis=0) + semantc_mask = pred_2.clone() + id2label = CONFIG_CITYSCAPES_ID2LABEL + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + #print(valid_mask) + length = len(pred) + all_scores = [0 for i in range(length)] + for i in range(length): + propose_classes_ids = pred[i][valid_mask] + #print(propose_classes_ids.shape) + all_scores[i] = torch.sum(propose_classes_ids) + #print(all_scores[i]) + top_1_propose_class_ids = np.argmax(all_scores) + #print(top_1_propose_class_ids) + top_1_propose_class_names = id2label['id2label'][str(top_1_propose_class_ids)] + + semantc_mask[valid_mask] = top_1_propose_class_ids + ann['class_name'] = top_1_propose_class_names + ann['class_proposals'] = top_1_propose_class_names + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + + del valid_mask + del propose_classes_ids + del top_1_propose_class_ids + del top_1_propose_class_names + + #print(semantc_mask.shape) + #print(semantc_mask) + + #self.draw_picture(img, image_name, pred_2, id2label, output_path, "_origin.png") + #self.draw_picture(img, image_name, semantc_mask, id2label, output_path, "_sam.png") + + del img + del anns + #del semantc_mask + # del bitmasks + del class_names + return semantc_mask, mask + def validate(self): #print("start validating 55") self.model.eval() self.evaluator.reset() tbar = tqdm(self.test_loader, desc='\r') predictions = [] + scores = [] for i, (sample, image_name) in enumerate(tbar):#self.test_loader: if self.args.depth: image, depth, target = sample['image'], sample['depth'], sample['label'] @@ -81,46 +335,131 @@ def validate(self): if self.args.cuda: torch.cuda.synchronize() + if len(output) == 1: + score = self.confidence(output[0]) + else: + score = self.confidence(output) + scores.append(score) + pred = output.data.cpu().numpy() # todo pred = np.argmax(pred, axis=1) predictions.append(pred) - if not self.args.save_predicted_image: - continue + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_origin.png") - pre_colors = Colorize()(torch.max(output, 1)[1].detach().cpu().byte()) - pre_labels = torch.max(output, 1)[1].detach().cpu().byte() - print(pre_labels.shape) - # save - for i in range(pre_colors.shape[0]): - print(image_name[0]) + #print("start validating 120") + return predictions, scores + + def vit_validate(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + rank = 'cuda:0' + semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") + semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + img = mmcv.imread(image_name[0]) + class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) + pred = class_ids.data.cpu().numpy() + predictions.append(pred) + + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") + + #print("start validating 120") + return predictions - if not image_name[0]: - img_name = "test.png" + def validate_cloud(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + scores = [] + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + if self.args.depth: + image, depth, target = sample['image'], sample['depth'], sample['label'] + else: + # spec = time.time() + image, target = sample['image'], sample['label'] + #print(self.args.cuda, self.args.gpu_ids) + if self.args.cuda: + image = image.cuda(self.args.gpu_ids) + if self.args.depth: + depth = depth.cuda(self.args.gpu_ids) + + with torch.no_grad(): + if self.args.depth: + output = self.model(image, depth) else: - img_name = os.path.basename(image_name[0]) + output = self.model(image) + + if self.args.cuda: + torch.cuda.synchronize() - color_label_name = os.path.join(self.args.color_label_save_path, img_name) - label_name = os.path.join(self.args.label_save_path, img_name) - merge_label_name = os.path.join(self.args.merge_label_save_path, img_name) + if len(output) == 1: + score = self.confidence(output[0]) + else: + score = self.confidence(output) + scores.append(score) + + pred = output.data.cpu().numpy() + # todo + pred_sam, mask = self.sam_predict(image_name[0], torch.from_numpy(pred[0])) + if pred_sam.ndim < 3: + h, w = pred_sam.shape + pred_sam = pred_sam.reshape(1, h, w) + #print(pred_sam.shape) - os.makedirs(os.path.dirname(color_label_name), exist_ok=True) - os.makedirs(os.path.dirname(merge_label_name), exist_ok=True) - os.makedirs(os.path.dirname(label_name), exist_ok=True) + predictions.append(np.array(pred_sam)) - pre_color_image = ToPILImage()(pre_colors[i]) # pre_colors.dtype = float64 - pre_color_image.save(color_label_name) + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_sam.png") + self.draw_mask(image_name[0], mask, output_path) + + #print("start validating 120") + return predictions, scores - pre_label_image = ToPILImage()(pre_labels[i]) - pre_label_image.save(label_name) + def vit_validate_cloud(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + rank = 'cuda:0' + semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") + semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + img = mmcv.imread(image_name[0]) + class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) + pred = class_ids.data.cpu().numpy() + pred_sam, mask = self.sam_predict_ssa(image_name[0], torch.from_numpy(pred[0])) + if pred_sam.ndim < 3: + h, w = pred_sam.shape + pred_sam = pred_sam.reshape(1, h, w) + #print(pred_sam.shape) + predictions.append(np.array(pred_sam)) - if (self.args.merge): - image_merge(image[i], pre_color_image, merge_label_name) - print('save image: {}'.format(merge_label_name)) + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") + self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_vit_sam.png") + self.draw_mask(image_name[0], mask, output_path) + #print("start validating 120") return predictions - + def task_divide(self): seen_task_samples, unseen_task_samples = [], [] self.model.eval() diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py index 8a56dd74..3691a1f3 100755 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py @@ -21,7 +21,7 @@ def __init__(self, **kwargs): self.gpu_ids = 0 self.seed = 1 - self.resume = None + self.resume = '/home/hsj/ianvs/project/pretrain/pretrain_model.pth' self.checkname = 'RFNet' self.ft = True self.eval_interval = kwargs.get("eval_interval", 50) diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py index b2e93683..0acef918 100755 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py @@ -188,6 +188,8 @@ def add_batch(self, gt_image, pre_image): gt_image = np.array(gt_image) pre_image = np.array(pre_image) print(gt_image.shape, pre_image.shape) + if gt_image.shape != pre_image.shape: + pre_image = pre_image[0] if gt_image.shape != pre_image.shape: pre_image = pre_image[0] assert gt_image.shape == pre_image.shape diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py new file mode 100755 index 00000000..8b0d2c39 --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py @@ -0,0 +1,200 @@ +import os +import gc +import numpy as np +import torch +from torch.utils.data import DataLoader +from sedna.common.class_factory import ClassType, ClassFactory +from sedna.common.config import Context +from sedna.common.file_ops import FileOps +from sedna.common.log import LOGGER +from PIL import Image +from torchvision import transforms + +from RFNet.train import Trainer +from RFNet.eval import Validator, load_my_state_dict +from RFNet.dataloaders import custom_transforms as tr +from RFNet.dataloaders import make_data_loader +from RFNet.utils.args import TrainArgs, ValArgs + +# set backend +os.environ['BACKEND_TYPE'] = 'PYTORCH' + +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") +class BaseModel: + def __init__(self, **kwargs): + self.train_args = TrainArgs(**kwargs) + self.trainer = None + + self.val_args = ValArgs(**kwargs) + label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + self.val_args.label_save_path = os.path.join(label_save_dir, "label") + self.validator = Validator(self.val_args) + + def get_weights(self): + return self.trainer.get_weight() + + def set_weights(self, weights): + self.trainer.set_weight(weights) + + epoch_num = 0 + print("Total epoch: ", epoch_num) + loss_all = [] + for epoch in range(epoch_num): + train_loss = self.trainer.my_training(epoch) + loss_all.append(train_loss) + with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: + np.savetxt(file, loss_all) + file.close + + def train(self, train_data, valid_data=None, **kwargs): + self.trainer = Trainer(self.train_args, train_data=train_data) + print("Total epoches:", self.trainer.args.epochs) + loss_all = [] + for epoch in range( + self.trainer.args.start_epoch, + self.trainer.args.epochs): + if epoch == 0 and self.trainer.val_loader: + self.trainer.validation(epoch) + loss = self.trainer.training(epoch) + loss_all.append(loss) + if self.trainer.args.no_val and ( + epoch % + self.trainer.args.eval_interval == ( + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - + 1): + # save checkpoint when it meets eval_interval or the training + # finished + is_best = False + self.train_model_url = self.trainer.saver.save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': self.trainer.model.state_dict(), + 'optimizer': self.trainer.optimizer.state_dict(), + 'best_pred': self.trainer.best_pred, + }, is_best) + + self.trainer.writer.close() + return self.train_model_url + + def predict(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions, scores = self.validator.validate() + return predictions + + def predict_cloud(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions = self.validator.validate_cloud() + return predictions + + def predict_score(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions, scores = self.validator.validate() + return scores + + def evaluate(self, data, **kwargs): + #print("evaluate starting 77") + self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + samples = self._preprocess(data.x) + predictions = self.predict(samples) + metric_name, metric_func = kwargs.get("metric") + if callable(metric_func): + return metric_func(data.y, predictions) + else: + raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + + def load(self, model_url, **kwargs): + if model_url: + print("load model url: ",model_url) + self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + self.train_args.resume = model_url + else: + raise Exception("model url does not exist.") + self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + + def save(self, model_path=None): + # TODO: save unstructured data model + if not model_path: + LOGGER.warning(f"Not specify model path.") + return self.train_model_url + + return FileOps.upload(self.train_model_url, model_path) + + def _preprocess(self, image_urls): + transformed_images = [] + for paths in image_urls: + if len(paths) == 2: + img_path, depth_path = paths + _img = Image.open(img_path).convert('RGB') + _depth = Image.open(depth_path) + #print(img_path, depth_path) + else: + img_path = paths[0] + #print(img_path) + _img = Image.open(img_path).convert('RGB') + _depth = _img + + sample = {'image': _img, 'depth': _depth, 'label': _img} + del _img + gc.collect() + composed_transforms = transforms.Compose([ + # tr.CropBlackArea(), + # tr.FixedResize(size=self.args.crop_size), + tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.ToTensor()]) + + transformed_images.append((composed_transforms(sample), img_path)) + + return transformed_images diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py index c2ad091e..5563a82c 100755 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py @@ -99,7 +99,50 @@ def predict(self, data, **kwargs): pin_memory=True) #print("predict starting 75") - return self.validator.validate() + predictions, scores = self.validator.validate() + return predictions + + def predict_cloud(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions, scores = self.validator.validate() + return predictions + + def predict_score(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions, scores = self.validator.validate() + return scores def evaluate(self, data, **kwargs): #print("evaluate starting 77") diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py new file mode 100755 index 00000000..bb339951 --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py @@ -0,0 +1,200 @@ +import os +import gc +import numpy as np +import torch +from torch.utils.data import DataLoader +from sedna.common.class_factory import ClassType, ClassFactory +from sedna.common.config import Context +from sedna.common.file_ops import FileOps +from sedna.common.log import LOGGER +from PIL import Image +from torchvision import transforms + +from RFNet.train import Trainer +from RFNet.eval import Validator, load_my_state_dict +from RFNet.dataloaders import custom_transforms as tr +from RFNet.dataloaders import make_data_loader +from RFNet.utils.args import TrainArgs, ValArgs + +# set backend +os.environ['BACKEND_TYPE'] = 'PYTORCH' + +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") +class BaseModel: + def __init__(self, **kwargs): + self.train_args = TrainArgs(**kwargs) + self.trainer = None + + self.val_args = ValArgs(**kwargs) + label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + self.val_args.label_save_path = os.path.join(label_save_dir, "label") + self.validator = Validator(self.val_args) + + def get_weights(self): + return self.trainer.get_weight() + + def set_weights(self, weights): + self.trainer.set_weight(weights) + + epoch_num = 0 + print("Total epoch: ", epoch_num) + loss_all = [] + for epoch in range(epoch_num): + train_loss = self.trainer.my_training(epoch) + loss_all.append(train_loss) + with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: + np.savetxt(file, loss_all) + file.close + + def train(self, train_data, valid_data=None, **kwargs): + self.trainer = Trainer(self.train_args, train_data=train_data) + print("Total epoches:", self.trainer.args.epochs) + loss_all = [] + for epoch in range( + self.trainer.args.start_epoch, + self.trainer.args.epochs): + if epoch == 0 and self.trainer.val_loader: + self.trainer.validation(epoch) + loss = self.trainer.training(epoch) + loss_all.append(loss) + if self.trainer.args.no_val and ( + epoch % + self.trainer.args.eval_interval == ( + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - + 1): + # save checkpoint when it meets eval_interval or the training + # finished + is_best = False + self.train_model_url = self.trainer.saver.save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': self.trainer.model.state_dict(), + 'optimizer': self.trainer.optimizer.state_dict(), + 'best_pred': self.trainer.best_pred, + }, is_best) + + self.trainer.writer.close() + return self.train_model_url + + def predict(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions = self.validator.vit_validate() + return predictions + + def predict_cloud(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions = self.validator.vit_validate_cloud() + return predictions + + def predict_score(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions, scores = self.validator.validate() + return scores + + def evaluate(self, data, **kwargs): + #print("evaluate starting 77") + self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + samples = self._preprocess(data.x) + predictions = self.predict(samples) + metric_name, metric_func = kwargs.get("metric") + if callable(metric_func): + return metric_func(data.y, predictions) + else: + raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + + def load(self, model_url, **kwargs): + if model_url: + print("load model url: ",model_url) + self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + self.train_args.resume = model_url + else: + raise Exception("model url does not exist.") + self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + + def save(self, model_path=None): + # TODO: save unstructured data model + if not model_path: + LOGGER.warning(f"Not specify model path.") + return self.train_model_url + + return FileOps.upload(self.train_model_url, model_path) + + def _preprocess(self, image_urls): + transformed_images = [] + for paths in image_urls: + if len(paths) == 2: + img_path, depth_path = paths + _img = Image.open(img_path).convert('RGB') + _depth = Image.open(depth_path) + #print(img_path, depth_path) + else: + img_path = paths[0] + #print(img_path) + _img = Image.open(img_path).convert('RGB') + _depth = _img + + sample = {'image': _img, 'depth': _depth, 'label': _img} + del _img + gc.collect() + composed_transforms = transforms.Compose([ + # tr.CropBlackArea(), + # tr.FixedResize(size=self.args.crop_size), + tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.ToTensor()]) + + transformed_images.append((composed_transforms(sample), img_path)) + + return transformed_images diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py new file mode 100755 index 00000000..e89200ce --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py @@ -0,0 +1,200 @@ +import os +import gc +import numpy as np +import torch +from torch.utils.data import DataLoader +from sedna.common.class_factory import ClassType, ClassFactory +from sedna.common.config import Context +from sedna.common.file_ops import FileOps +from sedna.common.log import LOGGER +from PIL import Image +from torchvision import transforms + +from RFNet.train import Trainer +from RFNet.eval import Validator, load_my_state_dict +from RFNet.dataloaders import custom_transforms as tr +from RFNet.dataloaders import make_data_loader +from RFNet.utils.args import TrainArgs, ValArgs + +# set backend +os.environ['BACKEND_TYPE'] = 'PYTORCH' + +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") +class BaseModel: + def __init__(self, **kwargs): + self.train_args = TrainArgs(**kwargs) + self.trainer = None + + self.val_args = ValArgs(**kwargs) + label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + self.val_args.label_save_path = os.path.join(label_save_dir, "label") + self.validator = Validator(self.val_args) + + def get_weights(self): + return self.trainer.get_weight() + + def set_weights(self, weights): + self.trainer.set_weight(weights) + + epoch_num = 0 + print("Total epoch: ", epoch_num) + loss_all = [] + for epoch in range(epoch_num): + train_loss = self.trainer.my_training(epoch) + loss_all.append(train_loss) + with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: + np.savetxt(file, loss_all) + file.close + + def train(self, train_data, valid_data=None, **kwargs): + self.trainer = Trainer(self.train_args, train_data=train_data) + print("Total epoches:", self.trainer.args.epochs) + loss_all = [] + for epoch in range( + self.trainer.args.start_epoch, + self.trainer.args.epochs): + if epoch == 0 and self.trainer.val_loader: + self.trainer.validation(epoch) + loss = self.trainer.training(epoch) + loss_all.append(loss) + if self.trainer.args.no_val and ( + epoch % + self.trainer.args.eval_interval == ( + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - + 1): + # save checkpoint when it meets eval_interval or the training + # finished + is_best = False + self.train_model_url = self.trainer.saver.save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': self.trainer.model.state_dict(), + 'optimizer': self.trainer.optimizer.state_dict(), + 'best_pred': self.trainer.best_pred, + }, is_best) + + self.trainer.writer.close() + return self.train_model_url + + def predict(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions = self.validator.vit_validate() + return predictions + + def predict_cloud(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions = self.validator.vit_validate() + return predictions + + def predict_score(self, data, **kwargs): + if len(data) > 10: + print("predict start for big data") + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + print("predict start for small data") + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + #print("predict starting 69") + if type(data) is np.ndarray: + data = data.tolist() + #print("predict starting 72") + #print("predict starting 73") + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + #print("predict starting 75") + predictions, scores = self.validator.validate() + return scores + + def evaluate(self, data, **kwargs): + #print("evaluate starting 77") + self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + samples = self._preprocess(data.x) + predictions = self.predict(samples) + metric_name, metric_func = kwargs.get("metric") + if callable(metric_func): + return metric_func(data.y, predictions) + else: + raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + + def load(self, model_url, **kwargs): + if model_url: + print("load model url: ",model_url) + self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + self.train_args.resume = model_url + else: + raise Exception("model url does not exist.") + self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + + def save(self, model_path=None): + # TODO: save unstructured data model + if not model_path: + LOGGER.warning(f"Not specify model path.") + return self.train_model_url + + return FileOps.upload(self.train_model_url, model_path) + + def _preprocess(self, image_urls): + transformed_images = [] + for paths in image_urls: + if len(paths) == 2: + img_path, depth_path = paths + _img = Image.open(img_path).convert('RGB') + _depth = Image.open(depth_path) + #print(img_path, depth_path) + else: + img_path = paths[0] + #print(img_path) + _img = Image.open(img_path).convert('RGB') + _depth = _img + + sample = {'image': _img, 'depth': _depth, 'label': _img} + del _img + gc.collect() + composed_transforms = transforms.Compose([ + # tr.CropBlackArea(), + # tr.FixedResize(size=self.args.crop_size), + tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.ToTensor()]) + + transformed_images.append((composed_transforms(sample), img_path)) + + return transformed_images diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py new file mode 100644 index 00000000..ab7772ce --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py @@ -0,0 +1,56 @@ +# @inproceedings{zhou2021learning, +# author = {Zhou, Da-Wei and Ye, Han-Jia and Zhan, De-Chuan}, +# title = {Learning Placeholders for Open-Set Recognition}, +# booktitle = {CVPR}, +# pages = {4401-4410}, +# year = {2021} +# } +from typing import List, Any, Tuple + +from sedna.datasources import BaseDataSource +from sedna.common.class_factory import ClassFactory, ClassType +import random + +__all__ = ('HardSampleMining',) + +@ClassFactory.register(ClassType.UTD, alias="HardSampleMining") +class HardSampleMining: + """ + Dividing the data set based on whether the sample is a known class of data. + + Parameters + ---------- + model_path: string + Path of the model + """ + def __init__(self, **kwargs): + self.threhold = kwargs.get("threhold") + + def __call__(self, + samples: BaseDataSource, scores, **kwargs) -> Tuple[BaseDataSource, + BaseDataSource]: + ''' + Parameters + ---------- + samples : BaseDataSource + inference samples + + Returns + ------- + seen_task_samples: BaseDataSource + unseen_task_samples: BaseDataSource + ''' + seen_task_samples = BaseDataSource(data_type=samples.data_type) + unseen_task_samples = BaseDataSource(data_type=samples.data_type) + + if scores[0] > self.threhold: + print(f"found easy sample, confidence score: {scores[0]}") + seen_task_samples.x = samples.x + unseen_task_samples.x = [] + + else: + print(f"found hard sample, confidence score: {scores[0]}") + seen_task_samples.x = [] + unseen_task_samples.x = samples.x + + return seen_task_samples, unseen_task_samples diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml index 628ceb9a..8777c34b 100755 --- a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml @@ -12,8 +12,9 @@ algorithm: # the method of splitting dataset; string type; optional; # currently the options of value are as follows: # 1> "default": the dataset is evenly divided based train_ratio; - splitting_method: "default" - #splitting_method: "my_splitting" + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" # algorithm module configuration in the paradigm; list type; modules: @@ -25,7 +26,7 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "BaseModel" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py" + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -34,30 +35,40 @@ algorithm: - 0.0001 - epochs: values: - - 1 + - 10 # 2> "task_definition": define lifelong task ; optional module; - type: "task_definition" # name of python module; string type; name: "TaskDefinitionByOrigin" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py" + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - origins: values: - - [ "real", "sim" ] + - [ "front", "garden" ] # 3> "task_allocation": allocate lifelong task ; optional module; - type: "task_allocation" # name of python module; string type; name: "TaskAllocationByOrigin" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py" + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - origins: values: - - [ "real", "sim" ] \ No newline at end of file + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml new file mode 100755 index 00000000..3b60b680 --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml @@ -0,0 +1,74 @@ +algorithm: + # paradigm type; string type; + # currently the options of value are as follows: + # 1> "singletasklearning" + # 2> "incrementallearning" + # 3> "lifelonglearning" + paradigm_type: "lifelonglearning" + lifelong_learning_data_setting: + # ratio of training dataset; float type; + # the default value is 0.8. + train_ratio: 0.9 + # the method of splitting dataset; string type; optional; + # currently the options of value are as follows: + # 1> "default": the dataset is evenly divided based train_ratio; + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" + + # algorithm module configuration in the paradigm; list type; + modules: + # type of algorithm module; string type; + # currently the options of value are as follows: + # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; + - type: "basemodel" + # name of python module; string type; + # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; + name: "BaseModel" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + - learning_rate: + values: + - 0.0001 + - epochs: + values: + - 1 + # 2> "task_definition": define lifelong task ; optional module; + - type: "task_definition" + # name of python module; string type; + name: "TaskDefinitionByOrigin" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + # 3> "task_allocation": allocate lifelong task ; optional module; + - type: "task_allocation" + # name of python module; string type; + name: "TaskAllocationByOrigin" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml new file mode 100755 index 00000000..ada7d004 --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml @@ -0,0 +1,74 @@ +algorithm: + # paradigm type; string type; + # currently the options of value are as follows: + # 1> "singletasklearning" + # 2> "incrementallearning" + # 3> "lifelonglearning" + paradigm_type: "lifelonglearning" + lifelong_learning_data_setting: + # ratio of training dataset; float type; + # the default value is 0.8. + train_ratio: 0.9 + # the method of splitting dataset; string type; optional; + # currently the options of value are as follows: + # 1> "default": the dataset is evenly divided based train_ratio; + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" + + # algorithm module configuration in the paradigm; list type; + modules: + # type of algorithm module; string type; + # currently the options of value are as follows: + # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; + - type: "basemodel" + # name of python module; string type; + # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; + name: "BaseModel" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + - learning_rate: + values: + - 0.0001 + - epochs: + values: + - 1 + # 2> "task_definition": define lifelong task ; optional module; + - type: "task_definition" + # name of python module; string type; + name: "TaskDefinitionByOrigin" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + # 3> "task_allocation": allocate lifelong task ; optional module; + - type: "task_allocation" + # name of python module; string type; + name: "TaskAllocationByOrigin" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml new file mode 100755 index 00000000..223a41db --- /dev/null +++ b/examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml @@ -0,0 +1,74 @@ +algorithm: + # paradigm type; string type; + # currently the options of value are as follows: + # 1> "singletasklearning" + # 2> "incrementallearning" + # 3> "lifelonglearning" + paradigm_type: "lifelonglearning" + lifelong_learning_data_setting: + # ratio of training dataset; float type; + # the default value is 0.8. + train_ratio: 0.9 + # the method of splitting dataset; string type; optional; + # currently the options of value are as follows: + # 1> "default": the dataset is evenly divided based train_ratio; + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" + + # algorithm module configuration in the paradigm; list type; + modules: + # type of algorithm module; string type; + # currently the options of value are as follows: + # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; + - type: "basemodel" + # name of python module; string type; + # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; + name: "BaseModel" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + - learning_rate: + values: + - 0.0001 + - epochs: + values: + - 1 + # 2> "task_definition": define lifelong task ; optional module; + - type: "task_definition" + # name of python module; string type; + name: "TaskDefinitionByOrigin" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + # 3> "task_allocation": allocate lifelong task ; optional module; + - type: "task_allocation" + # name of python module; string type; + name: "TaskAllocationByOrigin" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot-sam/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml index 092abcb8..cde00454 100755 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml +++ b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot-small.yaml @@ -14,7 +14,8 @@ testenv: name: "accuracy" # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" + #mode: "no-inference" + mode: "hard-example-mining" # condition of triggering inference model to update # threshold of the condition; types are float/int diff --git a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml index d224c3e6..e8645110 100755 --- a/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml +++ b/examples/robot-sam/lifelong_learning_bench/testenv/testenv-robot.yaml @@ -14,7 +14,8 @@ testenv: name: "accuracy" # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" + #mode: "no-inference" + mode: "hard-example-mining" # condition of triggering inference model to update # threshold of the condition; types are float/int