From f89d3c03c891bc1dce0597f5efa94fcafbfe4596 Mon Sep 17 00:00:00 2001 From: qxygxt Date: Fri, 26 Jan 2024 18:23:10 +0800 Subject: [PATCH 1/3] Optimize the problem of CI test and the structure of OSPP MDIL-SS Signed-off-by: qxygxt --- core/common/constant.py | 2 +- core/testcasecontroller/metrics/metrics.py | 42 +++--- .../benchmarkingjob.yaml | 6 - .../testalgorithms/erfnet/ERFNet/accuracy.py | 3 +- .../testalgorithms/erfnet/ERFNet/eval.py | 56 ++++---- .../erfnet/ERFNet/models/erfnet.py | 56 +++++--- .../testalgorithms/erfnet/ERFNet/train.py | 135 +++++++++++------- .../testalgorithms/erfnet/basemodel.py | 47 +++--- .../erfnet/task_allocation_by_domain.py | 9 +- .../erfnet/task_definition_by_domain.py | 8 +- 10 files changed, 210 insertions(+), 154 deletions(-) diff --git a/core/common/constant.py b/core/common/constant.py index ea4aa397..368e34fb 100644 --- a/core/common/constant.py +++ b/core/common/constant.py @@ -68,13 +68,13 @@ class SystemMetricType(Enum): """ System metric type of ianvs. """ - # pylint: disable=C0103 SAMPLES_TRANSFER_RATIO = "samples_transfer_ratio" FWT = "FWT" BWT = "BWT" Task_Avg_Acc = "Task_Avg_Acc" Matrix = "Matrix" + class TestObjectType(Enum): """ Test object type of ianvs. diff --git a/core/testcasecontroller/metrics/metrics.py b/core/testcasecontroller/metrics/metrics.py index b4f72983..7c53f8c3 100644 --- a/core/testcasecontroller/metrics/metrics.py +++ b/core/testcasecontroller/metrics/metrics.py @@ -39,7 +39,8 @@ def samples_transfer_ratio_func(system_metric_info: dict): """ - info = system_metric_info.get(SystemMetricType.SAMPLES_TRANSFER_RATIO.value) + info = system_metric_info.get( + SystemMetricType.SAMPLES_TRANSFER_RATIO.value) inference_num = 0 transfer_num = 0 for inference_data, transfer_data in info: @@ -47,37 +48,20 @@ def samples_transfer_ratio_func(system_metric_info: dict): transfer_num += len(transfer_data) return round(float(transfer_num) / (inference_num + 1), 4) + def compute(key, matrix): """ Compute BWT and FWT scores for a given matrix. """ - # pylint: disable=C0103 - # pylint: disable=C0301 - # pylint: disable=C0303 - # pylint: disable=R0912 + print( + f"compute function: key={key}, matrix={matrix}, type(matrix)={type(matrix)}") - print(f"compute function: key={key}, matrix={matrix}, type(matrix)={type(matrix)}") - length = len(matrix) accuracy = 0.0 BWT_score = 0.0 FWT_score = 0.0 flag = True - if key == 'all': - for i in range(length-1, 0, -1): - sum_before_i = sum(item['accuracy'] for item in matrix[i][:i]) - sum_after_i = sum(item['accuracy'] for item in matrix[i][-(length - i - 1):]) - if i == 0: - seen_class_accuracy = 0.0 - else: - seen_class_accuracy = sum_before_i / i - if length - 1 - i == 0: - unseen_class_accuracy = 0.0 - else: - unseen_class_accuracy = sum_after_i / (length - 1 - i) - print(f"round {i} : unseen class accuracy is {unseen_class_accuracy}, seen class accuracy is {seen_class_accuracy}") - for row in matrix: if not isinstance(row, list) or len(row) != length-1: flag = False @@ -92,8 +76,9 @@ def compute(key, matrix): for j in range(length-1): if 'accuracy' in matrix[i+1][j] and 'accuracy' in matrix[i][j]: accuracy += matrix[i+1][j]['accuracy'] - BWT_score += matrix[i+1][j]['accuracy'] - matrix[i][j]['accuracy'] - + BWT_score += matrix[i+1][j]['accuracy'] - \ + matrix[i][j]['accuracy'] + for i in range(0, length-1): if 'accuracy' in matrix[i][i] and 'accuracy' in matrix[0][i]: FWT_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy'] @@ -114,6 +99,7 @@ def compute(key, matrix): return my_matrix, BWT_score, FWT_score + def bwt_func(system_metric_info: dict): """ compute BWT @@ -124,6 +110,7 @@ def bwt_func(system_metric_info: dict): _, BWT_score, _ = compute("all", info["all"]) return BWT_score + def fwt_func(system_metric_info: dict): """ compute FWT @@ -134,6 +121,7 @@ def fwt_func(system_metric_info: dict): _, _, FWT_score = compute("all", info["all"]) return FWT_score + def matrix_func(system_metric_info: dict): """ compute FWT @@ -147,6 +135,7 @@ def matrix_func(system_metric_info: dict): my_dict[key] = my_matrix return my_dict + def task_avg_acc_func(system_metric_info: dict): """ compute Task_Avg_Acc @@ -154,6 +143,7 @@ def task_avg_acc_func(system_metric_info: dict): info = system_metric_info.get(SystemMetricType.Task_Avg_Acc.value) return info["accuracy"] + def get_metric_func(metric_dict: dict): """ get metric func by metric info @@ -175,9 +165,11 @@ def get_metric_func(metric_dict: dict): if url: try: load_module(url) - metric_func = ClassFactory.get_cls(type_name=ClassType.GENERAL, t_cls_name=name) + metric_func = ClassFactory.get_cls( + type_name=ClassType.GENERAL, t_cls_name=name) return name, metric_func except Exception as err: - raise RuntimeError(f"get metric func(url={url}) failed, error: {err}.") from err + raise RuntimeError( + f"get metric func(url={url}) failed, error: {err}.") from err return name, getattr(sys.modules[__name__], str.lower(name) + "_func") diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/benchmarkingjob.yaml b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/benchmarkingjob.yaml index 4eaa1cfe..bc6c1bb1 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/benchmarkingjob.yaml +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/benchmarkingjob.yaml @@ -64,9 +64,3 @@ benchmarkingjob: # 1> "selected_and_all": save selected and all dataitems; # 2> "selected_only": save selected dataitems; save_mode: "selected_and_all" - - - - - - diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/accuracy.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/accuracy.py index 51663185..5a0205d2 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/accuracy.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/accuracy.py @@ -6,6 +6,7 @@ __all__ = ('accuracy') + @ClassFactory.register(ClassType.GENERAL) def accuracy(y_true, y_pred, **kwargs): args = val_args() @@ -35,4 +36,4 @@ def accuracy(y_true, y_pred, **kwargs): FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union() print("CPA:{}, mIoU:{}, fwIoU: {}".format(CPA, mIoU, FWIoU)) - return CPA \ No newline at end of file + return CPA diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/eval.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/eval.py index 1db7c73e..6e58d611 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/eval.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/eval.py @@ -13,16 +13,17 @@ from models.erfnet_RA_parallel import Net as Net_RAP import torch.backends.cudnn as cudnn -os.environ["OMP_NUM_THREADS"] = "1" +os.environ["OMP_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" + class Validator(object): def __init__(self, args, data=None, unseen_detection=False): self.args = args self.time_train = [] - self.num_class = args.num_class # [13, 30, 30] - self.current_domain = args.current_domain # 0 when start - self.next_domain = args.next_domain # 1 when start + self.num_class = args.num_class # [13, 30, 30] + self.current_domain = args.current_domain # 0 when start + self.next_domain = args.next_domain # 1 when start if self.current_domain <= 0: self.current_class = [self.num_class[0]] @@ -35,18 +36,19 @@ def __init__(self, args, data=None, unseen_detection=False): # Define Dataloader kwargs = {'num_workers': args.workers, 'pin_memory': False} - _, _, self.test_loader, _ = make_data_loader(args, test_data=data, **kwargs) + _, _, self.test_loader, _ = make_data_loader( + args, test_data=data, **kwargs) # Define evaluator self.evaluator = Evaluator(self.num_class[self.current_domain]) # Define network - self.model = Net_RAP(num_classes=self.current_class, nb_tasks=self.current_domain + 1, cur_task=self.current_domain) + self.model = Net_RAP(num_classes=self.current_class, + nb_tasks=self.current_domain + 1, cur_task=self.current_domain) args.current_domain = self.next_domain args.next_domain += 1 if args.cuda: - #self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids) self.model = self.model.cuda(args.gpu_ids) cudnn.benchmark = True # accelarate speed print('Model loaded successfully!') @@ -60,32 +62,31 @@ def validate(self): if self.args.depth: image, depth, target = sample['image'], sample['depth'], sample['label'] else: - # spec = time.time() - image, target = sample['image'], sample['label'] - #print(self.args.cuda, self.args.gpu_ids) + image, target = sample['image'], sample['label'] + if self.args.cuda: image = image.cuda(self.args.gpu_ids) if self.args.depth: depth = depth.cuda(self.args.gpu_ids) - + with torch.no_grad(): if self.args.depth: output = self.model(image, depth) else: - output = self.model(image,self.current_domain) - + output = self.model(image, self.current_domain) + if self.args.cuda: torch.cuda.synchronize() pred = output.data.cpu().numpy() - # todo pred = np.argmax(pred, axis=1) predictions.append(pred) if not self.args.save_predicted_image: continue - - pre_colors = Colorize()(torch.max(output, 1)[1].detach().cpu().byte()) + + pre_colors = Colorize()(torch.max(output, 1)[ + 1].detach().cpu().byte()) pre_labels = torch.max(output, 1)[1].detach().cpu().byte() print(pre_labels.shape) # save @@ -97,15 +98,17 @@ def validate(self): else: img_name = os.path.basename(image_name[0]) - color_label_name = os.path.join(self.args.color_label_save_path, img_name) + color_label_name = os.path.join( + self.args.color_label_save_path, img_name) label_name = os.path.join(self.args.label_save_path, img_name) - merge_label_name = os.path.join(self.args.merge_label_save_path, img_name) + merge_label_name = os.path.join( + self.args.merge_label_save_path, img_name) os.makedirs(os.path.dirname(color_label_name), exist_ok=True) os.makedirs(os.path.dirname(merge_label_name), exist_ok=True) os.makedirs(os.path.dirname(label_name), exist_ok=True) - pre_color_image = ToPILImage()(pre_colors[i]) # pre_colors.dtype = float64 + pre_color_image = ToPILImage()(pre_colors[i]) pre_color_image.save(color_label_name) pre_label_image = ToPILImage()(pre_labels[i]) @@ -114,9 +117,9 @@ def validate(self): if (self.args.merge): image_merge(image[i], pre_color_image, merge_label_name) print('save image: {}'.format(merge_label_name)) - #print("start validating 120") + return predictions - + def task_divide(self): seen_task_samples, unseen_task_samples = [], [] self.model.eval() @@ -149,7 +152,8 @@ def task_divide(self): time.sleep(0.1) # to avoid overheating the GPU too much # pred colorize - pre_colors = Colorize()(torch.max(output, 1)[1].detach().cpu().byte()) + pre_colors = Colorize()(torch.max(output, 1)[ + 1].detach().cpu().byte()) pre_labels = torch.max(output, 1)[1].detach().cpu().byte() for i in range(pre_colors.shape[0]): task_sample = dict() @@ -164,7 +168,8 @@ def task_divide(self): seen_task_samples.append((task_sample, image_name[i])) return seen_task_samples, unseen_task_samples - + + def image_merge(image, label, save_name): image = ToPILImage()(image.detach().cpu().byte()) # width, height = image.size @@ -182,11 +187,12 @@ def image_merge(image, label, save_name): image = Image.blend(image, label, 0.6) image.save(save_name) -def load_my_state_dict(model, state_dict): # custom function to load model when not all dict elements + +# custom function to load model when not all dict elements +def load_my_state_dict(model, state_dict): own_state = model.state_dict() for name, param in state_dict.items(): if name not in own_state: - # print('{} not in model_state'.format(name)) continue else: own_state[name].copy_(param) diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/models/erfnet.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/models/erfnet.py index fe9b0f7d..a035e80e 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/models/erfnet.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/models/erfnet.py @@ -8,13 +8,16 @@ import torch.nn.init as init import torch.nn.functional as F + class DownsamplerBlock (nn.Module): def __init__(self, ninput, noutput, nb_tasks=1): super().__init__() - self.conv = nn.Conv2d(ninput, noutput-ninput, (3, 3), stride=2, padding=1, bias=True) + self.conv = nn.Conv2d(ninput, noutput-ninput, + (3, 3), stride=2, padding=1, bias=True) self.pool = nn.MaxPool2d(2, stride=2) - self.bn_ini = nn.ModuleList([nn.BatchNorm2d(noutput, eps=1e-3) for i in range(nb_tasks)]) + self.bn_ini = nn.ModuleList( + [nn.BatchNorm2d(noutput, eps=1e-3) for i in range(nb_tasks)]) def forward(self, input): task = current_task @@ -27,9 +30,11 @@ class non_bottleneck_1d (nn.Module): def __init__(self, chann, dropprob, dilated): super().__init__() - self.conv3x1_1 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=(1, 0), bias=True) + self.conv3x1_1 = nn.Conv2d( + chann, chann, (3, 1), stride=1, padding=(1, 0), bias=True) - self.conv1x3_1 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=(0, 1), bias=True) + self.conv1x3_1 = nn.Conv2d( + chann, chann, (1, 3), stride=1, padding=(0, 1), bias=True) self.bn1 = nn.BatchNorm2d(chann, eps=1e-03) @@ -64,15 +69,19 @@ def forward(self, input): class non_bottleneck_1d_RAP (nn.Module): def __init__(self, chann, dropprob, dilated, nb_tasks=1): - #chann = #channels, dropprob=dropout probability, dilated=dilation rate + # chann = #channels, dropprob=dropout probability, dilated=dilation rate super().__init__() - self.conv3x1_1 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=(1, 0), bias=True) - self.conv1x3_1 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=(0, 1), bias=True) + self.conv3x1_1 = nn.Conv2d( + chann, chann, (3, 1), stride=1, padding=(1, 0), bias=True) + self.conv1x3_1 = nn.Conv2d( + chann, chann, (1, 3), stride=1, padding=(0, 1), bias=True) # domain-specific 1x1conv - self.parallel_conv_1 = nn.ModuleList([nn.Conv2d(chann, chann, kernel_size=1, stride=1, padding=0, bias=True) for i in range(nb_tasks)]) #nb_tasks=1 for 1st time, its only on CS - self.bns_1 = nn.ModuleList([nn.BatchNorm2d(chann, eps=1e-03) for i in range(nb_tasks)]) + self.parallel_conv_1 = nn.ModuleList([nn.Conv2d(chann, chann, kernel_size=1, stride=1, padding=0, bias=True) for i in range( + nb_tasks)]) # nb_tasks=1 for 1st time, its only on CS + self.bns_1 = nn.ModuleList( + [nn.BatchNorm2d(chann, eps=1e-03) for i in range(nb_tasks)]) self.conv3x1_2 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=( 1*dilated, 0), bias=True, dilation=(dilated, 1)) @@ -80,8 +89,10 @@ def __init__(self, chann, dropprob, dilated, nb_tasks=1): self.conv1x3_2 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=( 0, 1*dilated), bias=True, dilation=(1, dilated)) - self.parallel_conv_2 = nn.ModuleList([nn.Conv2d(chann, chann, kernel_size=1, stride=1, padding=0, bias=True) for i in range(nb_tasks)]) - self.bns_2 = nn.ModuleList([nn.BatchNorm2d(chann, eps=1e-03) for i in range(nb_tasks)]) + self.parallel_conv_2 = nn.ModuleList([nn.Conv2d( + chann, chann, kernel_size=1, stride=1, padding=0, bias=True) for i in range(nb_tasks)]) + self.bns_2 = nn.ModuleList( + [nn.BatchNorm2d(chann, eps=1e-03) for i in range(nb_tasks)]) self.dropout = nn.Dropout2d(dropprob) @@ -93,7 +104,8 @@ def forward(self, input): output = self.conv1x3_1(output) # print('output 2nd 1x3: ', output.size()) - output = output + self.parallel_conv_1[task](input) # RAP skip connection for conv2 + # RAP skip connection for conv2 + output = output + self.parallel_conv_1[task](input) output = self.bns_1[task](output) output_ = F.relu(output) @@ -102,7 +114,8 @@ def forward(self, input): output = F.relu(output) output = self.conv1x3_2(output) - output = output + self.parallel_conv_2[task](output_) # RAP skip connection for conv2 + # RAP skip connection for conv2 + output = output + self.parallel_conv_2[task](output_) output = self.bns_2[task](output) if (self.dropout.p != 0): @@ -110,6 +123,7 @@ def forward(self, input): return F.relu(output+input) # +input = identity (residual connection) + ''' ENCODER will use the non_bottleneck_1d_RAP modules as they have the parallel residual adapters. DECODER will use non_bottleneck_1d modules as they don't have RAPs and we need RAPs only in the encoder. @@ -118,6 +132,8 @@ def forward(self, input): it'll be like decoder.0, decoder.1 for domain-specific RAPs and bns, it'll be like parallel_conv_2.0.weight, parallel_conv_2.1.weight ''' + + class Encoder(nn.Module): def __init__(self, nb_tasks=1): super().__init__() @@ -133,7 +149,8 @@ def __init__(self, nb_tasks=1): self.layers.append(DownsamplerBlock(64, 128, nb_tasks)) for x in range(0, 2): # 2 times - self.layers.append(non_bottleneck_1d_RAP(128, 0.3, 2, nb_tasks)) # dropprob for imagenet pretrained encoder is 0.1 not 0.3, here using 0.3 for imagenet pretrained encoder + # dropprob for imagenet pretrained encoder is 0.1 not 0.3, here using 0.3 for imagenet pretrained encoder + self.layers.append(non_bottleneck_1d_RAP(128, 0.3, 2, nb_tasks)) self.layers.append(non_bottleneck_1d_RAP(128, 0.3, 4, nb_tasks)) self.layers.append(non_bottleneck_1d_RAP(128, 0.3, 8, nb_tasks)) self.layers.append(non_bottleneck_1d_RAP(128, 0.3, 16, nb_tasks)) @@ -187,10 +204,11 @@ def forward(self, input): return output -# ERFNet +# ERFNet class Net(nn.Module): - def __init__(self, num_classes = [13], nb_tasks=1, cur_task=0): # use encoder to pass pretrained encoder + # use encoder to pass pretrained encoder + def __init__(self, num_classes=[13], nb_tasks=1, cur_task=0): # the encoder has been passed here in this manner because we needed an Imagenet pretrained encoder. so used erfnet_imagenet to initialize encoder and read it from saved pretrained model. we want to attach that encoder to our decoder # encoder is not being passed. figure out another way of initialising with the imagenet pretrained encoder weights, on this encoder. init to be handled. super().__init__() @@ -200,11 +218,13 @@ def __init__(self, num_classes = [13], nb_tasks=1, cur_task=0): # use encoder t self.encoder = Encoder(nb_tasks) - self.decoder = nn.ModuleList([Decoder(num_classes[i]) for i in range(nb_tasks)]) + self.decoder = nn.ModuleList( + [Decoder(num_classes[i]) for i in range(nb_tasks)]) def forward(self, input, task): global current_task - current_task = task # chose which branch of forward pass you need based on if training on current dataset or validating on a previous dataset. + # chose which branch of forward pass you need based on if training on current dataset or validating on a previous dataset. + current_task = task output = self.encoder(input) output = self.decoder[task].forward(output) return output diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/train.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/train.py index ecd87553..df9bd564 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/train.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/train.py @@ -19,9 +19,10 @@ from utils.metrics import Evaluator from sedna.datasources import BaseDataSource -os.environ["OMP_NUM_THREADS"] = "1" +os.environ["OMP_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" + class Trainer(object): def __init__(self, args, train_data=None, valid_data=None): self.args = args @@ -32,12 +33,15 @@ def __init__(self, args, train_data=None, valid_data=None): self.summary = TensorboardSummary(self.saver.experiment_dir) self.writer = self.summary.create_summary() # denormalize for detph image - self.mean_depth = torch.as_tensor(0.12176, dtype=torch.float32, device='cpu') - self.std_depth = torch.as_tensor(0.09752, dtype=torch.float32, device='cpu') + self.mean_depth = torch.as_tensor( + 0.12176, dtype=torch.float32, device='cpu') + self.std_depth = torch.as_tensor( + 0.09752, dtype=torch.float32, device='cpu') - self.nclass = args.num_class # [13, 30, 30] - self.current_domain = min(args.current_domain, 2) # current domain start from 0 and maximum is 2 - self.next_domain = args.next_domain # next_domain start from 1 + self.nclass = args.num_class # [13, 30, 30] + # current domain start from 0 and maximum is 2 + self.current_domain = min(args.current_domain, 2) + self.next_domain = args.next_domain # next_domain start from 1 if self.current_domain <= 0: self.current_class = [self.nclass[0]] @@ -50,37 +54,43 @@ def __init__(self, args, train_data=None, valid_data=None): # Define Dataloader kwargs = {'num_workers': args.workers, 'pin_memory': True} - self.train_loader, self.val_loader, self.test_loader, _ = make_data_loader(args, train_data=train_data, - valid_data=valid_data, **kwargs) + self.train_loader, self.val_loader, self.test_loader, _ = make_data_loader(args, train_data=train_data, + valid_data=valid_data, **kwargs) self.print_domain_info() # Define network - model = Net_RAP(num_classes=self.current_class, nb_tasks=self.current_domain + 1, cur_task=self.current_domain) - model_old = Net_RAP(num_classes=self.current_class, nb_tasks=self.current_domain, cur_task=max(self.current_domain-1, 0)) + model = Net_RAP(num_classes=self.current_class, + nb_tasks=self.current_domain + 1, cur_task=self.current_domain) + model_old = Net_RAP(num_classes=self.current_class, + nb_tasks=self.current_domain, cur_task=max(self.current_domain-1, 0)) args.current_domain = self.next_domain args.next_domain += 1 optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, - weight_decay=args.weight_decay) + weight_decay=args.weight_decay) # Define Criterion # whether to use class balanced weights if args.use_balanced_weights: - classes_weights_path = os.path.join(Path.db_root_dir(args.dataset), args.dataset+'_classes_weights.npy') + classes_weights_path = os.path.join(Path.db_root_dir( + args.dataset), args.dataset+'_classes_weights.npy') if os.path.isfile(classes_weights_path): weight = np.load(classes_weights_path) else: - weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass) + weight = calculate_weigths_labels( + args.dataset, self.train_loader, self.nclass) weight = torch.from_numpy(weight.astype(np.float32)) else: weight = None # Define loss function - self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda, gpu_ids=args.gpu_ids).build_loss(mode=args.loss_type) + self.criterion = SegmentationLosses( + weight=weight, cuda=args.cuda, gpu_ids=args.gpu_ids).build_loss(mode=args.loss_type) self.model, self.model_old, self.optimizer = model, model_old, optimizer # Define Evaluator self.evaluator = Evaluator(self.nclass[self.current_domain]) # # Define lr scheduler - self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr, args.epochs, len(self.train_loader)) + self.scheduler = LR_Scheduler( + args.lr_scheduler, args.lr, args.epochs, len(self.train_loader)) # Using cuda if args.cuda: self.model = torch.nn.DataParallel(self.model) @@ -91,9 +101,11 @@ def __init__(self, args, train_data=None, valid_data=None): self.best_pred = 0.0 if args.resume is not None: if not os.path.isfile(args.resume): - raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume)) + raise RuntimeError( + "=> no checkpoint found at '{}'" .format(args.resume)) print(f"Training: load model from {args.resume}") - checkpoint = torch.load(args.resume, map_location=torch.device('cuda:0')) + checkpoint = torch.load( + args.resume, map_location=torch.device('cuda:0')) args.start_epoch = checkpoint['epoch'] self.model.load_state_dict(checkpoint['state_dict'], False) @@ -101,7 +113,8 @@ def __init__(self, args, train_data=None, valid_data=None): if not args.ft: self.optimizer.load_state_dict(checkpoint['optimizer']) self.best_pred = checkpoint['best_pred'] - print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch'])) + print("=> loaded checkpoint '{}' (epoch {})".format( + args.resume, checkpoint['epoch'])) # Clear start epoch if fine-tuning if args.ft: @@ -111,18 +124,20 @@ def get_weight(self): print("get weight") current_model = copy.deepcopy(self.model) return current_model.parameters() - + def set_weight(self, weights): length = len(weights) print("set weight", length) print("model:", self.args.resume) tau = 0.2 - if length == 1: + if length == 1: for param, target_param in zip(weights[0], self.model.parameters()): - target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data) + target_param.data.copy_( + tau * param.data + (1 - tau) * target_param.data) elif length == 2: for param1, param2, target_param in zip(weights[0], weights[1], self.model.parameters()): - target_param.data.copy_(0.5 * tau * param1.data + 0.5 * tau * param2.data + (1 - tau) * target_param.data) + target_param.data.copy_( + 0.5 * tau * param1.data + 0.5 * tau * param2.data + (1 - tau) * target_param.data) def my_training(self, epoch): train_loss = 0.0 @@ -135,12 +150,12 @@ def my_training(self, epoch): for i, sample in enumerate(tbar): if self.args.depth: image, depth, target = sample['image'], sample['depth'], sample['label'] - #print(target.shape) else: image, target = sample['image'], sample['label'] print(image.shape) if self.args.cuda: - image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids) + image, target = image.cuda( + self.args.gpu_ids), target.cuda(self.args.gpu_ids) if self.args.depth: depth = depth.cuda(self.args.gpu_ids) self.scheduler(self.optimizer, i, epoch, self.best_pred) @@ -154,35 +169,40 @@ def my_training(self, epoch): loss = self.criterion(output, target) loss.backward() self.optimizer.step() - #print(self.optimizer.state_dict()['param_groups'][0]['lr']) train_loss += loss.item() tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1))) - self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch) + self.writer.add_scalar( + 'train/total_loss_iter', loss.item(), i + num_img_tr * epoch) # Show 10 * 3 inference results each epoch if i % (num_img_tr // 10 + 1) == 0: global_step = i + num_img_tr * epoch if self.args.depth: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) + self.summary.visualize_image( + self.writer, self.args.dataset, image, target, output, global_step) depth_display = depth[0].cpu().unsqueeze(0) - depth_display = depth_display.mul_(self.std_depth).add_(self.mean_depth) + depth_display = depth_display.mul_( + self.std_depth).add_(self.mean_depth) depth_display = depth_display.numpy() depth_display = depth_display*255 depth_display = depth_display.astype(np.uint8) self.writer.add_image('Depth', depth_display, global_step) else: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) + self.summary.visualize_image( + self.writer, self.args.dataset, image, target, output, global_step) self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch) - print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) + print('[Epoch: %d, numImages: %5d]' % + (epoch, i * self.args.batch_size + image.data.shape[0])) print('Loss: %.3f' % train_loss) tau = 0.3 flag = True for param, target_param in zip(current_model.parameters(), self.model.parameters()): if flag: flag = False - target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data) + target_param.data.copy_( + tau * param.data + (1 - tau) * target_param.data) del current_model return train_loss @@ -217,12 +237,11 @@ def training(self, epoch): for i, sample in enumerate(tbar): if self.args.depth: image, depth, target = sample['image'], sample['depth'], sample['label'] - #print(target.shape) else: image, target = sample['image'], sample['label'] - # print(image.shape) if self.args.cuda: - image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids) + image, target = image.cuda( + self.args.gpu_ids), target.cuda(self.args.gpu_ids) if self.args.depth: depth = depth.cuda(self.args.gpu_ids) self.scheduler(self.optimizer, i, epoch, self.best_pred) @@ -233,17 +252,19 @@ def training(self, epoch): else: output = self.model(image, self.current_domain) - output = torch.tensor(output, dtype=torch.float32) + output = torch.tensor(output, dtype=torch.float32) target[target > self.nclass[self.current_domain]-1] = 255 target = self.my_to_label(target) - target = self.my_relabel(target, 255, self.nclass[self.current_domain] - 1) + target = self.my_relabel( + target, 255, self.nclass[self.current_domain] - 1) target = target.squeeze(0) target = target.cuda(self.gpu_ids) - outputs_prev_task = self.model(image, max(self.current_domain-1, 0)) - loss = self.criterion(output, target) + outputs_prev_task = self.model( + image, max(self.current_domain-1, 0)) + loss = self.criterion(output, target) loss.requires_grad_(True) loss.backward() @@ -251,35 +272,40 @@ def training(self, epoch): train_loss += loss.item() tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1))) - self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch) + self.writer.add_scalar( + 'train/total_loss_iter', loss.item(), i + num_img_tr * epoch) # Show 10 * 3 inference results each epoch if i % (num_img_tr // 10 + 1) == 0: global_step = i + num_img_tr * epoch if self.args.depth: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) + self.summary.visualize_image( + self.writer, self.args.dataset, image, target, output, global_step) depth_display = depth[0].cpu().unsqueeze(0) - depth_display = depth_display.mul_(self.std_depth).add_(self.mean_depth) + depth_display = depth_display.mul_( + self.std_depth).add_(self.mean_depth) depth_display = depth_display.numpy() depth_display = depth_display*255 depth_display = depth_display.astype(np.uint8) self.writer.add_image('Depth', depth_display, global_step) else: - self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step) + self.summary.visualize_image( + self.writer, self.args.dataset, image, target, output, global_step) self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch) - print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) + print('[Epoch: %d, numImages: %5d]' % + (epoch, i * self.args.batch_size + image.data.shape[0])) print('Loss: %.3f' % train_loss) # save checkpoint every epoch checkpoint_path = self.saver.save_checkpoint({ - 'epoch': epoch + 1, - 'state_dict': self.model.state_dict(), - 'optimizer': self.optimizer.state_dict(), - 'best_pred': self.best_pred, - }, True) + 'epoch': epoch + 1, + 'state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'best_pred': self.best_pred, + }, True) return train_loss def validation(self, epoch): @@ -294,7 +320,8 @@ def validation(self, epoch): image, target = sample['image'], sample['label'] # print(f"val image is {image}") if self.args.cuda: - image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids) + image, target = image.cuda( + self.args.gpu_ids), target.cuda(self.args.gpu_ids) if self.args.depth: depth = depth.cuda(self.args.gpu_ids) with torch.no_grad(): @@ -323,8 +350,10 @@ def validation(self, epoch): self.writer.add_scalar('val/Acc_class', Acc_class, epoch) self.writer.add_scalar('val/fwIoU', FWIoU, epoch) print('Validation:') - print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) - print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU)) + print('[Epoch: %d, numImages: %5d]' % + (epoch, i * self.args.batch_size + image.data.shape[0])) + print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format( + Acc, Acc_class, mIoU, FWIoU)) print('Loss: %.3f' % test_loss) new_pred = mIoU @@ -338,9 +367,8 @@ def validation(self, epoch): 'best_pred': self.best_pred, }, is_best) - def print_domain_info(self): - + domain_map = { 0: "Synthia", 1: "CityScapes", @@ -358,4 +386,3 @@ def my_relabel(self, tensor, olabel, nlabel): def my_to_label(self, image): image = image.cpu() return torch.from_numpy(np.array(image)).long().unsqueeze(0) - diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/basemodel.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/basemodel.py index 97aca3de..838851a6 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/basemodel.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/basemodel.py @@ -19,9 +19,10 @@ # set backend os.environ['BACKEND_TYPE'] = 'PYTORCH' -os.environ["OMP_NUM_THREADS"] = "1" +os.environ["OMP_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" + @ClassFactory.register(ClassType.GENERAL, alias="BaseModel") class BaseModel: def __init__(self, **kwargs): @@ -29,18 +30,21 @@ def __init__(self, **kwargs): self.trainer = None self.val_args = ValArgs(**kwargs) - label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") - self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") - self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + label_save_dir = Context.get_parameters( + "INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join( + label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join( + label_save_dir, "merge") self.val_args.label_save_path = os.path.join(label_save_dir, "label") self.validator = Validator(self.val_args) def get_weights(self): return self.trainer.get_weight() - + def set_weights(self, weights): self.trainer.set_weight(weights) - + epoch_num = 0 print("Total epoch: ", epoch_num) loss_all = [] @@ -62,8 +66,8 @@ def train(self, train_data, valid_data=None, **kwargs): if self.trainer.args.no_val and ( epoch % self.trainer.args.eval_interval == ( - self.trainer.args.eval_interval - - 1) or epoch == self.trainer.args.epochs - + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - 1): is_best = False self.train_model_url = self.trainer.saver.save_checkpoint({ @@ -79,8 +83,10 @@ def train(self, train_data, valid_data=None, **kwargs): def predict(self, data, **kwargs): if len(data) > 10: print("predict start for big data") - my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} - _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + my_kwargs = {'num_workers': self.val_args.workers, + 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader( + self.val_args, test_data=data, **my_kwargs) else: print("predict start for small data") if not isinstance(data[0][0], dict): @@ -88,28 +94,32 @@ def predict(self, data, **kwargs): if type(data) is np.ndarray: data = data.tolist() self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, - pin_memory=True) - + pin_memory=True) + return self.validator.validate() def evaluate(self, data, **kwargs): - self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + self.val_args.save_predicted_image = kwargs.get( + "save_predicted_image", True) samples = self._preprocess(data.x) predictions = self.predict(samples) metric_name, metric_func = kwargs.get("metric") if callable(metric_func): return metric_func(data.y, predictions) else: - raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + raise Exception( + f"not found model metric func(name={metric_name}) in model eval phase") def load(self, model_url, **kwargs): if model_url: - print("load model url: ",model_url) - self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + print("load model url: ", model_url) + self.validator.new_state_dict = torch.load( + model_url, map_location=torch.device("cpu")) self.train_args.resume = model_url else: raise Exception("model url does not exist.") - self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + self.validator.model = load_my_state_dict( + self.validator.model, self.validator.new_state_dict['state_dict']) def save(self, model_path=None): if not model_path: @@ -134,7 +144,8 @@ def _preprocess(self, image_urls): del _img gc.collect() composed_transforms = transforms.Compose([ - tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.Normalize(mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225)), tr.ToTensor()]) transformed_images.append((composed_transforms(sample), img_path)) diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_allocation_by_domain.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_allocation_by_domain.py index dc4d9522..ab44f1ab 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_allocation_by_domain.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_allocation_by_domain.py @@ -22,12 +22,14 @@ def __init__(self, **kwargs): self.default_origin = kwargs.get("default", None) def __call__(self, task_extractor, samples: BaseDataSource): - self.task_extractor = {"Synthia": 0, "Cityscapes": 1, "Cloud-Robotics": 2} # Mapping of origins to task indices + # Mapping of origins to task indices + self.task_extractor = {"Synthia": 0, "Cityscapes": 1, "Cloud-Robotics": 2} if self.default_origin: return samples, [int(self.task_extractor.get(self.default_origin))] * len(samples.x) - categories = ["Cityscapes", "Synthia", "Cloud-Robotics"] # List of all possible origins + # List of all possible origins + categories = ["Cityscapes", "Synthia", "Cloud-Robotics"] sample_origins = [] for _x in samples.x: @@ -41,6 +43,7 @@ def __call__(self, task_extractor, samples: BaseDataSource): sample_origin = self.default_origin if self.default_origin else categories[0] sample_origins.append(sample_origin) - allocations = [int(self.task_extractor.get(sample_origin)) for sample_origin in sample_origins] + allocations = [int(self.task_extractor.get(sample_origin)) + for sample_origin in sample_origins] return samples, allocations diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_definition_by_domain.py b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_definition_by_domain.py index eba0f74f..68e64851 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_definition_by_domain.py +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/task_definition_by_domain.py @@ -34,7 +34,8 @@ def __call__(self, task_index = dict(zip(categories, range(len(categories)))) - data_sources = {category: BaseDataSource(data_type=d_type) for category in categories} + data_sources = {category: BaseDataSource( + data_type=d_type) for category in categories} print(data_sources) for category in data_sources.values(): @@ -50,7 +51,8 @@ def __call__(self, for category, data_source in data_sources.items(): task_name = f"{category}_semantic_segmentation_model" - task_obj = Task(entry=task_name, samples=data_source, meta_attr=category) + task_obj = Task(entry=task_name, + samples=data_source, meta_attr=category) tasks.append(task_obj) - return tasks, task_index, samples \ No newline at end of file + return tasks, task_index, samples From 94d7f2b8bf34aa7479579ca6c1aebc143fba788e Mon Sep 17 00:00:00 2001 From: qxygxt Date: Fri, 26 Jan 2024 18:54:30 +0800 Subject: [PATCH 2/3] Optimize the naming convention for test metrics Signed-off-by: qxygxt --- core/common/constant.py | 4 ++-- .../paradigm/lifelong_learning/lifelong_learning.py | 12 ++++++------ core/testcasecontroller/metrics/metrics.py | 10 +++++----- .../semantic-segmentation/testenv/testenv.yaml | 3 ++- .../semantic-segmentation/benchmarkingjob-sam.yaml | 2 +- .../benchmarkingjob-simple.yaml | 2 +- .../testenv/testenv-robot-small.yaml | 4 ++-- .../semantic-segmentation/testenv/testenv-robot.yaml | 4 ++-- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/core/common/constant.py b/core/common/constant.py index 368e34fb..5444e289 100644 --- a/core/common/constant.py +++ b/core/common/constant.py @@ -71,8 +71,8 @@ class SystemMetricType(Enum): SAMPLES_TRANSFER_RATIO = "samples_transfer_ratio" FWT = "FWT" BWT = "BWT" - Task_Avg_Acc = "Task_Avg_Acc" - Matrix = "Matrix" + TASK_AVG_ACC = "task_avg_acc" + MATRIX = "MATRIX" class TestObjectType(Enum): diff --git a/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py b/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py index 2e50ba7d..c49efeca 100644 --- a/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py +++ b/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py @@ -64,8 +64,8 @@ def __init__(self, workspace, **kwargs): self.cloud_task_index = '/tmp/cloud_task/index.pkl' self.edge_task_index = '/tmp/edge_task/index.pkl' self.system_metric_info = {SystemMetricType.SAMPLES_TRANSFER_RATIO.value: [], - SystemMetricType.Matrix.value : {}, - SystemMetricType.Task_Avg_Acc.value: {}} + SystemMetricType.MATRIX.value : {}, + SystemMetricType.TASK_AVG_ACC.value: {}} def run(self): # pylint:disable=duplicate-code @@ -147,7 +147,7 @@ def run(self): LOGGER.info(f"{entry} scores: {scores}") task_avg_score['accuracy'] += scores['accuracy'] task_avg_score['accuracy'] = task_avg_score['accuracy']/i - self.system_metric_info[SystemMetricType.Task_Avg_Acc.value] = task_avg_score + self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value] = task_avg_score LOGGER.info(task_avg_score) job = self.build_paradigm_job(ParadigmType.LIFELONG_LEARNING.value) inference_dataset = self.dataset.load_data(self.dataset.test_url, "eval", @@ -160,7 +160,7 @@ def run(self): for key in my_dict.keys(): matrix = my_dict[key] #BWT, FWT = self.compute(key, matrix) - self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix + self.system_metric_info[SystemMetricType.MATRIX.value][key] = matrix elif mode == 'hard-example-mining': dataset_files = self._split_dataset(splitting_dataset_times=rounds) @@ -246,7 +246,7 @@ def run(self): LOGGER.info(f"{entry} scores: {scores}") task_avg_score['accuracy'] += scores['accuracy'] task_avg_score['accuracy'] = task_avg_score['accuracy']/i - self.system_metric_info[SystemMetricType.Task_Avg_Acc.value] = task_avg_score + self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value] = task_avg_score LOGGER.info(task_avg_score) test_res, unseen_task_train_samples = self._inference(self.edge_task_index, self.dataset.test_url, @@ -256,7 +256,7 @@ def run(self): for key in my_dict.keys(): matrix = my_dict[key] #BWT, FWT = self.compute(key, matrix) - self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix + self.system_metric_info[SystemMetricType.MATRIX.value][key] = matrix elif mode != 'multi-inference': dataset_files = self._split_dataset(splitting_dataset_times=rounds) diff --git a/core/testcasecontroller/metrics/metrics.py b/core/testcasecontroller/metrics/metrics.py index 7c53f8c3..7b3265aa 100644 --- a/core/testcasecontroller/metrics/metrics.py +++ b/core/testcasecontroller/metrics/metrics.py @@ -106,7 +106,7 @@ def bwt_func(system_metric_info: dict): """ # pylint: disable=C0103 # pylint: disable=W0632 - info = system_metric_info.get(SystemMetricType.Matrix.value) + info = system_metric_info.get(SystemMetricType.MATRIX.value) _, BWT_score, _ = compute("all", info["all"]) return BWT_score @@ -117,7 +117,7 @@ def fwt_func(system_metric_info: dict): """ # pylint: disable=C0103 # pylint: disable=W0632 - info = system_metric_info.get(SystemMetricType.Matrix.value) + info = system_metric_info.get(SystemMetricType.MATRIX.value) _, _, FWT_score = compute("all", info["all"]) return FWT_score @@ -128,7 +128,7 @@ def matrix_func(system_metric_info: dict): """ # pylint: disable=C0103 # pylint: disable=W0632 - info = system_metric_info.get(SystemMetricType.Matrix.value) + info = system_metric_info.get(SystemMetricType.MATRIX.value) my_dict = {} for key in info.keys(): my_matrix, _, _ = compute(key, info[key]) @@ -138,9 +138,9 @@ def matrix_func(system_metric_info: dict): def task_avg_acc_func(system_metric_info: dict): """ - compute Task_Avg_Acc + compute task average accuracy """ - info = system_metric_info.get(SystemMetricType.Task_Avg_Acc.value) + info = system_metric_info.get(SystemMetricType.TASK_AVG_ACC.value) return info["accuracy"] diff --git a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testenv/testenv.yaml b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testenv/testenv.yaml index d00b4aa6..e3a48b22 100644 --- a/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testenv/testenv.yaml +++ b/examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testenv/testenv.yaml @@ -30,9 +30,10 @@ testenv: # the url address of python file url: "./examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/accuracy.py" - name: "samples_transfer_ratio" + - name: "task_avg_acc" - name: "BWT" - name: "FWT" - - name: "Matrix" + - name: "MATRIX" # incremental rounds setting; int type; default value is 2; incremental_rounds: 3 \ No newline at end of file diff --git a/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-sam.yaml b/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-sam.yaml index 01cb80c2..ac049581 100644 --- a/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-sam.yaml +++ b/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-sam.yaml @@ -67,7 +67,7 @@ benchmarkingjob: # currently the options of value are as follows: # 1> "all": select all metrics in the leaderboard; # 2> metrics in the leaderboard, e.g., "F1_SCORE" - metrics: [ "accuracy", "Task_Avg_Acc"] + metrics: [ "accuracy", "task_avg_acc"] # model of save selected and all dataitems in workspace `./rank` ; string type; # currently the options of value are as follows: diff --git a/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-simple.yaml b/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-simple.yaml index 24f795a3..b0656c7a 100644 --- a/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-simple.yaml +++ b/examples/robot/lifelong_learning_bench/semantic-segmentation/benchmarkingjob-simple.yaml @@ -57,7 +57,7 @@ benchmarkingjob: # currently the options of value are as follows: # 1> "all": select all metrics in the leaderboard; # 2> metrics in the leaderboard, e.g., "F1_SCORE" - metrics: [ "accuracy", "Task_Avg_Acc", "BWT", "FWT"] + metrics: [ "accuracy", "task_avg_acc", "BWT", "FWT"] # model of save selected and all dataitems in workspace `./rank` ; string type; # currently the options of value are as follows: diff --git a/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot-small.yaml b/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot-small.yaml index cde00454..35a37275 100644 --- a/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot-small.yaml +++ b/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot-small.yaml @@ -31,10 +31,10 @@ testenv: # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - name: "samples_transfer_ratio" - - name: "Task_Avg_Acc" + - name: "task_avg_acc" - name: "BWT" - name: "FWT" - - name: "Matrix" + - name: "MATRIX" # incremental rounds setting; int type; default value is 2; incremental_rounds: 3 \ No newline at end of file diff --git a/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot.yaml b/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot.yaml index e8645110..487a677b 100644 --- a/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot.yaml +++ b/examples/robot/lifelong_learning_bench/semantic-segmentation/testenv/testenv-robot.yaml @@ -31,10 +31,10 @@ testenv: # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - name: "samples_transfer_ratio" - - name: "Task_Avg_Acc" + - name: "task_avg_acc" - name: "BWT" - name: "FWT" - - name: "Matrix" + - name: "MATRIX" # incremental rounds setting; int type; default value is 2; incremental_rounds: 5 \ No newline at end of file From 89636a3069d4c5f3975ae536de9e148e57e7b5be Mon Sep 17 00:00:00 2001 From: qxygxt Date: Fri, 26 Jan 2024 19:01:14 +0800 Subject: [PATCH 3/3] Optimize variable names in metrics.py Signed-off-by: qxygxt --- core/testcasecontroller/metrics/metrics.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/core/testcasecontroller/metrics/metrics.py b/core/testcasecontroller/metrics/metrics.py index 7b3265aa..4e9c886a 100644 --- a/core/testcasecontroller/metrics/metrics.py +++ b/core/testcasecontroller/metrics/metrics.py @@ -58,8 +58,8 @@ def compute(key, matrix): length = len(matrix) accuracy = 0.0 - BWT_score = 0.0 - FWT_score = 0.0 + bwt_score = 0.0 + fwt_score = 0.0 flag = True for row in matrix: @@ -68,27 +68,27 @@ def compute(key, matrix): break if not flag: - BWT_score = np.nan - FWT_score = np.nan - return BWT_score, FWT_score + bwt_score = np.nan + fwt_score = np.nan + return bwt_score, fwt_score for i in range(length-1): for j in range(length-1): if 'accuracy' in matrix[i+1][j] and 'accuracy' in matrix[i][j]: accuracy += matrix[i+1][j]['accuracy'] - BWT_score += matrix[i+1][j]['accuracy'] - \ + bwt_score += matrix[i+1][j]['accuracy'] - \ matrix[i][j]['accuracy'] for i in range(0, length-1): if 'accuracy' in matrix[i][i] and 'accuracy' in matrix[0][i]: - FWT_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy'] + fwt_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy'] accuracy = accuracy / ((length-1) * (length-1)) - BWT_score = BWT_score / ((length-1) * (length-1)) - FWT_score = FWT_score / (length-1) + bwt_score = bwt_score / ((length-1) * (length-1)) + fwt_score = fwt_score / (length-1) - print(f"{key} BWT_score: {BWT_score}") - print(f"{key} FWT_score: {FWT_score}") + print(f"{key} BWT_score: {bwt_score}") + print(f"{key} FWT_score: {fwt_score}") my_matrix = [] for i in range(length-1): @@ -97,7 +97,7 @@ def compute(key, matrix): if 'accuracy' in matrix[i+1][j]: my_matrix[i].append(matrix[i+1][j]['accuracy']) - return my_matrix, BWT_score, FWT_score + return my_matrix, bwt_score, fwt_score def bwt_func(system_metric_info: dict):