From 844fa1418a2c35b72f04e714e5b5c247a1c12b77 Mon Sep 17 00:00:00 2001 From: Caroline Malin-Mayor Date: Mon, 5 Aug 2024 12:58:24 -0400 Subject: [PATCH 1/6] Add failing tests for various buggy situations --- tests/matchers/test_iou.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/matchers/test_iou.py b/tests/matchers/test_iou.py index f5eb4f31..3afa2671 100644 --- a/tests/matchers/test_iou.py +++ b/tests/matchers/test_iou.py @@ -34,6 +34,22 @@ def get_two_to_one(w, h, imw, imh): return merge.astype("int"), split.astype("int") +def get_no_overlap(imw, imh): + """two non-overlapping segmentations that start at high number labels + + Mapping [] + + """ + + im1 = np.zeros((imw, imh)) + im1[0:2, 0:2] = 100 + + im2 = np.zeros((imw, imh)) + im2[4:6, 4:6] = 222 + + return im1.astype("int"), im2.astype("int") + + def test__match_nodes(): # creat dummy image to test against num_labels = 5 @@ -63,6 +79,44 @@ def test__match_nodes(): assert ((2, 4) in matches) != ((2, 5) in matches) +def test__match_nodes_threshold(): + im1, im2 = get_two_to_one(10, 10, 30, 30) + # Test high threshold + gtcells, rescells = _match_nodes(im1, im2, threshold=0.7) + # Create match tuples + matches = list(zip(gtcells, rescells)) + # Check that nothing is matched + assert len(matches) == 0 + + # Test for high threshold and one to one + gtcells, rescells = _match_nodes(im1, im2, threshold=0.7, one_to_one=True) + # Create match tuples + matches = list(zip(gtcells, rescells)) + # Check that nothing is matched + assert len(matches) == 0 + + +def test__match_nodes_non_sequential(): + # test when the segmentation ids are high numbers (the lower numbers should never appear) + + im1, im2 = get_no_overlap(30, 30) + + # Test that phantom segmentations are not matched, even with threshold 0 + gtcells, rescells = _match_nodes(im1, im2, threshold=0.0) + # Create match tuples + matches = list(zip(gtcells, rescells)) + # Check that nothing is matched + assert len(matches) == 0 + + # Test that with one-to-one, phantom segmentations are not matched, + # even with threshold 0 + gtcells, rescells = _match_nodes(im1, im2, threshold=0.0, one_to_one=True) + # Create match tuples + matches = list(zip(gtcells, rescells)) + # Check that nothing is matched + assert len(matches) == 0 + + def test__construct_time_to_seg_id_map(): # Test 2d data n_frames = 3 From 3fc2dfb1c94c05e7d4ecc93d861f97572015d7b1 Mon Sep 17 00:00:00 2001 From: Caroline Malin-Mayor Date: Mon, 5 Aug 2024 13:04:22 -0400 Subject: [PATCH 2/6] Assert that threshold 0 is invalid --- tests/matchers/test_iou.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/matchers/test_iou.py b/tests/matchers/test_iou.py index 3afa2671..38d10d0f 100644 --- a/tests/matchers/test_iou.py +++ b/tests/matchers/test_iou.py @@ -78,22 +78,26 @@ def test__match_nodes(): # Check that only one of the merge matches is present assert ((2, 4) in matches) != ((2, 5) in matches) + with pytest.raises(ValueError): + # Test that threshold 0 is not valid when not one-to-one + gtcells, rescells = _match_nodes(im1, im2, threshold=0.0) + def test__match_nodes_threshold(): im1, im2 = get_two_to_one(10, 10, 30, 30) # Test high threshold - gtcells, rescells = _match_nodes(im1, im2, threshold=0.7) + gtcells, rescells = _match_nodes(im1, im2, threshold=1) # Create match tuples matches = list(zip(gtcells, rescells)) # Check that nothing is matched - assert len(matches) == 0 + assert len(matches) == 1 # Test for high threshold and one to one gtcells, rescells = _match_nodes(im1, im2, threshold=0.7, one_to_one=True) # Create match tuples matches = list(zip(gtcells, rescells)) # Check that nothing is matched - assert len(matches) == 0 + assert len(matches) == 1 def test__match_nodes_non_sequential(): @@ -101,8 +105,8 @@ def test__match_nodes_non_sequential(): im1, im2 = get_no_overlap(30, 30) - # Test that phantom segmentations are not matched, even with threshold 0 - gtcells, rescells = _match_nodes(im1, im2, threshold=0.0) + # Test that phantom segmentations are not matched + gtcells, rescells = _match_nodes(im1, im2, threshold=0.1) # Create match tuples matches = list(zip(gtcells, rescells)) # Check that nothing is matched From b4cefde31a4bf17de76d550349484ee391989891 Mon Sep 17 00:00:00 2001 From: Caroline Malin-Mayor Date: Mon, 5 Aug 2024 13:04:43 -0400 Subject: [PATCH 3/6] Raise a ValueError if threshold is 0 and not one-to-one --- src/traccuracy/matchers/_iou.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/traccuracy/matchers/_iou.py b/src/traccuracy/matchers/_iou.py index 1eac7044..44e49359 100644 --- a/src/traccuracy/matchers/_iou.py +++ b/src/traccuracy/matchers/_iou.py @@ -30,6 +30,8 @@ def _match_nodes(gt, res, threshold=0.5, one_to_one=False): gtcells (np arr): Array of overlapping ids in the gt frame. rescells (np arr): Array of overlapping ids in the res frame. """ + if threshold == 0.0 and not one_to_one: + raise ValueError("Threshold of 0 is not valid unless one_to_one is True") iou = np.zeros((np.max(gt) + 1, np.max(res) + 1)) overlapping_gt_labels, overlapping_res_labels, _ = get_labels_with_overlap(gt, res) From 59b30d44d07e518dcc85abc8bc281950d539dec7 Mon Sep 17 00:00:00 2001 From: Caroline Malin-Mayor Date: Mon, 5 Aug 2024 13:08:37 -0400 Subject: [PATCH 4/6] Threshold before adding ious to table --- src/traccuracy/matchers/_iou.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/traccuracy/matchers/_iou.py b/src/traccuracy/matchers/_iou.py index 44e49359..7b4e1140 100644 --- a/src/traccuracy/matchers/_iou.py +++ b/src/traccuracy/matchers/_iou.py @@ -41,12 +41,14 @@ def _match_nodes(gt, res, threshold=0.5, one_to_one=False): iou_res_idx = overlapping_res_labels[index] intersection = np.logical_and(gt == iou_gt_idx, res == iou_res_idx) union = np.logical_or(gt == iou_gt_idx, res == iou_res_idx) - iou[iou_gt_idx, iou_res_idx] = intersection.sum() / union.sum() + iou_value = intersection.sum() / union.sum() + if iou_value >= threshold: + iou[iou_gt_idx, iou_res_idx] = iou_value if one_to_one: pairs = _one_to_one_assignment(iou) else: - pairs = np.where(iou >= threshold) + pairs = np.where(iou) # Catch the case where there are no overlaps if len(pairs) < 2: From 615385e88471ddaed9a648906fe687a2bb6d6f54 Mon Sep 17 00:00:00 2001 From: Caroline Malin-Mayor Date: Mon, 5 Aug 2024 13:24:22 -0400 Subject: [PATCH 5/6] Increase cost in linear assigment for those with 0 IOU --- src/traccuracy/matchers/_iou.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/traccuracy/matchers/_iou.py b/src/traccuracy/matchers/_iou.py index 7b4e1140..9d420ccf 100644 --- a/src/traccuracy/matchers/_iou.py +++ b/src/traccuracy/matchers/_iou.py @@ -80,6 +80,8 @@ def _one_to_one_assignment(iou, unmapped_cost=4): # Assign 1 - iou to top left and bottom right cost = 1 - iou[1:, 1:] + # increase the cost for those with no IOU to higher than the unmapped cost + cost[cost == 1] = unmapped_cost + 1 matrix[:n0, :n1] = cost matrix[n_obj - n1 :, n_obj - n0 :] = cost.T From 141cedef2ba6d2380adcd1daebbbe67067691967 Mon Sep 17 00:00:00 2001 From: Caroline Malin-Mayor Date: Mon, 5 Aug 2024 16:01:43 -0400 Subject: [PATCH 6/6] Cast to int to fix #152 --- src/traccuracy/matchers/_iou.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/traccuracy/matchers/_iou.py b/src/traccuracy/matchers/_iou.py index 9d420ccf..973e8ef4 100644 --- a/src/traccuracy/matchers/_iou.py +++ b/src/traccuracy/matchers/_iou.py @@ -32,7 +32,8 @@ def _match_nodes(gt, res, threshold=0.5, one_to_one=False): """ if threshold == 0.0 and not one_to_one: raise ValueError("Threshold of 0 is not valid unless one_to_one is True") - iou = np.zeros((np.max(gt) + 1, np.max(res) + 1)) + # casting to int to avoid issue #152 (result is float with numpy<2, dtype=uint64) + iou = np.zeros((int(np.max(gt) + 1), int(np.max(res) + 1))) overlapping_gt_labels, overlapping_res_labels, _ = get_labels_with_overlap(gt, res)