diff --git a/docs/img/documentation_images/visualize_display_instances/mask_0.png b/docs/img/documentation_images/visualize_display_instances/mask_0.png new file mode 100644 index 000000000..d57cccb31 Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_0.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_1.png b/docs/img/documentation_images/visualize_display_instances/mask_1.png new file mode 100644 index 000000000..1e18bd5d1 Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_1.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_10.png b/docs/img/documentation_images/visualize_display_instances/mask_10.png new file mode 100644 index 000000000..3b888ac67 Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_10.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_2.png b/docs/img/documentation_images/visualize_display_instances/mask_2.png new file mode 100644 index 000000000..b8e691c8e Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_2.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_3.png b/docs/img/documentation_images/visualize_display_instances/mask_3.png new file mode 100644 index 000000000..3d383b24a Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_3.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_4.png b/docs/img/documentation_images/visualize_display_instances/mask_4.png new file mode 100644 index 000000000..8c04ef9bd Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_4.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_5.png b/docs/img/documentation_images/visualize_display_instances/mask_5.png new file mode 100644 index 000000000..9e969382f Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_5.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_6.png b/docs/img/documentation_images/visualize_display_instances/mask_6.png new file mode 100644 index 000000000..357fc7137 Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_6.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_7.png b/docs/img/documentation_images/visualize_display_instances/mask_7.png new file mode 100644 index 000000000..74ecf383f Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_7.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_8.png b/docs/img/documentation_images/visualize_display_instances/mask_8.png new file mode 100644 index 000000000..0746cbc52 Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_8.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/mask_9.png b/docs/img/documentation_images/visualize_display_instances/mask_9.png new file mode 100644 index 000000000..a9cc724fc Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/mask_9.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/result1.png b/docs/img/documentation_images/visualize_display_instances/result1.png new file mode 100644 index 000000000..84249232d Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/result1.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/result2.png b/docs/img/documentation_images/visualize_display_instances/result2.png new file mode 100644 index 000000000..016eb4edc Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/result2.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/result3.png b/docs/img/documentation_images/visualize_display_instances/result3.png new file mode 100644 index 000000000..27ad59c7e Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/result3.png differ diff --git a/docs/img/documentation_images/visualize_display_instances/visualize_inst_seg_img.png b/docs/img/documentation_images/visualize_display_instances/visualize_inst_seg_img.png new file mode 100644 index 000000000..69221ceb5 Binary files /dev/null and b/docs/img/documentation_images/visualize_display_instances/visualize_inst_seg_img.png differ diff --git a/docs/visualize_display_instances.md b/docs/visualize_display_instances.md new file mode 100644 index 000000000..57ddf5f87 --- /dev/null +++ b/docs/visualize_display_instances.md @@ -0,0 +1,64 @@ +## Display Instances + +This function displays different object intances in different colors on top of the original image. + +**plantcv.visualize.display_instances**(*img, masks, figsize=(16, 16), title="", colors=None, captions=None, show_bbox=True, ax=None*) +**returns** masked_img, colors + +- **Parameters:** + - img - (required, ndarray)input image + - masks - (required, ndarray) instance masks represented by a 3-d array, the 3rd dimension represents the number of insatnces to show. + - figsize - (optional, tuple) the size of the generated figure + - title - (optional, str) the title of the figure + - colors - (optional, list of tuples, every value should be in the range of [0.0,1.0]) a list of colors to use with each object. If no value is passed, a set of random colors would be used + - captions - (optional, str) a list of strings to use as captions for each object. If no list of captions is provided, show the local index of the instance + - show_bbox - (optional, bool) indicator of whether showing the bounding-box + - ax - (optional, matplotlib.axes._subplots.AxesSubplot) the axis to plot on. If no axis is passed, a new one will be created +- **Context:** + - Used to display different segmented instances on top of the original image. +- **Example use:** + - Below + +**Original image: RGB image** + +![Screenshot](img/documentation_images/visualize_display_instances/visualize_inst_seg_img.png) + +**masks: 10 segmentation masks represent for different leaves** + +![Screenshot](img/documentation_images/visualize_display_instances/mask_0.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_1.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_2.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_3.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_4.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_5.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_6.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_7.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_8.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_9.png) +![Screenshot](img/documentation_images/visualize_display_instances/mask_10.png) + + +```python + +from plantcv import plantcv as pcv + +masked_img,colors = pcv.visualize.display_instances(img, masks, figsize=(10, 10), title="", ax=None, colors=None, captions=None, show_bbox=True) + +# option to add customized captions and/or customized figure title and/or not showing the bounding box +captions = ["leaf{}".format(i) for i in range(masks.shape[2])] +_,_ = pcv.visualize.display_instances(img, masks, figsize=(16, 16), title="Visualization of segmentation", ax=None, colors=None, captions=captions, show_bbox=False) + +# the number of instances to display depends on the number of input masks +# the colors can also be customized +masks_reduced = masks[:,:,2:7], +colors = [(0.0,1.0,0.2),(0.4,0.0,1.0),(0.5,1.0,0.0),(1.0,0.6,0.0),(0.9,0.0,1.0)] +_,_= pcv.visualize.display_instances(img, masks_reduced, figsize=(16, 16), title="", ax=None, colors=colors, captions=None, show_bbox=True) + +``` + +**Blended Image** + +![Screenshot](img/documentation_images/visualize_display_instances/result1.png) +![Screenshot](img/documentation_images/visualize_display_instances/result2.png) +![Screenshot](img/documentation_images/visualize_display_instances/result3.png) +**Source Code:** [Here](https://github.com/danforthcenter/plantcv/blob/master/plantcv/plantcv/visualize/display_instances.py) diff --git a/plantcv/plantcv/hyperspectral/read_data.py b/plantcv/plantcv/hyperspectral/read_data.py index cf5bc603c..7c0c1ec97 100644 --- a/plantcv/plantcv/hyperspectral/read_data.py +++ b/plantcv/plantcv/hyperspectral/read_data.py @@ -118,6 +118,7 @@ def read_data(filename): hdata = hdata.replace("{\n", "{") hdata = hdata.replace("\n}", "}") hdata = hdata.replace(" \n ", "") + hdata = hdata.replace(" \n", "") hdata = hdata.replace(";", "") hdata = hdata.split("\n") @@ -125,9 +126,11 @@ def read_data(filename): for i, string in enumerate(hdata): if ' = ' in string: header_data = string.split(" = ") + header_data[0] = header_data[0].lower() header_dict.update({header_data[0].rstrip(): header_data[1].rstrip()}) elif ' : ' in string: header_data = string.split(" : ") + header_data[0] = header_data[0].lower() header_dict.update({header_data[0].rstrip(): header_data[1].rstrip()}) # Reformat wavelengths diff --git a/plantcv/plantcv/visualize/__init__.py b/plantcv/plantcv/visualize/__init__.py index 9ebef870d..7702d1020 100644 --- a/plantcv/plantcv/visualize/__init__.py +++ b/plantcv/plantcv/visualize/__init__.py @@ -5,10 +5,12 @@ from plantcv.plantcv.visualize.colorspaces import colorspaces from plantcv.plantcv.visualize.auto_threshold_methods import auto_threshold_methods from plantcv.plantcv.visualize.overlay_two_imgs import overlay_two_imgs +from plantcv.plantcv.visualize.display_instances import display_instances from plantcv.plantcv.visualize.colorize_label_img import colorize_label_img from plantcv.plantcv.visualize.obj_sizes import obj_sizes from plantcv.plantcv.visualize.obj_size_ecdf import obj_size_ecdf from plantcv.plantcv.visualize.hyper_histogram import hyper_histogram __all__ = ["pseudocolor", "colorize_masks", "histogram", "clustered_contours", "colorspaces", "auto_threshold_methods", - "overlay_two_imgs", "colorize_label_img", "obj_size_ecdf", "obj_sizes", "hyper_histogram"] + "overlay_two_imgs","display_instances", "colorize_label_img", "obj_size_ecdf", "obj_sizes", "hyper_histogram"] + diff --git a/plantcv/plantcv/visualize/display_instances.py b/plantcv/plantcv/visualize/display_instances.py new file mode 100644 index 000000000..2af31fae5 --- /dev/null +++ b/plantcv/plantcv/visualize/display_instances.py @@ -0,0 +1,152 @@ +# display instances in an image +import cv2 +from matplotlib.patches import Polygon +import colorsys +import random +from skimage.measure import find_contours +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import patches, lines +from matplotlib.patches import Polygon +from plantcv.plantcv import fatal_error, params, color_palette +import os +from cv2 import cvtColor, COLOR_BGR2RGB + +def _overlay_mask_on_img(img, mask, color, alpha=0.5): + """ Overlay a given mask on top of an image such that the masked area (the non-zero areas in the mask) is shown in user + defined color, the other area (the zero-valued areas in the mask) is shown in original image. + Inputs: + img = image to show, can be either visible or grayscale + mask = mask to be put on top of the image + color = a tuple of desired color to show the mask on top of the image + alpha = a value (between 0 and 1) indicating the transparency when blending mask and image, by default 0.5 + Output: + img = the original image with mask overlaied on top + + :param img: numpy.ndarray + :param mask: numpy.ndarray + :param color: tuple + :param alpha: float + :return: img: numpy.ndarray + """ + # if len(img.shape) == 2: + # img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + for c in range(img.shape[-1]): + img[:, :, c] = np.where(mask == 1, + img[:, :, c] * + (1 - alpha) + alpha * color[c] * 255, + img[:, :, c]) + return img + + +def display_instances(img, masks, figsize=(16, 16), title="", colors=None, captions=None, show_bbox=True, ax=None): + """ Display multiple instances in image based on the given masks + This function is inspired by the same function used in mrcnn, showing different instances with different colors on top of the original image + Users also have the option to specify the color for every instance, as well as the caption for every instance. Showing bounding boxes is another option. + Inputs: + img = rgb image to show + mask = a bunch of masks in a matrix + figsize = desired size of figure, by default (16,16) + title = desired title to show, by default "" + colors = a list of colors for every instance to display, by default colors=None + captions = a list of names for every instance, by default captions=None + show_bbox = a flog indicating whether show bounding box, by default show_bbox=True + ax = axis to show, by default ax=None + Outputs: + masked_img = image with instances masks overlaied on top + colors = colors used to display every instance + + :param img: numpy.ndarray + :param masks: numpy.ndarray + :param figsize: tuple + :param title: str + :param colors: list (of tuples) + :param captions: str + :param show_bbox: bool + :param ax: + :return: masked_img: numpy.ndarray + :return: colors: list (of tuples) + """ + + debug = params.debug + params.debug = None + + # Auto-increment the device counter + params.device += 1 + + if img.shape[0:2] != masks.shape[0:2]: + fatal_error("Sizes of image and mask mismatch!") + # + # auto_show = False + if not ax: + _, ax = plt.subplots(1, figsize=figsize) + # auto_show = True + + num_insts = masks.shape[2] + + # # Generate random colors + # colors = colors or _random_colors(num_insts) + + if colors is not None: + if max(max(colors)) > 1 or min(min(colors)) < 0: + fatal_error("RGBA values should be within 0-1 range!") + else: + colors_ = color_palette(num_insts) + colors = [tuple([ci / 255 for ci in c]) for c in colors_] + + if len(colors) < num_insts: + fatal_error("Not enough colors provided to show all instances!") + if len(colors) > num_insts: + colors = colors[0:num_insts] + + # Show area outside image boundaries. + height, width = img.shape[:2] + ax.set_ylim(height + 10, -10) + ax.set_xlim(-10, width + 10) + ax.axis('off') + ax.set_title(title) + + masked_img = img.astype(np.uint32).copy() + for i in range(num_insts): + color = colors[i] + + # Mask + mask = masks[:, :, i] + # the color is in order of r-g-b, however the image is in order of b-g-r, so take the reverse of color + masked_img = _overlay_mask_on_img(masked_img, mask, color[::-1]) + + ys, xs = np.where(mask > 0) + x1, x2 = min(xs), max(xs) + y1, y2 = min(ys), max(ys) + if show_bbox: + p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, alpha=0.7, linestyle="dashed", edgecolor=color, facecolor='none') + ax.add_patch(p) + + # Mask Polygon + # Pad to ensure proper polygons for masks that touch image edges. + padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8) + padded_mask[1:-1, 1:-1] = mask + contours = find_contours(padded_mask, 0.5) + for verts in contours: + # Subtract the padding and flip (y, x) to (x, y) + verts = np.fliplr(verts) - 1 + p = Polygon(verts, facecolor="none", edgecolor=color) + ax.add_patch(p) + if not captions: + caption = str(i) + else: + caption = captions[i] + ax.text(x1, y1 + 8, caption, color='w', size=13, backgroundcolor="none") + + masked_img = cvtColor(masked_img.astype(np.uint8), COLOR_BGR2RGB) + params.debug = debug + if params.debug is not None: + if params.debug == "plot": + ax.imshow(masked_img) + + if params.debug == "print": + ax.imshow(masked_img) + plt.savefig(os.path.join(params.debug_outdir, str(params.device) + "_displayed_instances.png")) + plt.close("all") + + return masked_img, colors diff --git a/tests/data/visualize_inst_seg_img.png b/tests/data/visualize_inst_seg_img.png new file mode 100644 index 000000000..09f806da1 Binary files /dev/null and b/tests/data/visualize_inst_seg_img.png differ diff --git a/tests/data/visualize_inst_seg_mask.pkl b/tests/data/visualize_inst_seg_mask.pkl new file mode 100644 index 000000000..eb3a3f41b Binary files /dev/null and b/tests/data/visualize_inst_seg_mask.pkl differ diff --git a/tests/tests.py b/tests/tests.py index 44cc85cd6..4b7e790a2 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -20,8 +20,10 @@ import matplotlib.pyplot as plt import dask from dask.distributed import Client +import pickle as pkl from skimage import img_as_ubyte + PARALLEL_TEST_DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parallel_data") TEST_TMPDIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".cache") TEST_IMG_DIR = "images" @@ -1082,6 +1084,8 @@ def test_plantcv_parallel_process_results_invalid_json(): # TEST_IM_BAD_NAN = "bad_mask_nan.pkl" # TEST_IM_BAD_INF = "bad_mask_inf.pkl" PIXEL_VALUES = "pixel_inspector_rgb_values.txt" +TEST_INPUT_INSTANCE_IMG = "visualize_inst_seg_img.png" +TEST_INPUT_INSTANCE_MASK = "visualize_inst_seg_mask.pkl" TEST_INPUT_LEAF_MASK = "leaves_mask.png" # leaving photosynthesis data here so it can be used to test plot_image and print_image @@ -6766,6 +6770,64 @@ def test_plantcv_visualize_overlay_two_imgs_size_mismatch(): with pytest.raises(RuntimeError): _ = pcv.visualize.overlay_two_imgs(img1=img1, img2=img2) +def test_plantcv_visualize_display_instances(): + cache_dir = os.path.join(TEST_TMPDIR, "test_plantcv_visualize_display_instances") + os.mkdir(cache_dir) + # create synthetic test data + img = img_as_ubyte(np.random.rand(10,10,3)) + masks = np.zeros((10, 10, 2)) + masks[1:3, 3:5, 0] = 1 + masks[2:5, 1:3, 1] = 1 + colors = [(1.0,0.0,0.15),(1.0,0.0,0.74)] + pcv.params.debug = "plot" + _, ax = plt.subplots(1, 1, figsize=(16, 16)) + _, colors = pcv.visualize.display_instances(img, masks, title="test", colors=colors, captions=['1','2'], show_bbox=True, ax=ax) + pcv.params.debug = "print" + pcv.params.debug_outdir = cache_dir + _, colors = pcv.visualize.display_instances(img, masks) + assert len(colors) == masks.shape[2] + +def test_plantcv_visualize_display_instances_bad_color(): + pcv.params.debug = None + # create synthetic test data + img = img_as_ubyte(np.random.rand(10,10,3)) + masks = np.zeros((10, 10, 2), dtype=np.uint8) + masks[1:3, 3:5, 0] = 1 + masks[2:5, 1:3, 1] = 1 + colors = [(1.0,0.0,0.15)] + with pytest.raises(RuntimeError): + _, _ = pcv.visualize.display_instances(img, masks, colors=colors) + +def test_plantcv_visualize_display_instances_bad_color2(): + pcv.params.debug = None + # create synthetic test data + img = img_as_ubyte(np.random.rand(10,10,3)) + masks = np.zeros((10, 10, 2), dtype=np.uint8) + masks[1:3, 3:5, 0] = 1 + masks[2:5, 1:3, 1] = 1 + colors = [(1.0,0.0,0.15),(1.0,0.0,0.74),(0.0,1.0,0.74)] + _, colors = pcv.visualize.display_instances(img, masks, colors=colors) + assert len(colors) == masks.shape[2] + +def test_plantcv_visualize_display_instances_bad_color3(): + pcv.params.debug = None + # create synthetic test data + img = img_as_ubyte(np.random.rand(10,10,3)) + masks = np.zeros((10, 10, 2), dtype=np.uint8) + masks[1:3, 3:5, 0] = 1 + masks[2:5, 1:3, 1] = 1 + colors = [(10,0.0,15.0),(10,0.0,74)] + with pytest.raises(RuntimeError): + _, _ = pcv.visualize.display_instances(img, masks, colors=colors) + +def test_plantcv_visualize_display_instances_bad_size(): + pcv.params.debug = None + # create synthetic test data + img = img_as_ubyte(np.random.rand(5,5,3)) + masks = np.zeros((10, 10, 2), dtype=np.uint8) + with pytest.raises(RuntimeError): + _, _ = pcv.visualize.display_instances(img, masks) + @pytest.mark.parametrize("num,expected", [[100, 35], [30, 33]]) def test_plantcv_visualize_size(num, expected):