From 8318d5cb74784348fa481faa3a05f501a4808dc4 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Mon, 26 Feb 2024 16:32:58 +0100 Subject: [PATCH 01/40] Add "image_diff" comparison for test files --- lib/galaxy/tool_util/verify/__init__.py | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index c989355d8bce..6acc3bfa97b4 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -10,6 +10,7 @@ import re import shutil import tempfile +import math from typing import ( Any, Callable, @@ -17,6 +18,9 @@ Optional, ) +import imageio +import numpy + try: import pysam except ImportError: @@ -171,6 +175,8 @@ def get_filename(filename: str) -> str: files_delta(local_name, temp_name, attributes=attributes) elif compare == "contains": files_contains(local_name, temp_name, attributes=attributes) + elif compare == "image_diff": + files_image_diff(local_name, temp_name, attributes=attributes) else: raise Exception(f"Unimplemented Compare type: {compare}") except AssertionError as err: @@ -432,3 +438,33 @@ def files_contains(file1, file2, attributes=None): line_diff_count += 1 if line_diff_count > lines_diff: raise AssertionError(f"Failed to find '{contains}' in history data. (lines_diff={lines_diff}).") + + +def get_image_metric(attributes): + attributes = attributes or {} + metrics = { + "mse": lambda im1, im2: (im1 - im2).square().mean(), + "rms": lambda im1, im2: math.sqrt((im1 - im2).square().mean()), + "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), + } + return metrics[attributes.get("metric")] + + +def files_image_diff(file1, file2, attributes=None): + """Check the pixel data of 2 image files for differences.""" + attributes = attributes or {} + + im1 = imageio.imread(file1) + im2 = imageio.imread(file2) + + if im1.dtype != im2.dtype: + raise AssertionError(f"Image data types did not match ({im1.dtype}, {im2.dtype}).") + + if im1.shape != im2.shape: + raise AssertionError(f"Image dimensions did not match ({im1.shape}, {im2.shape}).") + + distance = get_image_metric(attributes)(im1, im2) + distance_eps = attributes.get("eps", 0.) + if distance > distance_eps: + raise AssertionError(f"Image difference {distance} exceeds eps={distance_eps}.") + From 94352dc2c8e897a1ccc618fa07da88505dd7a3f3 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 10:40:42 +0100 Subject: [PATCH 02/40] Add `files_image_diff` --- lib/galaxy/tool_util/verify/__init__.py | 29 ++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 6acc3bfa97b4..1e5721d75629 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -18,7 +18,7 @@ Optional, ) -import imageio +import pillow import numpy try: @@ -440,12 +440,35 @@ def files_contains(file1, file2, attributes=None): raise AssertionError(f"Failed to find '{contains}' in history data. (lines_diff={lines_diff}).") +def multiobject_intersection_over_union(mask1, mask2, background=0, repeat_reverse=True): + iou_list = list() + for label1 in mask1.unique(): + if label1 == background: continue + cc1 = (mask1 == label1) + for label2 in mask2[cc1].unique(): + if label2 == background: continue + cc2 = (mask2 == label2) + iou_list.append(intersection_over_union(cc1, cc2)) + if repeat_reverse: + iou_list += intersection_over_union(mask2, mask1, background, repeat_reverse=False) + return iou_list + + +def intersection_over_union(mask1, mask2, background=0): + assert mask1.dtype == mask2.dtype + if mask1.dtype == numpy.bool: + return numpy.logical_and(mask1, mask2) / numpy.logical_or(mask1, mask2) + else: + return min(multiobject_intersection_over_union(mask1, mask2, background)) + + def get_image_metric(attributes): attributes = attributes or {} metrics = { "mse": lambda im1, im2: (im1 - im2).square().mean(), "rms": lambda im1, im2: math.sqrt((im1 - im2).square().mean()), "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), + "iou": lambda im1, im2: 1 - intersection_over_union(mask1, mask2, attributes.get("background", 0)), } return metrics[attributes.get("metric")] @@ -454,8 +477,8 @@ def files_image_diff(file1, file2, attributes=None): """Check the pixel data of 2 image files for differences.""" attributes = attributes or {} - im1 = imageio.imread(file1) - im2 = imageio.imread(file2) + im1 = numpy.array(pillow.Image.open(file1)) + im2 = numpy.array(pillow.Image.open(file2)) if im1.dtype != im2.dtype: raise AssertionError(f"Image data types did not match ({im1.dtype}, {im2.dtype}).") From bcf78ecc28d9947f1162c9f6748935250644ccd5 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:21:36 +0100 Subject: [PATCH 03/40] Add tests for `files_image_diff` --- lib/galaxy/tool_util/verify/__init__.py | 1 + test/unit/tool_util/test_verify.py | 79 ++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 1e5721d75629..3a0ec2e30e3a 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -465,6 +465,7 @@ def intersection_over_union(mask1, mask2, background=0): def get_image_metric(attributes): attributes = attributes or {} metrics = { + "mad": lambda im1, im2: numpy.abs(im1 - im2).mean(), "mse": lambda im1, im2: (im1 - im2).square().mean(), "rms": lambda im1, im2: math.sqrt((im1 - im2).square().mean()), "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index f03da0857cfb..e5de00382b45 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -1,6 +1,7 @@ import collections import gzip import tempfile +import io from typing import ( Any, Dict, @@ -12,12 +13,16 @@ import pytest +import pillow +import numpy + from galaxy.tool_util.verify import ( files_contains, files_delta, files_diff, files_re_match, files_re_match_multiline, + files_image_diff, ) F1 = b"A\nB\nC" @@ -30,9 +35,42 @@ TestDef = Tuple[bytes, bytes, Optional[Dict[str, Any]], Optional[Type[AssertionError]]] +def _encode_image(im, **kwargs): + buf = io.BytesIO() + pil_im = pillow.Image.fromarray(im) + pil_im.save(buf, **kwargs) + return buf.getvalue() + +F6 = _encode_image(numpy.array( + [ + [1.0, 1.0, 1.0], + [1.0, 0.9, 1.0], + [1.0, 1.0, 1.0], + ], + dtype=float), format="PNG") +F7 = _encode_image(numpy.array( + [ + [1.0, 1.0, 1.0], + [1.0, 0.8, 1.0], + [1.0, 1.0, 1.0], + ], + dtype=float), format="TIFF") +F8 = _encode_image((F5 * 0xFF).astype(np.uint8), format="PNG") + + def _test_file_list(): files = [] - for b, ext in [(F1, ".txt"), (F2, ".txt"), (F3, ".pdf"), (F4, ".txt"), (MULTILINE_MATCH, ".txt"), (F1, ".txt.gz")]: + for b, ext in [ + (F1, ".txt"), + (F2, ".txt"), + (F3, ".pdf"), + (F4, ".txt"), + (MULTILINE_MATCH, ".txt"), + (F1, ".txt.gz"), + (F6, ".png"), + (F7, ".tiff"), + (F8, ".png"), + ]: with tempfile.NamedTemporaryFile(mode="wb", suffix=ext, delete=False) as out: if ext == ".txt.gz": b = gzip.compress(b) @@ -42,7 +80,7 @@ def _test_file_list(): def generate_tests(multiline=False): - f1, f2, f3, f4, multiline_match, f5 = _test_file_list() + f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() tests: List[TestDef] if multiline: tests = [(multiline_match, f1, {"lines_diff": 0, "sort": True}, None)] @@ -60,7 +98,7 @@ def generate_tests(multiline=False): def generate_tests_sim_size(): - f1, f2, f3, f4, multiline_match, f5 = _test_file_list() + f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() # tests for equal files tests: List[TestDef] = [ (f1, f1, None, None), # pass default values @@ -85,6 +123,32 @@ def generate_tests_sim_size(): return tests +def generate_tests_image_diff(): + f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() + metrics = ["mad", "mse", "rms", "fro", "iou"] + # tests for equal files (float) + tests: List[TestDef] = [ + (f6, f6, {"metric": metric}, None) for metric in metrics + ] + # tests for equal files (uint8) + tests += [ + (f8, f8, {"metric": metric}, None) for metric in metrics + ] + # tests for two different files + tests += [ + (f6, f8, {"metric": metric}, AssertionError) for metric in metrics + ] + tests += [ + (f7, f8, {"metric": metric}, AssertionError) for metric in metrics + ] + tests += [ + (f6, f7, {"metric": "iou"}, None), + (f6, f7, {"metric": "mad", "eps": 0.1 / 9}, None), + (f6, f7, {"metric": "mad", "eps": (0.1 / 9) * 0.99}, AssertionError), + ] + return tests + + @pytest.mark.parametrize("file1,file2,attributes,expect", generate_tests()) def test_files_contains(file1, file2, attributes, expect): if expect is not None: @@ -128,3 +192,12 @@ def test_files_re_match_multiline(file1, file2, attributes, expect): files_re_match_multiline(file1.path, file2.path, attributes) else: files_re_match_multiline(file1.path, file2.path, attributes) + + +@pytest.mark.parametrize("file1,file2,attributes,expect", generate_tests_image_diff()) +def test_files_image_diff(file1, file2, attributes, expect): + if expect is not None: + with pytest.raises(expect): + files_image_diff(file1.path, file2.path, attributes) + else: + files_image_diff(file1.path, file2.path, attributes) From 908116c72afb4d3c09f92afd000f9fecd8d05e91 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:24:11 +0100 Subject: [PATCH 04/40] Add `pillow` to pinned-requirements.txt --- lib/galaxy/dependencies/pinned-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index d9d002fd0eea..2eabe24f4503 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -126,6 +126,7 @@ parsley==1.3 ; python_version >= "3.8" and python_version < "3.13" paste==3.7.1 ; python_version >= "3.8" and python_version < "3.13" pastedeploy==3.1.0 ; python_version >= "3.8" and python_version < "3.13" pebble==5.0.6 ; python_version >= "3.8" and python_version < "3.13" +pillow==10.2.0 ; python_version >= "3.8" and python_version < "3.13" pkgutil-resolve-name==1.3.10 ; python_version >= "3.8" and python_version < "3.9" promise==2.3 ; python_version >= "3.8" and python_version < "3.13" prompt-toolkit==3.0.43 ; python_version >= "3.8" and python_version < "3.13" From b325e2728bb48e9074de60e3aaa54c70b79ad160 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:27:17 +0100 Subject: [PATCH 05/40] Fix linting issues --- lib/galaxy/tool_util/verify/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 3a0ec2e30e3a..3ca27514e06f 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -443,10 +443,12 @@ def files_contains(file1, file2, attributes=None): def multiobject_intersection_over_union(mask1, mask2, background=0, repeat_reverse=True): iou_list = list() for label1 in mask1.unique(): - if label1 == background: continue + if label1 == background: + continue cc1 = (mask1 == label1) for label2 in mask2[cc1].unique(): - if label2 == background: continue + if label2 == background: + continue cc2 = (mask2 == label2) iou_list.append(intersection_over_union(cc1, cc2)) if repeat_reverse: @@ -469,7 +471,7 @@ def get_image_metric(attributes): "mse": lambda im1, im2: (im1 - im2).square().mean(), "rms": lambda im1, im2: math.sqrt((im1 - im2).square().mean()), "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), - "iou": lambda im1, im2: 1 - intersection_over_union(mask1, mask2, attributes.get("background", 0)), + "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2, attributes.get("background", 0)), } return metrics[attributes.get("metric")] From 56506e49ddb45c87c0640a3baaf99b059f7d077f Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:32:50 +0100 Subject: [PATCH 06/40] Fix linting issues --- test/unit/tool_util/test_verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index e5de00382b45..f6fe28d98f62 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -55,7 +55,7 @@ def _encode_image(im, **kwargs): [1.0, 1.0, 1.0], ], dtype=float), format="TIFF") -F8 = _encode_image((F5 * 0xFF).astype(np.uint8), format="PNG") +F8 = _encode_image((F6 * 0xFF).astype(numpy.uint8), format="PNG") def _test_file_list(): From 3af5443a4572c5b1451ae66913fd8ff0635bcf0e Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:35:29 +0100 Subject: [PATCH 07/40] Fix linting issues --- lib/galaxy/tool_util/verify/__init__.py | 1 - test/unit/tool_util/test_verify.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 3ca27514e06f..5573620e4a61 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -493,4 +493,3 @@ def files_image_diff(file1, file2, attributes=None): distance_eps = attributes.get("eps", 0.) if distance > distance_eps: raise AssertionError(f"Image difference {distance} exceeds eps={distance_eps}.") - diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index f6fe28d98f62..f29d3f28b413 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -41,6 +41,7 @@ def _encode_image(im, **kwargs): pil_im.save(buf, **kwargs) return buf.getvalue() + F6 = _encode_image(numpy.array( [ [1.0, 1.0, 1.0], From a1b51ea3c90c6b27fdfadb036362d61ad0acf821 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:44:35 +0100 Subject: [PATCH 08/40] Fix linting issues --- lib/galaxy/tool_util/verify/__init__.py | 6 +-- test/unit/tool_util/test_verify.py | 52 ++++++++++++------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 5573620e4a61..2856b94f4b00 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -445,11 +445,11 @@ def multiobject_intersection_over_union(mask1, mask2, background=0, repeat_rever for label1 in mask1.unique(): if label1 == background: continue - cc1 = (mask1 == label1) + cc1 = mask1 == label1 for label2 in mask2[cc1].unique(): if label2 == background: continue - cc2 = (mask2 == label2) + cc2 = mask2 == label2 iou_list.append(intersection_over_union(cc1, cc2)) if repeat_reverse: iou_list += intersection_over_union(mask2, mask1, background, repeat_reverse=False) @@ -490,6 +490,6 @@ def files_image_diff(file1, file2, attributes=None): raise AssertionError(f"Image dimensions did not match ({im1.shape}, {im2.shape}).") distance = get_image_metric(attributes)(im1, im2) - distance_eps = attributes.get("eps", 0.) + distance_eps = attributes.get("eps", 0.0) if distance > distance_eps: raise AssertionError(f"Image difference {distance} exceeds eps={distance_eps}.") diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index f29d3f28b413..95839636bac7 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -42,20 +42,28 @@ def _encode_image(im, **kwargs): return buf.getvalue() -F6 = _encode_image(numpy.array( - [ - [1.0, 1.0, 1.0], - [1.0, 0.9, 1.0], - [1.0, 1.0, 1.0], - ], - dtype=float), format="PNG") -F7 = _encode_image(numpy.array( - [ - [1.0, 1.0, 1.0], - [1.0, 0.8, 1.0], - [1.0, 1.0, 1.0], - ], - dtype=float), format="TIFF") +F6 = _encode_image( + numpy.array( + [ + [1.0, 1.0, 1.0], + [1.0, 0.9, 1.0], + [1.0, 1.0, 1.0], + ], + dtype=float + ), + format="PNG", +) +F7 = _encode_image( + numpy.array( + [ + [1.0, 1.0, 1.0], + [1.0, 0.8, 1.0], + [1.0, 1.0, 1.0], + ], + dtype=float, + ), + format="TIFF", +) F8 = _encode_image((F6 * 0xFF).astype(numpy.uint8), format="PNG") @@ -128,20 +136,12 @@ def generate_tests_image_diff(): f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() metrics = ["mad", "mse", "rms", "fro", "iou"] # tests for equal files (float) - tests: List[TestDef] = [ - (f6, f6, {"metric": metric}, None) for metric in metrics - ] + tests: List[TestDef] = [(f6, f6, {"metric": metric}, None) for metric in metrics] # tests for equal files (uint8) - tests += [ - (f8, f8, {"metric": metric}, None) for metric in metrics - ] + tests += [(f8, f8, {"metric": metric}, None) for metric in metrics] # tests for two different files - tests += [ - (f6, f8, {"metric": metric}, AssertionError) for metric in metrics - ] - tests += [ - (f7, f8, {"metric": metric}, AssertionError) for metric in metrics - ] + tests += [(f6, f8, {"metric": metric}, AssertionError) for metric in metrics] + tests += [(f7, f8, {"metric": metric}, AssertionError) for metric in metrics] tests += [ (f6, f7, {"metric": "iou"}, None), (f6, f7, {"metric": "mad", "eps": 0.1 / 9}, None), From b73091246ded3408b4dbeb885f1a3aed69830af7 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:49:55 +0100 Subject: [PATCH 09/40] Fix linting issues --- test/unit/tool_util/test_verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 95839636bac7..8a5488397607 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -49,7 +49,7 @@ def _encode_image(im, **kwargs): [1.0, 0.9, 1.0], [1.0, 1.0, 1.0], ], - dtype=float + dtype=float, ), format="PNG", ) From 5bf2f3f1a51f4f1d6a6bac2bfea7fbac57278ca8 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 11:58:34 +0100 Subject: [PATCH 10/40] Fix linting issues --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- test/unit/tool_util/test_verify.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 2856b94f4b00..1bf7a692c90d 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -5,12 +5,12 @@ import hashlib import json import logging +import math import os import os.path import re import shutil import tempfile -import math from typing import ( Any, Callable, @@ -18,8 +18,8 @@ Optional, ) -import pillow import numpy +import pillow try: import pysam diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 8a5488397607..056730807382 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -1,7 +1,7 @@ import collections import gzip -import tempfile import io +import tempfile from typing import ( Any, Dict, @@ -11,18 +11,17 @@ Type, ) -import pytest - -import pillow import numpy +import pillow +import pytest from galaxy.tool_util.verify import ( files_contains, files_delta, files_diff, + files_image_diff, files_re_match, files_re_match_multiline, - files_image_diff, ) F1 = b"A\nB\nC" From 7c447f087d8054712317021927fb069899ab420d Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 12:52:27 +0100 Subject: [PATCH 11/40] Fix pillow imports --- lib/galaxy/tool_util/verify/__init__.py | 2 +- test/unit/tool_util/test_verify.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 1bf7a692c90d..902470fb0f49 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -19,7 +19,7 @@ ) import numpy -import pillow +import PIL try: import pysam diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 056730807382..2768c1585d5e 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -12,7 +12,7 @@ ) import numpy -import pillow +import PIL import pytest from galaxy.tool_util.verify import ( From a147d72ada6fdd83653366a7e73d42568a0cf1dd Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 12:52:45 +0100 Subject: [PATCH 12/40] Add pillow dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0d310defef63..6163ade7ab00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ paramiko = "!=2.9.0, !=2.9.1" # https://github.com/paramiko/paramiko/issues/196 Parsley = "*" Paste = "*" pebble = "*" +pillow = "*" psutil = "*" pulsar-galaxy-lib = ">=0.15.0.dev0" pycryptodome = "*" From ea1c980701f1bca33aa50ede5ee7bf36e6fb25d4 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 12:54:48 +0100 Subject: [PATCH 13/40] Fix pillow usage --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- test/unit/tool_util/test_verify.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 902470fb0f49..924f247e94f5 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -480,8 +480,8 @@ def files_image_diff(file1, file2, attributes=None): """Check the pixel data of 2 image files for differences.""" attributes = attributes or {} - im1 = numpy.array(pillow.Image.open(file1)) - im2 = numpy.array(pillow.Image.open(file2)) + im1 = numpy.array(PIL.Image.open(file1)) + im2 = numpy.array(PIL.Image.open(file2)) if im1.dtype != im2.dtype: raise AssertionError(f"Image data types did not match ({im1.dtype}, {im2.dtype}).") diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 2768c1585d5e..96ccbabb2922 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -36,7 +36,7 @@ def _encode_image(im, **kwargs): buf = io.BytesIO() - pil_im = pillow.Image.fromarray(im) + pil_im = PIL.Image.fromarray(im) pil_im.save(buf, **kwargs) return buf.getvalue() From 0d6077db8f078e285002fff5146f9299e987b10e Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 13:10:25 +0100 Subject: [PATCH 14/40] Fix bugs --- lib/galaxy/tool_util/verify/__init__.py | 12 +++++++----- test/unit/tool_util/test_verify.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 924f247e94f5..01518cd7f082 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -19,7 +19,7 @@ ) import numpy -import PIL +from PIL import Image try: import pysam @@ -440,7 +440,7 @@ def files_contains(file1, file2, attributes=None): raise AssertionError(f"Failed to find '{contains}' in history data. (lines_diff={lines_diff}).") -def multiobject_intersection_over_union(mask1, mask2, background=0, repeat_reverse=True): +def _multiobject_intersection_over_union(mask1, mask2, background=0, repeat_reverse=True): iou_list = list() for label1 in mask1.unique(): if label1 == background: @@ -458,10 +458,12 @@ def multiobject_intersection_over_union(mask1, mask2, background=0, repeat_rever def intersection_over_union(mask1, mask2, background=0): assert mask1.dtype == mask2.dtype + assert mask1.ndim == mask2.ndim == 2 + assert mask1.shape == mask2.shape if mask1.dtype == numpy.bool: return numpy.logical_and(mask1, mask2) / numpy.logical_or(mask1, mask2) else: - return min(multiobject_intersection_over_union(mask1, mask2, background)) + return min(_multiobject_intersection_over_union(mask1, mask2, background)) def get_image_metric(attributes): @@ -480,8 +482,8 @@ def files_image_diff(file1, file2, attributes=None): """Check the pixel data of 2 image files for differences.""" attributes = attributes or {} - im1 = numpy.array(PIL.Image.open(file1)) - im2 = numpy.array(PIL.Image.open(file2)) + im1 = numpy.array(Image.open(file1)) + im2 = numpy.array(Image.open(file2)) if im1.dtype != im2.dtype: raise AssertionError(f"Image data types did not match ({im1.dtype}, {im2.dtype}).") diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 96ccbabb2922..5e35e3a4ac65 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -12,7 +12,7 @@ ) import numpy -import PIL +from PIL import Image import pytest from galaxy.tool_util.verify import ( @@ -36,7 +36,7 @@ def _encode_image(im, **kwargs): buf = io.BytesIO() - pil_im = PIL.Image.fromarray(im) + pil_im = Image.fromarray(im) pil_im.save(buf, **kwargs) return buf.getvalue() From 79967031d016a6ecd6d50d64e90c1770876328f4 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 13:24:43 +0100 Subject: [PATCH 15/40] Fix bugs --- test/unit/tool_util/test_verify.py | 36 ++++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 5e35e3a4ac65..6de6a7f58b57 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -44,26 +44,26 @@ def _encode_image(im, **kwargs): F6 = _encode_image( numpy.array( [ - [1.0, 1.0, 1.0], - [1.0, 0.9, 1.0], - [1.0, 1.0, 1.0], + [255, 255, 255], + [255, 200, 255], + [255, 255, 255], ], - dtype=float, + dtype=np.uint8, ), format="PNG", ) F7 = _encode_image( numpy.array( [ - [1.0, 1.0, 1.0], - [1.0, 0.8, 1.0], - [1.0, 1.0, 1.0], + [255, 255, 255], + [255, 100, 255], + [255, 255, 255], ], - dtype=float, + dtype=np.uint8, ), format="TIFF", ) -F8 = _encode_image((F6 * 0xFF).astype(numpy.uint8), format="PNG") +F8 = _encode_image(F6 / 0xFF, format="TIFF") def _test_file_list(): @@ -77,7 +77,7 @@ def _test_file_list(): (F1, ".txt.gz"), (F6, ".png"), (F7, ".tiff"), - (F8, ".png"), + (F8, ".tiff"), ]: with tempfile.NamedTemporaryFile(mode="wb", suffix=ext, delete=False) as out: if ext == ".txt.gz": @@ -134,17 +134,19 @@ def generate_tests_sim_size(): def generate_tests_image_diff(): f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() metrics = ["mad", "mse", "rms", "fro", "iou"] - # tests for equal files (float) + # tests for equal files (uint8, PNG) tests: List[TestDef] = [(f6, f6, {"metric": metric}, None) for metric in metrics] - # tests for equal files (uint8) + # tests for equal files (uint8, TIFF) + tests += [(f7, f7, {"metric": metric}, None) for metric in metrics] + # tests for equal files (float, TIFF) tests += [(f8, f8, {"metric": metric}, None) for metric in metrics] - # tests for two different files - tests += [(f6, f8, {"metric": metric}, AssertionError) for metric in metrics] - tests += [(f7, f8, {"metric": metric}, AssertionError) for metric in metrics] + # tests for pairs of different files + tests += [(f6, f8, {"metric": metric}, AssertionError) for metric in metrics] # uint8 vs float + tests += [(f7, f8, {"metric": metric}, AssertionError) for metric in metrics] # uint8 vs float tests += [ (f6, f7, {"metric": "iou"}, None), - (f6, f7, {"metric": "mad", "eps": 0.1 / 9}, None), - (f6, f7, {"metric": "mad", "eps": (0.1 / 9) * 0.99}, AssertionError), + (f6, f7, {"metric": "mad", "eps": 100 / 9 + 1e-4}, None), + (f6, f7, {"metric": "mad", "eps": 100 / 9 - 1e-4}, AssertionError), ] return tests From f22d051e8a14e4398baa1dba72e6fe5822d366b2 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 13:25:56 +0100 Subject: [PATCH 16/40] Fix bugs --- test/unit/tool_util/test_verify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 6de6a7f58b57..ca68249ed744 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -48,7 +48,7 @@ def _encode_image(im, **kwargs): [255, 200, 255], [255, 255, 255], ], - dtype=np.uint8, + dtype=numpy.uint8, ), format="PNG", ) @@ -59,7 +59,7 @@ def _encode_image(im, **kwargs): [255, 100, 255], [255, 255, 255], ], - dtype=np.uint8, + dtype=numpy.uint8, ), format="TIFF", ) From e5eb5d43bee1a99c72ef624f6ec9a125fb5f55de Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 13:29:35 +0100 Subject: [PATCH 17/40] Fix linting issues --- test/unit/tool_util/test_verify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index ca68249ed744..f767eba46c96 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -12,8 +12,8 @@ ) import numpy -from PIL import Image import pytest +from PIL import Image from galaxy.tool_util.verify import ( files_contains, From 4d800a8f36597b5f291452fb0dafdd5abf0b74da Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 14:16:55 +0100 Subject: [PATCH 18/40] Fix bug --- test/unit/tool_util/test_verify.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index f767eba46c96..93ed2c27b343 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -63,7 +63,17 @@ def _encode_image(im, **kwargs): ), format="TIFF", ) -F8 = _encode_image(F6 / 0xFF, format="TIFF") +F8 = _encode_image( + numpy.array( + [ + [255, 255, 255], + [255, 100, 255], + [255, 255, 255], + ], + dtype=float, + ) / 0xFF, + format="TIFF", +) def _test_file_list(): From 8255e9d2c417507f467e19637a5381a6ece9808b Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 14:24:01 +0100 Subject: [PATCH 19/40] Set default `eps` to 0.01 --- lib/galaxy/tool_util/verify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 01518cd7f082..5b1716679f28 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -492,6 +492,6 @@ def files_image_diff(file1, file2, attributes=None): raise AssertionError(f"Image dimensions did not match ({im1.shape}, {im2.shape}).") distance = get_image_metric(attributes)(im1, im2) - distance_eps = attributes.get("eps", 0.0) + distance_eps = attributes.get("eps", 0.01) if distance > distance_eps: raise AssertionError(f"Image difference {distance} exceeds eps={distance_eps}.") From 1937e2ba78d375f2c50bae8ab12bd60e1a1867ec Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 15:09:38 +0100 Subject: [PATCH 20/40] Fix bug --- lib/galaxy/tool_util/verify/__init__.py | 24 +++++++++++------------- test/unit/tool_util/test_verify.py | 6 +++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 5b1716679f28..a4686193e5a1 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -440,42 +440,40 @@ def files_contains(file1, file2, attributes=None): raise AssertionError(f"Failed to find '{contains}' in history data. (lines_diff={lines_diff}).") -def _multiobject_intersection_over_union(mask1, mask2, background=0, repeat_reverse=True): +def _multiobject_intersection_over_union(mask1, mask2, repeat_reverse=True): iou_list = list() for label1 in mask1.unique(): - if label1 == background: - continue cc1 = mask1 == label1 + cc1_iou_list = list() for label2 in mask2[cc1].unique(): - if label2 == background: - continue cc2 = mask2 == label2 - iou_list.append(intersection_over_union(cc1, cc2)) + cc1_iou_list.append(intersection_over_union(cc1, cc2)) + iou_list.append(max(cc1_iou_list)) if repeat_reverse: - iou_list += intersection_over_union(mask2, mask1, background, repeat_reverse=False) + iou_list += intersection_over_union(mask2, mask1, repeat_reverse=False) return iou_list -def intersection_over_union(mask1, mask2, background=0): +def intersection_over_union(mask1, mask2): assert mask1.dtype == mask2.dtype assert mask1.ndim == mask2.ndim == 2 assert mask1.shape == mask2.shape - if mask1.dtype == numpy.bool: + if mask1.dtype == bool: return numpy.logical_and(mask1, mask2) / numpy.logical_or(mask1, mask2) else: - return min(_multiobject_intersection_over_union(mask1, mask2, background)) + return min(_multiobject_intersection_over_union(mask1, mask2)) def get_image_metric(attributes): attributes = attributes or {} metrics = { - "mad": lambda im1, im2: numpy.abs(im1 - im2).mean(), + "mae": lambda im1, im2: numpy.abs(im1 - im2).mean(), "mse": lambda im1, im2: (im1 - im2).square().mean(), "rms": lambda im1, im2: math.sqrt((im1 - im2).square().mean()), "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), - "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2, attributes.get("background", 0)), + "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), } - return metrics[attributes.get("metric")] + return metrics[attributes.get("metric", "mae")] def files_image_diff(file1, file2, attributes=None): diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 93ed2c27b343..a8697b2452ef 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -143,7 +143,7 @@ def generate_tests_sim_size(): def generate_tests_image_diff(): f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() - metrics = ["mad", "mse", "rms", "fro", "iou"] + metrics = ["mae", "mse", "rms", "fro", "iou"] # tests for equal files (uint8, PNG) tests: List[TestDef] = [(f6, f6, {"metric": metric}, None) for metric in metrics] # tests for equal files (uint8, TIFF) @@ -155,8 +155,8 @@ def generate_tests_image_diff(): tests += [(f7, f8, {"metric": metric}, AssertionError) for metric in metrics] # uint8 vs float tests += [ (f6, f7, {"metric": "iou"}, None), - (f6, f7, {"metric": "mad", "eps": 100 / 9 + 1e-4}, None), - (f6, f7, {"metric": "mad", "eps": 100 / 9 - 1e-4}, AssertionError), + (f6, f7, {"metric": "mae", "eps": 100 / 9 + 1e-4}, None), + (f6, f7, {"metric": "mae", "eps": 100 / 9 - 1e-4}, AssertionError), ] return tests From 172188fc81314e5f6012a4ab7ce35841f7e4d067 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 15:09:46 +0100 Subject: [PATCH 21/40] Update XSD --- lib/galaxy/tool_util/xsd/galaxy.xsd | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index e540e310251f..6dc1a54187ee 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1659,7 +1659,7 @@ Different methods can be chosen for the comparison with the local file specified by ``file`` using the ``compare`` attribute: - ``diff``: uses diff to compare the history data set and the file provided by - ``file``. Compressed files are decompressed before the compariopm if + ``file``. Compressed files are decompressed before the comparison if ``decompress`` is set to ``true``. BAM files are converted to SAM before the comparision and for pdf some special rules are implemented. The number of allowed differences can be set with ``lines_diff``. If ``sort="true"`` history @@ -1677,6 +1677,10 @@ by ``file`` using the ``compare`` attribute: - ``sim_size``: compares the size of the history dataset and the ``file`` subject to the values of the ``delta`` and ``delta_frac`` attributes. Note that a ``has_size`` content assertion should be preferred, because this avoids storing the test file. +- ``image_diff``: compares the pixel data of the history data set and the file + provided by ``file``. The difference of the images is quantified according to their + pixel-wise distance with respect to a specific ``metric``. The check passes if the + distance is not larger than the value set for ``eps``. Only 2-D images can be used. ]]> @@ -1813,6 +1817,13 @@ will be infered from the last component of the location URL. For example, `locat If you specify a `checksum`, it will be also used to check the integrity of the download. + + + + + If ``compare`` is set to ``image_diff``, this is the maximum allowed distance between the data set that is generated in the test and the file in ``test-data/`` that is referenced by the ``file`` attribute, with distances computed with respect to the specified ``metric``. Default value is 0.01. + + @@ -7450,8 +7461,9 @@ and ``bibtex`` are the only supported options. Type of comparison to use when comparing test generated output files to expected output files. Currently valid value are -``diff`` (the default), ``re_match``, ``re_match_multiline``, -and ``contains``. In addition there is ``sim_size`` which is discouraged in favour of a ``has_size`` assertion. +``diff`` (the default), ``re_match``, ``re_match_multiline``, ``contains``, +and ``image_diff``. In addition there is ``sim_size`` which is discouraged in +favour of a ``has_size`` assertion. @@ -7461,6 +7473,18 @@ and ``contains``. In addition there is ``sim_size`` which is discouraged in favo + + + If ``compare`` is set to ``image_diff``, this is the metric used to compute the distance between images for quantification of their difference. For intensity images, possible metrics are *mean absolute error* (``mae``, the default), *mean squared error* (``mse``), *root mean squared* error (``rms``), and the *Frobenius norm* (``fro``). In addition, for binary images and label maps (with multiple objects), ``iou`` can be used to compute *one minus* the *intersection of the union* (IoU). Object correspondances are established by taking the pair of objects, for which the IoU is highest, and the distance of the images is the worst value determined for any pair of corresponding objects. + + + + + + + + + Documentation for PermissiveBoolean From d2a40cb955d5b3a05c4db8b66b124bd6b6dc1511 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 15:21:39 +0100 Subject: [PATCH 22/40] Fix linting issues --- test/unit/tool_util/test_verify.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index a8697b2452ef..e9f37eca86cf 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -71,7 +71,8 @@ def _encode_image(im, **kwargs): [255, 255, 255], ], dtype=float, - ) / 0xFF, + ) + / 0xFF, format="TIFF", ) From 2d858f12428da2000037451c9942282ce2dd8105 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 15:52:01 +0100 Subject: [PATCH 23/40] Fix bug --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index a4686193e5a1..c87c0792ad6f 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -442,10 +442,10 @@ def files_contains(file1, file2, attributes=None): def _multiobject_intersection_over_union(mask1, mask2, repeat_reverse=True): iou_list = list() - for label1 in mask1.unique(): + for label1 in numpy.unique(mask1): cc1 = mask1 == label1 cc1_iou_list = list() - for label2 in mask2[cc1].unique(): + for label2 in numpy.unique(mask2[cc1]): cc2 = mask2 == label2 cc1_iou_list.append(intersection_over_union(cc1, cc2)) iou_list.append(max(cc1_iou_list)) From b3d91d8082e521c469da012de0877094d4344a85 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 16:17:30 +0100 Subject: [PATCH 24/40] Fix bug --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index c87c0792ad6f..d205854d23f7 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -468,8 +468,8 @@ def get_image_metric(attributes): attributes = attributes or {} metrics = { "mae": lambda im1, im2: numpy.abs(im1 - im2).mean(), - "mse": lambda im1, im2: (im1 - im2).square().mean(), - "rms": lambda im1, im2: math.sqrt((im1 - im2).square().mean()), + "mse": lambda im1, im2: numpy.square(im1 - im2).mean(), + "rms": lambda im1, im2: math.sqrt(numpy.square(im1 - im2).mean()), "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), } From 88b707ad12765cd61c7f4a464ea9f7efff943079 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 16:45:30 +0100 Subject: [PATCH 25/40] Fix bug --- lib/galaxy/tool_util/verify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index d205854d23f7..8f4144957f58 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -450,7 +450,7 @@ def _multiobject_intersection_over_union(mask1, mask2, repeat_reverse=True): cc1_iou_list.append(intersection_over_union(cc1, cc2)) iou_list.append(max(cc1_iou_list)) if repeat_reverse: - iou_list += intersection_over_union(mask2, mask1, repeat_reverse=False) + iou_list += _multiobject_intersection_over_union(mask2, mask1, repeat_reverse=False) return iou_list From cc7791b341d1705480d76d3f7896b260dbcff114 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 17:12:28 +0100 Subject: [PATCH 26/40] Fix bug --- lib/galaxy/tool_util/verify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 8f4144957f58..67b779a36234 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -459,7 +459,7 @@ def intersection_over_union(mask1, mask2): assert mask1.ndim == mask2.ndim == 2 assert mask1.shape == mask2.shape if mask1.dtype == bool: - return numpy.logical_and(mask1, mask2) / numpy.logical_or(mask1, mask2) + return numpy.logical_and(mask1, mask2).sum() / numpy.logical_or(mask1, mask2).sum() else: return min(_multiobject_intersection_over_union(mask1, mask2)) From 3bb63ee5faaa4336fdab30fffe9bc376628bc3e3 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 19:22:33 +0100 Subject: [PATCH 27/40] Add functional test for `image_diff` --- test-data/im1_float.tif | Bin 0 -> 8464 bytes test-data/im1_uint8.png | Bin 0 -> 1124 bytes test-data/im1_uint8.tif | Bin 0 -> 1280 bytes test-data/im2_a.png | Bin 0 -> 75 bytes test-data/im2_b.png | Bin 0 -> 86 bytes test/functional/tools/image_diff.xml | 29 +++++++++++++++++++++ test/functional/tools/sample_tool_conf.xml | 1 + 7 files changed, 30 insertions(+) create mode 100644 test-data/im1_float.tif create mode 100644 test-data/im1_uint8.png create mode 100644 test-data/im1_uint8.tif create mode 100644 test-data/im2_a.png create mode 100644 test-data/im2_b.png create mode 100644 test/functional/tools/image_diff.xml diff --git a/test-data/im1_float.tif b/test-data/im1_float.tif new file mode 100644 index 0000000000000000000000000000000000000000..65395c1b6a2ce854258875fb1866910246cea279 GIT binary patch literal 8464 zcmYj$cRZEv|39UoC|T_?Dr968X*@NJ(i;u3HISqtQK^)a5m^-?5+y5yC}d@iV{eXg zIOlMdBuV0TzrVlFAMg9|xbDa6T<5y3`?|;VdX7t9e;W@!4-d}@9v)sk9$q4nBnt0; z;sv}5|F2$$s9*TM_}_Mm{}V6gUBDwo)Rhrs<^TE$6OF|G(?;SyW3KvN{O`rq@-F(n z@o7Zgb^nVwL@f88zN*Cdf<)aEQBM4?ejyS6d)$_GbF{m9L0VhVLS0Q!QeDmJ&j0`8 zKj(kv``>c?qP@NSMdu63S8x8O)xTk~h|TGtr$bPGY}?dEq2Z{%%cTx|DtslH18L$5?V?cKL)+d=xChq?pzSwER#e|f`pPaB!tR9yyB>VM4P4Sb->KO)9oL9Wq zty_%w*%NPf-x@};Sd^drnKJmb@c;8!n}cO;&N_dx$>8JG>~8VzLF3Ci?{9HPNROWQ z>|4n|$=xsVp{o5zxDjwZMumbBw>$C)d&6OUDC%KqITf18)C&FQR8*7)TCs|#s5}s( zuw9*l%&+wgp3Ng51<{?A7IeaIf!5&3>0W$&e<69!ga#h-m3If*Gf+f|*(!T_1gr%g zp7gtqpyKc6b`rZ5o=poBlf6q|($RN*W_uefv!en8bUQI;CVNZ>6daio5f>EcL;X~{ zLRmZ)-pSOqmzzGrOI=?3W??P%d}aMpn^b`?(g~M?v|-$eH~PcgL4t;Iy^KItKk`R@ z-#o3~gtS|Zz8lU|KrZaXv%vj@@FN`>eaka~^J}li%zP+?<26;A82K7ddXBlPMs=cK zL?|-+@E9289MRXk`Dkm+>bOQ4gRSD)uk@&VY(4D5lm!>#xuP+4bNx6hw97MoKNlIZ z4S$EtsYvn=TP>JO2l?-z_0GbH2vO~@d9bDh$)Vn<>fefyGtjkg>PHLYd<344pD)Jv z&SjfA&a|N;(2T3+(}QV^ar@~@nHVTJVDLqb2G@RfnIJdE)?uPyWXy@kd0`0QT{9E+Q167UU=DJ1czG9qe}*e=Nw_RM3zNE z*S3w)uYFsfchl+-$+ZgY5<4sNDv0-9vHqFJiwOMAxV1O^(HMA~W(-1hyFt)E_jp?7 z2o~&r?c1c=3@dxt*xW~S^ndvB8nqmpT(~A?b7vE3qMzL#x!g1O+Gd2WwOR zp&+91p!IlAJ-lr5?=rnOn3;T0{xP->_m+>nsS;-*;qMJg>9aJL^h?ia2(&^eC#m4| z(lJCIbBH}-+l|QD`IGaiEF3S?39X5uVCC~ZQ6)_lUOYEnpe0Yk2JbgF^mfqE%p>j6 zy_Ae@VOGwbNDgAnB2ISpnJcg~dsN0aQZh&~MOk9z4#28Ln1<=aZjm|l3j3-Nl8N#qHYDlO?p#5yq^ zoZm6hi>BvGyAuLf*gd-Cv}ye)(w=H5c)zU*BMS z?B~yuX=KE6vQDYAq(V`3(A6ud2HsT4GuvNb7!K=XAD`%;vn^B<7N94I$W2Po2H)|MB4-EHrBcZT>X&YXsb&? zyhV)9d~Q0fucDv3-O!7T#{@s!m~2H*2VKThkA{S)aX|^CE~toRbm{!9gwD*l%XWvO zu=;Mpi^G~(kQE;reO5Gr;`dH>tOfd^6?=JGvceF2x_jpyXR$C)PB!e#j zZYu=}+r>B&FUIg-Igj5f$zkZ5VPz?c3F#9KZh7~cG5IXOC(3^mhib^%m}OPS5N%~2 zx(2a(ipS!eOa4m!;S0kH-Exs;L#tFqzW+ z#jT{ke!<@dIf+bMJrj8(-HRt(^(;(rOQ@8X?H;zSF zUg4}F@}ZRO$2|8cFjC)$*aVcpGkM<8XwM+DS}NHVWgS?teN!RdY7UmOX3qxRVc>DI zwWH8aI!sdsk`FUkupR!bsc4Z4$^Gnrr>R4T&Q+FMl0(4(rH*>#Y7W*%9}Ni1Zh@dk zhxV=3Mvxu(r2Eqf5bv|8iro?d?HdAza*Ej)UvjE-^2!L7Pjcs{H;rPTG(NX~NedqT zEzlV(r(pWg1tS}qOx!3D1 zbg$TGpWJ|L;?w=jwj2l-_DNT4;-IPZyWz_4Q6vSmZ+)#e4A0ebrQ2qh@O)FcwX38B zNxW5Qf!mmOf+=SB&md84T0LJ+@pT@bsT0cVER0 zz}kR6@<96_oF+t9KD{xFwV8TU>hn%$@%H3R_?E);uS$2#@i7z%T*~7ftH%AH+wldK zWNed+A!a^f%9ASBYTB=kXX^I++EOs2v3c6^PeTqY5!B$Xr78Yddt&R zYSB@m5$dPw@EGR9nm;3x8K_s4Y~0dRhVeCv*8O1X9%lD@lm4QqyoGc+&g0isq~RBam8i@o2~a8hSE3b}Mda!A_Uh ztp!G1=sUR8adLYVcDR|fQ{E(<|+S3Om6Q=CNLMR}I7p?QL{j`XPMh z+hym@w~&pR@@Kyog`U)6uG7bl_&)qt;aD^oeg_$?l?kn|eVBY^_iYLs46>i3f8^ju zHjVutc^D=7vaXBQCE!q~TUuUVFVu40^es0bgI6y+$4YJp9fDCQzZynx%m*Gn-8oP%bo&_OOo6%6YsZ3v zW4MOtOAu$ncd4o4x7A}fMtg8A%b^1INq(=mn}%^jKGI?0UN`bve-}*(WkO_kgo2q& zFRq)k#Oelq~xSlL!s9&O}gbb`+3aFmfzE^m47&8?c)8*|3(ZTi1 z+V`GI;NODU%1@c_7u%7#Uw9NTM?$7Iy0<`OjJvIo)(C39Z_OU{5mZNPj-J2UgxXEH zRiBQJK=+E%i`pPEp6~l3b7XTbG}q=O@stcfFgoJ2kWC9xzyAn65JSV(lJM4rK|R=G zRPsSdv=f13xBC_k$FNBB#g3CvY{Z8j*?u_q9ag(;f0tj`fbrma+czKYz)r^{&osow zu$1-4c#A(5cLr-FNW5GGMm^ts`X>u}IKL#TW?E2U`pYu7n2W++Z-30Ons9YR%9WJ$ zUtq!vQP6PVV7T$~qEF-oq~8u0i4dWoiq_6FaWv*NRNhk@(lR4I9s-=B6>n5?{ zqE{|VtKlScR3g#q2E^L_aG-uC{LoX1Pw0seVmh2tbLhb!@k#r{fY+1c;2z#UoPsW zIL(2MII~>Ldk9CwRibjAH{-x4|Bj#^6g>Obcl}T{3+vy%ImTSl12+ZwiujZv7;M^Z zPKKt>Dj)i)$F{#Ams6 zq~LloC~v)c2KgAocbJg9-K-rvrv@By!`L_)mnOU+l8MZbtX*7oGLGfG=}F&Bg5}l? zzitQhfrr;VbGZ65qASGrxEOZ8Y5f!4p$s|_nl>Jk>9TAcm7{+r!EI}&_Vk5s)G!9ML(-!r9IC~ST4u<61OY;(Jhl&orjcYuEL-=D8g z+^%LQ_JszWpCtEyE#7e2t!DJQyAw|CyOrf<$jE8FHp0EnhLM_t(T34Hh`iRm|8u4r zCHx*cpN>R=uM=^n+3h&^iGRKEC>1TcOPbKvj0l?0I*IOD=v`8l`TVK~ykF#+IUW>@ zERdtT1q}-yj~qzz9>GEN;F^_I9PoVfV+d}sM@?mOX?yN4A}>;_F7Bklwe6JKWCxKS zi<3VtY~q0Y{9=3Kkx)$UE%$wPCyu(cyfg^|CwvG3djlV}9~(Z^(GGL|b%Ng9h8sl3@KqpTL)PEUTUc$OfcOacI!)! z@kd5sY|w`dy_Vagza~S-<}cV*5Xi=h1zXtqp3%FYWuP}K7*)4)>467CiNTt&eD^zlbUC6>Uo4(U^ zz9X0oUgaXnCgaJ&kZ3_--9J)X=y#mUMcI)mzIDo-nE6Nb+xkrd=!svdqp+h3t@np| z^~|{7Q;YkoVNnSQOS7}3f7@_eu_46i+Xz;h$MxGNXTmyd&)(-Qgf6Vz^`=c9zJX(%<*ju%(;kwxGv)g@;C?^Pq9agK$&bMD6BgX5jzGlX8P zd#3K>+W}XZk}KunbUaqbY!P)B#_Z~q{{DPpkci$Ny>~AcduXkJDmPhpyW@JAfd>Oe z%OzBW%xD-EN;YcnY=`BOea@D2!q-a6DjDU^gr~AvSi)2zZdHg++)L@j%D=ZCtLBh# z_|vESHAM_uO8;_=H-q3ORt`%)@fSk}sbO*Fzu@loVvni;3JzJ8uQ2SP<735+LxP)` z*yek7iy1o3Jnu$^R+imG(=L#l_6lBLmLV~!H2%-n2wYA^zEkBR z@M?>|F}~+a=%`T7(nj*JUScvWymt_`uO3V7ilpG7Z=kU$vk}hvWrJ}E-*92g^VARb znHXoDS!pnxj=ohjGVeAHpn9|UievAJFr&QkyqiuxcHJEK(QUzn;HP>W-i^JGik90G zdYOr7-&;@56Zeztlai`x*$l7dq5DOh1Ro3WO!g(_E7E*diggf^9c&MhSjM8_$4kaqOr{5zuyFJ6~pc8g=3)sq?=w{tqTFs~N_ z91W$TbYgw0hmf?bIuB9SeP?-_jUs37=Xr?UH3A7XvSh^OhcCKrAEaPxi@9Bv*tcwFw>f}s})=De%FI% zL)iK3{i#DWYy_|VN)=5Z!->Djm${}H)J&=Px1ubVopwC>uz}D&#Yan2QclBDD*5T= zoF>fiM~jVHwc(6pIA!f^CLAw~rEDD9uUkJSJzgX+fJs5bwSDK1|Lh8uP{uqSkox#%jX9w=*db z_`Q$`MsSpkbVDhuB^*=iO$a}*--wh#e3#iK4CRwbV@OWh`L#y#Gd5rKiLvIZK*z-I zn!g^c7{9Mz_r>5N^lbNkNmuFx$xp(o{mB*$YohV&sv|gHl~eYt<+ZOOPMrm_i0&4_vc`> zT0_L9M;&-q6&szJ&w|X{on2vX7!V9JOIk+mKxlXCXvQ!V-kKSb9FG>P`Nr`R|HDLA zWb|2q>Ov&`+4@XSwGKM-RsIQ=t1vq<9;Q^+jWx;i+KDbYG*0C$xZFoZ+X=Z+iJgPs zF*D4hQ1|2Wl#g8Xu3k)Ebaz~FI1>*C!bX-R*W#;{IZ2^|4Z$x7YZSujVC36-*V}`M zwz>GLhFPOfym0dZ-MRwmIgF)qcEr4ByLRx!6e0AN)4DzbI)0358mzJ>{9>E65%zyZ z@!-6>-=2&C`2R7U3_e2efXdZF2M@i+TKAw%b!+J?h2 zl@Qjt&@S)#1&bDKH&2u$_7Nz#UcYq-+@5>(D`hteYD32=*Lo0s2F2#!q*f0$C&d*- z(+S+Iy8L&YHyOVV4EHLz5kBCaJE$S@r#D5z=*T*ETn{XK6tI)n51>ldM-#qqrCY!! zA%dT(_&(eqnOuq4A~C+H?*rJgaG41CXDNCrr%zV>?m(H?PjcTZ8^Q}8g?%FU(v#XJ zMf1ddgkns0mrr3kyw5XYziUuYn`A2O)51Yp_wwtq1kN2veR<#H8VSp{1-U9sGLg?C z5iEau5ILW(2pLqcFqvq|6#QC+QO`4Tua*sAwcxGA(L-(U8-HD}`#>8GEwbr&9o2^$ z(NebpJ`p;(>#Xayfe!pi8m+pj%ZBpC?bWVMbbLEr7B8$vhkePm&EJDskjQucW~3Jd zvy5Lk0$c{p|0(3$bRPvH()>=1UX9TE3_UZJcn!)T=2;nrNj z*Lh><7_+YrJ9k7IoRJ#D)r{Y7#IpM!ax&AlPJu#P*YxBoG8M;F;sO;ZgGdwna^g!N z6Oy-u)Ey*-;mEqvGnd(otCV$g(bOIsUQpX|ZekQU?3~_8y$)m!?LPlQnDAwvv~RxY zkO9evs_Pf|S-7(3*P}1z*^o2(vsbO87~0B5g7_TU;TRqj_`|x;H{-;Z(rP^?P$KH%m{tj2^qJ;^G}Z28ovVEh!;97baem)_jt47_Ksnnsrwag@m83fqJC&MF2|!O zqcf{z+99MVWPN~~02BoWol%+;3tXLF9i!^A!`k zV-z@#3C!l-Wkcb0U`~F;D3*(LZ+*Otjxqr&r;_0zcy!fo^(1srj4IWs`@tYi++sK9@}i-12I2E)@T;=@Ffrz(wfB7+GMk*kzuaIO!8d9frgQzzQLgel-++pmq{H#zD33| z@-ziJDrHtjVjg1B!u1o^Z2(nd<$cJYfy^vQobY*z1FXC~OSiCaHt|OF{X{Z&KIv|X zxey04iNDRy3hEK|c6j)~Lk>DBo%K6hG`F~C;$9oaAFW%Ujh|7RGMJMWnGsdbgjqW&qaZ28OZFQ_mZ|ZV(*m2 YOX*%JUYr>duJIVgh`hhbHFFOBA9L_+CIA2c literal 0 HcmV?d00001 diff --git a/test-data/im1_uint8.png b/test-data/im1_uint8.png new file mode 100644 index 0000000000000000000000000000000000000000..c629ee397615b666220290a11383b38a6a6fe19f GIT binary patch literal 1124 zcmV-q1e^PbP)n{y7-;5p`^nRK)$LsTpd;-4B%zQe02+Ex+3;UjQ7X1MYe?gQgQ19%l+HCooqYMIM zrvm8LEW&+8`Pzu_)F!%N;=r^~P+XZNlcV}d} z)oVp8a2+f(V)kRj0J3zr-TB%BT)HfU4l@PHIauwq2r8bFJ?@=}mby@8HUxdb%%Hso z3=@`yt{~G4<2x1jNiTAm!n|Y_7G~Is=y(&6`D}75{u4H?SGgLzclvaLPqu zuw()$V0&Q%t+qqxhFz@F!}6e|Q5+6paL6A2;qggGfKG}4(df`Apk@$kLfHuQTU!J# zX6+gA-gHwtod^VN5kNRW03ib%p7{M}Dx-b;5y(nDsXwZOY>i2@YFyLo3_DrS`vCXn zO#MI`@;tB)y5U7(U%x#pb57PhfbgQ-@V~P6{&N8mIow0HdB}njl)F(*gC)2x^4UVp z5GOg7m+&w(F{wTTtMTy`%Dq+ml)b`l9bNjSEK$c8P^i|Jb;bJT4`e6(0y$O_8czNm zIp#g7VmJyI_lLUlhvB4E%|S*$7c9P7w*oWXz!|4hru#v?#R3O}q)(=@- z!Ut{_Rqz3W+ZN)XEB8M?5xtsQUh6C>>frCB(CRKQ7eFs6{d*_@*cS|I`^Aa)H9ck( zs_n2eX{J|&a&JQ?BL79LUhTsO0iAWg)e_K?O91P(v|xW2?^7kiDuLf0*VNR1Q;6E! z0RcGTD$M}NPi8i+vkW^K(r)=I8{L3>&`idgBjEk!J1_(fY?v719@w7^q0HAli}b}| zm_7XaAU~ms81eRt5Da4kuhb@z4l?Rs*lFb-ElP^|Ee<9#SsPNWy{X&vcG?0v1QeOY zB5vmh4{_g>L??1S85KBJIQHe+(e}_jq-k7ng8;gt&wA+%OSWq)8_;>-yJ%m7a{IV)hGzA6DT3-NVKNWs2R| z&mS3k&iQJOSlsw(r$**c0L3@v3so6;(M%>(&?Gt%v}I4XR=UtQ_$f** zSbBxp;{Z6H9CcDrd%6eoQ19`Oz q;Drb|^OPG&bOmXBK9;~do}=;^-T1GS?M`q20000_&Qd z#zr~{#zwKV5D6p_dN3$y z+LZ<|C>i;;Zem z`)==aRpRq?E?KwHBT?@p-=^cooZ_ds#T%7!pAl*CoBVYtXI@k1I(NmCF9y%MOCOf* zOnbgzQ^>(ZwwkBEmQKH2`g!uj7Z)@aqzU9W-Q@Th9nGSj_Ez%ay^>(-xg0FHf({mr zip;X}e*DeQT3q`_@PwD$O8ZqUIlZ1+vSKd3;jxaq@ca97pT7>$pKLbp?|SH(7{A|E zr`YeBZNrB}cR%dk^!;D4khgl4;;f`6-@DzWKh5ZMElIO)^p9`bANuAFk7JQ^YGHK0drDOijQpul48`Iq7NpZoL1UmZ-LV`5npnDR$bo`77i~f92ek zb4^cM9kFe3THaitOUrk5%A}_UaqN+qy?N1Bsk`hQ>20xwtS#5$Ek(sFq+A3AZ)x@) zEm1elYmYpB;0^nelm{FB9Exp7h;^=8=iVl#$@7CzZ;t509p4&#d)~IXJcwto54>Gk z820ev--K_c*4UYAz4k7tJ9b?;_xeRC4L{)=>529;ATAYz^ZtLf$r$?ga zd)=2c(@)>mR>P1ZVEWGd_F2tYvbz?auX@Sry)9cu`a}#S|#TF zYme}m-_)G$*FM$n2o5aDi+;I#m)GRurz4K;QMp@KQ#5g2{oMs=S?43o`0u}x57m`> z%KuC9&V;z|g8?gAYM%98-pO^;{26zsWaUL4wU7%cRzh1+{kMnhx?u4`(@QU+s`b`m zi}^C8L4nn~gdXhHec$-ssKd)|`s=eh`(M}HziNJzImo(kk7V*0K+W^%>V!Z literal 0 HcmV?d00001 diff --git a/test-data/im2_a.png b/test-data/im2_a.png new file mode 100644 index 0000000000000000000000000000000000000000..166cdb5b319bfe61d3bcc1f6903ad45514ee8606 GIT binary patch literal 75 zcmeAS@N?(olHy`uVBq!ia0vp^3Lwk@BpAX3RW*Q=kf)1dNX4Aw1PRu~2_ijAZx|R@ W6c`xWHru*@RC&7kxvX + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/sample_tool_conf.xml b/test/functional/tools/sample_tool_conf.xml index 8df30a78eff6..77539c2433db 100644 --- a/test/functional/tools/sample_tool_conf.xml +++ b/test/functional/tools/sample_tool_conf.xml @@ -8,6 +8,7 @@ + From a72db4c169fcf5ef72a10200f1e93b477d7e6f4d Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 20:25:19 +0100 Subject: [PATCH 28/40] Fix functional test for `image_diff` comparison --- lib/galaxy/tool_util/parser/util.py | 3 +++ lib/galaxy/tool_util/parser/xml.py | 5 +++++ lib/galaxy/tool_util/verify/__init__.py | 6 ++++-- lib/galaxy/tool_util/xsd/galaxy.xsd | 1 + test/functional/tools/image_diff.xml | 8 ++++---- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tool_util/parser/util.py b/lib/galaxy/tool_util/parser/util.py index 9ecec25559f6..228312b1af46 100644 --- a/lib/galaxy/tool_util/parser/util.py +++ b/lib/galaxy/tool_util/parser/util.py @@ -3,6 +3,9 @@ DEFAULT_DELTA = 10000 DEFAULT_DELTA_FRAC = None +DEFAULT_METRIC = "mae" +DEFAULT_EPS = 0.01 + def is_dict(item): return isinstance(item, dict) or isinstance(item, OrderedDict) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 5c723c3a851f..38e625f3eeb8 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -19,6 +19,8 @@ from galaxy.tool_util.parser.util import ( DEFAULT_DELTA, DEFAULT_DELTA_FRAC, + DEFAULT_METRIC, + DEFAULT_EPS, ) from galaxy.util import ( Element, @@ -788,6 +790,9 @@ def __parse_test_attributes(output_elem, attrib, parse_elements=False, parse_dis attributes["decompress"] = string_as_bool(attrib.pop("decompress", False)) # `location` may contain an URL to a remote file that will be used to download `file` (if not already present on disk). location = attrib.get("location") + # Parameters for "image_diff" comparison + attributes["metric"] = attrib.pop("metric", DEFAULT_METRIC) + attributes["eps"] = float(attrib.pop("eps", DEFAULT_EPS)) if location and file is None: file = os.path.basename(location) # If no file specified, try to get filename from URL last component attributes["location"] = location diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 67b779a36234..49b1dfd4c828 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -29,6 +29,8 @@ from galaxy.tool_util.parser.util import ( DEFAULT_DELTA, DEFAULT_DELTA_FRAC, + DEFAULT_METRIC, + DEFAULT_EPS, ) from galaxy.util import unicodify from galaxy.util.compression_utils import get_fileobj @@ -473,7 +475,7 @@ def get_image_metric(attributes): "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), } - return metrics[attributes.get("metric", "mae")] + return metrics[attributes.get("metric", DEFAULT_METRIC)] def files_image_diff(file1, file2, attributes=None): @@ -490,6 +492,6 @@ def files_image_diff(file1, file2, attributes=None): raise AssertionError(f"Image dimensions did not match ({im1.shape}, {im2.shape}).") distance = get_image_metric(attributes)(im1, im2) - distance_eps = attributes.get("eps", 0.01) + distance_eps = attributes.get("eps", DEFAULT_EPS) if distance > distance_eps: raise AssertionError(f"Image difference {distance} exceeds eps={distance_eps}.") diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 6dc1a54187ee..2c3c90828f9f 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -7471,6 +7471,7 @@ favour of a ``has_size`` assertion. + diff --git a/test/functional/tools/image_diff.xml b/test/functional/tools/image_diff.xml index c65c7eca8f5c..b0bad1cb5466 100644 --- a/test/functional/tools/image_diff.xml +++ b/test/functional/tools/image_diff.xml @@ -11,19 +11,19 @@ - + - + - + - + From cea6a6c84634df6c6a1af4bb736ac4cbe634e8c6 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 21:08:52 +0100 Subject: [PATCH 29/40] Add error handler --- lib/galaxy/tool_util/verify/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 49b1dfd4c828..9fee6afcf73d 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -467,6 +467,7 @@ def intersection_over_union(mask1, mask2): def get_image_metric(attributes): + metric_name = attributes.get("metric", DEFAULT_METRIC) attributes = attributes or {} metrics = { "mae": lambda im1, im2: numpy.abs(im1 - im2).mean(), @@ -475,7 +476,10 @@ def get_image_metric(attributes): "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), } - return metrics[attributes.get("metric", DEFAULT_METRIC)] + try: + return metrics[metric_name] + except KeyError: + raise ValueError(f"No such metric: \"{metric_name}\"") def files_image_diff(file1, file2, attributes=None): From 14e179785db5b75bf9125b8d0c59adbc6f2ca196 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 21:09:09 +0100 Subject: [PATCH 30/40] Update functional test --- test/functional/tools/image_diff.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/tools/image_diff.xml b/test/functional/tools/image_diff.xml index b0bad1cb5466..754f9353af4b 100644 --- a/test/functional/tools/image_diff.xml +++ b/test/functional/tools/image_diff.xml @@ -15,11 +15,11 @@ - + - + From 49fe197d8be3a1e83a48bdf8be47e0a6f38a3f63 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 21:40:20 +0100 Subject: [PATCH 31/40] Fix bugs --- lib/galaxy/tool_util/verify/__init__.py | 4 ++-- test/unit/tool_util/test_verify.py | 27 ++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 9fee6afcf73d..9fd85ec9d2ca 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -471,8 +471,8 @@ def get_image_metric(attributes): attributes = attributes or {} metrics = { "mae": lambda im1, im2: numpy.abs(im1 - im2).mean(), - "mse": lambda im1, im2: numpy.square(im1 - im2).mean(), - "rms": lambda im1, im2: math.sqrt(numpy.square(im1 - im2).mean()), + "mse": lambda im1, im2: numpy.square((im1 - im2).astype(float)).mean(), + "rms": lambda im1, im2: math.sqrt(numpy.square((im1 - im2).astype(float)).mean()), "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), } diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index e9f37eca86cf..2d5f2f4a3313 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -1,6 +1,7 @@ import collections import gzip import io +import math import tempfile from typing import ( Any, @@ -75,6 +76,17 @@ def _encode_image(im, **kwargs): / 0xFF, format="TIFF", ) +F9 = _encode_image( + numpy.array( + [ + [0, 0, 0], + [0, 1, 0], + [0, 1, 2], + ], + dtype=numpy.uint8, + ), + format="PNG", +) def _test_file_list(): @@ -89,6 +101,7 @@ def _test_file_list(): (F6, ".png"), (F7, ".tiff"), (F8, ".tiff"), + (F9, ".png"), ]: with tempfile.NamedTemporaryFile(mode="wb", suffix=ext, delete=False) as out: if ext == ".txt.gz": @@ -99,7 +112,7 @@ def _test_file_list(): def generate_tests(multiline=False): - f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() + f1, f2, f3, f4, multiline_match, f5, f6, f7, f8, f9 = _test_file_list() tests: List[TestDef] if multiline: tests = [(multiline_match, f1, {"lines_diff": 0, "sort": True}, None)] @@ -117,7 +130,7 @@ def generate_tests(multiline=False): def generate_tests_sim_size(): - f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() + f1, f2, f3, f4, multiline_match, f5, f6, f7, f8, f9 = _test_file_list() # tests for equal files tests: List[TestDef] = [ (f1, f1, None, None), # pass default values @@ -143,7 +156,7 @@ def generate_tests_sim_size(): def generate_tests_image_diff(): - f1, f2, f3, f4, multiline_match, f5, f6, f7, f8 = _test_file_list() + f1, f2, f3, f4, multiline_match, f5, f6, f7, f8, f9 = _test_file_list() metrics = ["mae", "mse", "rms", "fro", "iou"] # tests for equal files (uint8, PNG) tests: List[TestDef] = [(f6, f6, {"metric": metric}, None) for metric in metrics] @@ -158,6 +171,14 @@ def generate_tests_image_diff(): (f6, f7, {"metric": "iou"}, None), (f6, f7, {"metric": "mae", "eps": 100 / 9 + 1e-4}, None), (f6, f7, {"metric": "mae", "eps": 100 / 9 - 1e-4}, AssertionError), + (f6, f7, {"metric": "mse", "eps": (100 ** 2) / 9 + 1e-4}, None), + (f6, f7, {"metric": "mse", "eps": (100 ** 2) / 9 - 1e-4}, AssertionError), + (f6, f7, {"metric": "rms", "eps": math.sqrt((100 ** 2) / 9) + 1e-4}, None), + (f6, f7, {"metric": "rms", "eps": math.sqrt((100 ** 2) / 9) - 1e-4}, AssertionError), + (f6, f7, {"metric": "fro", "eps": 100 + 1e-4}, None), + (f6, f7, {"metric": "fro", "eps": 100 - 1e-4}, AssertionError), + (f6, f9, {"metric": "iou", "eps": (1 - 1 / 8) + 1e-4}, None), + (f6, f9, {"metric": "iou", "eps": (1 - 1 / 8) - 1e-4}, AssertionError), ] return tests From e034928e2fb0afb8147635c0ce1939c293aee50e Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 21:54:23 +0100 Subject: [PATCH 32/40] Fix bugs --- lib/galaxy/tool_util/verify/__init__.py | 2 +- test-data/im3_a.png | Bin 0 -> 82 bytes test-data/im3_b.tif | Bin 0 -> 3344 bytes test/functional/tools/image_diff.xml | 7 +++++++ 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 test-data/im3_a.png create mode 100644 test-data/im3_b.tif diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 9fd85ec9d2ca..21092b051424 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -473,7 +473,7 @@ def get_image_metric(attributes): "mae": lambda im1, im2: numpy.abs(im1 - im2).mean(), "mse": lambda im1, im2: numpy.square((im1 - im2).astype(float)).mean(), "rms": lambda im1, im2: math.sqrt(numpy.square((im1 - im2).astype(float)).mean()), - "fro": lambda im1, im2: numpy.linalg.norm(im1 - im2, "fro"), + "fro": lambda im1, im2: numpy.linalg.norm((im1 - im2).reshape(1, -1), "fro"), "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), } try: diff --git a/test-data/im3_a.png b/test-data/im3_a.png new file mode 100644 index 0000000000000000000000000000000000000000..8c1870d9916639a823063b9ff3956cabbfb83a3e GIT binary patch literal 82 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1SJ1Ryj={WBt2amLn`LHJ!r@X5Ga5d~X#js{V9FxvC5DY@0)7!<%tTg$-!x2FWWB_&F-^cPLX4Tn hYVezeDT}O^7&fK}qavdrFd71*Aut*OqalD90sx(h5y$`l literal 0 HcmV?d00001 diff --git a/test/functional/tools/image_diff.xml b/test/functional/tools/image_diff.xml index 754f9353af4b..a079992e5fe5 100644 --- a/test/functional/tools/image_diff.xml +++ b/test/functional/tools/image_diff.xml @@ -9,6 +9,7 @@ + @@ -21,9 +22,15 @@ + + + + + + From 8567aced02f61de7f46f0cafea0da72e69811e16 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 22:30:21 +0100 Subject: [PATCH 33/40] Fix linting issues --- lib/galaxy/tool_util/verify/__init__.py | 2 +- test/unit/tool_util/test_verify.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 21092b051424..849c4447959f 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -479,7 +479,7 @@ def get_image_metric(attributes): try: return metrics[metric_name] except KeyError: - raise ValueError(f"No such metric: \"{metric_name}\"") + raise ValueError(f'No such metric: "{metric_name}"') def files_image_diff(file1, file2, attributes=None): diff --git a/test/unit/tool_util/test_verify.py b/test/unit/tool_util/test_verify.py index 2d5f2f4a3313..db6955e08d69 100644 --- a/test/unit/tool_util/test_verify.py +++ b/test/unit/tool_util/test_verify.py @@ -171,10 +171,10 @@ def generate_tests_image_diff(): (f6, f7, {"metric": "iou"}, None), (f6, f7, {"metric": "mae", "eps": 100 / 9 + 1e-4}, None), (f6, f7, {"metric": "mae", "eps": 100 / 9 - 1e-4}, AssertionError), - (f6, f7, {"metric": "mse", "eps": (100 ** 2) / 9 + 1e-4}, None), - (f6, f7, {"metric": "mse", "eps": (100 ** 2) / 9 - 1e-4}, AssertionError), - (f6, f7, {"metric": "rms", "eps": math.sqrt((100 ** 2) / 9) + 1e-4}, None), - (f6, f7, {"metric": "rms", "eps": math.sqrt((100 ** 2) / 9) - 1e-4}, AssertionError), + (f6, f7, {"metric": "mse", "eps": (100**2) / 9 + 1e-4}, None), + (f6, f7, {"metric": "mse", "eps": (100**2) / 9 - 1e-4}, AssertionError), + (f6, f7, {"metric": "rms", "eps": math.sqrt((100**2) / 9) + 1e-4}, None), + (f6, f7, {"metric": "rms", "eps": math.sqrt((100**2) / 9) - 1e-4}, AssertionError), (f6, f7, {"metric": "fro", "eps": 100 + 1e-4}, None), (f6, f7, {"metric": "fro", "eps": 100 - 1e-4}, AssertionError), (f6, f9, {"metric": "iou", "eps": (1 - 1 / 8) + 1e-4}, None), From 356d3bd59071f409a74ae7d23673acf9f2e5b845 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Tue, 27 Feb 2024 22:42:49 +0100 Subject: [PATCH 34/40] Fix linting issues --- lib/galaxy/tool_util/parser/xml.py | 2 +- lib/galaxy/tool_util/verify/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 38e625f3eeb8..f4f6a8791096 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -19,8 +19,8 @@ from galaxy.tool_util.parser.util import ( DEFAULT_DELTA, DEFAULT_DELTA_FRAC, - DEFAULT_METRIC, DEFAULT_EPS, + DEFAULT_METRIC, ) from galaxy.util import ( Element, diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 849c4447959f..ea3ade7d25c7 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -29,8 +29,8 @@ from galaxy.tool_util.parser.util import ( DEFAULT_DELTA, DEFAULT_DELTA_FRAC, - DEFAULT_METRIC, DEFAULT_EPS, + DEFAULT_METRIC, ) from galaxy.util import unicodify from galaxy.util.compression_utils import get_fileobj From aa79578b6c957dab7083fe96189af4a748135eb4 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Wed, 28 Feb 2024 10:11:46 +0100 Subject: [PATCH 35/40] Fix spelling --- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 2c3c90828f9f..ee23085f632c 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -7476,7 +7476,7 @@ favour of a ``has_size`` assertion. - If ``compare`` is set to ``image_diff``, this is the metric used to compute the distance between images for quantification of their difference. For intensity images, possible metrics are *mean absolute error* (``mae``, the default), *mean squared error* (``mse``), *root mean squared* error (``rms``), and the *Frobenius norm* (``fro``). In addition, for binary images and label maps (with multiple objects), ``iou`` can be used to compute *one minus* the *intersection of the union* (IoU). Object correspondances are established by taking the pair of objects, for which the IoU is highest, and the distance of the images is the worst value determined for any pair of corresponding objects. + If ``compare`` is set to ``image_diff``, this is the metric used to compute the distance between images for quantification of their difference. For intensity images, possible metrics are *mean absolute error* (``mae``, the default), *mean squared error* (``mse``), *root mean squared* error (``rms``), and the *Frobenius norm* (``fro``). In addition, for binary images and label maps (with multiple objects), ``iou`` can be used to compute *one minus* the *intersection over the union* (IoU). Object correspondances are established by taking the pair of objects, for which the IoU is highest, and the distance of the images is the worst value determined for any pair of corresponding objects. From 90761750e61fb5b84682e679ae7d459e1f7b5630 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Wed, 28 Feb 2024 10:37:35 +0100 Subject: [PATCH 36/40] Add linting for `image_diff` --- lib/galaxy/tool_util/linters/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py index 97de651b65bd..9ea7dbf081c2 100644 --- a/lib/galaxy/tool_util/linters/tests.py +++ b/lib/galaxy/tool_util/linters/tests.py @@ -278,6 +278,8 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): "decompress": ["diff"], "delta": ["sim_size"], "delta_frac": ["sim_size"], + "metric": ["image_diff"], + "eps": ["image_diff"], } for test_idx, test in enumerate(tests, start=1): for output in test.xpath(".//*[self::output or self::element or self::discovered_dataset]"): From 711fd6a262a944b3a757c027356757d7a0257416 Mon Sep 17 00:00:00 2001 From: Leonid Kostrykin Date: Wed, 28 Feb 2024 12:08:22 +0100 Subject: [PATCH 37/40] Remove unrequired test file --- test-data/im1_float.tif | Bin 8464 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test-data/im1_float.tif diff --git a/test-data/im1_float.tif b/test-data/im1_float.tif deleted file mode 100644 index 65395c1b6a2ce854258875fb1866910246cea279..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8464 zcmYj$cRZEv|39UoC|T_?Dr968X*@NJ(i;u3HISqtQK^)a5m^-?5+y5yC}d@iV{eXg zIOlMdBuV0TzrVlFAMg9|xbDa6T<5y3`?|;VdX7t9e;W@!4-d}@9v)sk9$q4nBnt0; z;sv}5|F2$$s9*TM_}_Mm{}V6gUBDwo)Rhrs<^TE$6OF|G(?;SyW3KvN{O`rq@-F(n z@o7Zgb^nVwL@f88zN*Cdf<)aEQBM4?ejyS6d)$_GbF{m9L0VhVLS0Q!QeDmJ&j0`8 zKj(kv``>c?qP@NSMdu63S8x8O)xTk~h|TGtr$bPGY}?dEq2Z{%%cTx|DtslH18L$5?V?cKL)+d=xChq?pzSwER#e|f`pPaB!tR9yyB>VM4P4Sb->KO)9oL9Wq zty_%w*%NPf-x@};Sd^drnKJmb@c;8!n}cO;&N_dx$>8JG>~8VzLF3Ci?{9HPNROWQ z>|4n|$=xsVp{o5zxDjwZMumbBw>$C)d&6OUDC%KqITf18)C&FQR8*7)TCs|#s5}s( zuw9*l%&+wgp3Ng51<{?A7IeaIf!5&3>0W$&e<69!ga#h-m3If*Gf+f|*(!T_1gr%g zp7gtqpyKc6b`rZ5o=poBlf6q|($RN*W_uefv!en8bUQI;CVNZ>6daio5f>EcL;X~{ zLRmZ)-pSOqmzzGrOI=?3W??P%d}aMpn^b`?(g~M?v|-$eH~PcgL4t;Iy^KItKk`R@ z-#o3~gtS|Zz8lU|KrZaXv%vj@@FN`>eaka~^J}li%zP+?<26;A82K7ddXBlPMs=cK zL?|-+@E9289MRXk`Dkm+>bOQ4gRSD)uk@&VY(4D5lm!>#xuP+4bNx6hw97MoKNlIZ z4S$EtsYvn=TP>JO2l?-z_0GbH2vO~@d9bDh$)Vn<>fefyGtjkg>PHLYd<344pD)Jv z&SjfA&a|N;(2T3+(}QV^ar@~@nHVTJVDLqb2G@RfnIJdE)?uPyWXy@kd0`0QT{9E+Q167UU=DJ1czG9qe}*e=Nw_RM3zNE z*S3w)uYFsfchl+-$+ZgY5<4sNDv0-9vHqFJiwOMAxV1O^(HMA~W(-1hyFt)E_jp?7 z2o~&r?c1c=3@dxt*xW~S^ndvB8nqmpT(~A?b7vE3qMzL#x!g1O+Gd2WwOR zp&+91p!IlAJ-lr5?=rnOn3;T0{xP->_m+>nsS;-*;qMJg>9aJL^h?ia2(&^eC#m4| z(lJCIbBH}-+l|QD`IGaiEF3S?39X5uVCC~ZQ6)_lUOYEnpe0Yk2JbgF^mfqE%p>j6 zy_Ae@VOGwbNDgAnB2ISpnJcg~dsN0aQZh&~MOk9z4#28Ln1<=aZjm|l3j3-Nl8N#qHYDlO?p#5yq^ zoZm6hi>BvGyAuLf*gd-Cv}ye)(w=H5c)zU*BMS z?B~yuX=KE6vQDYAq(V`3(A6ud2HsT4GuvNb7!K=XAD`%;vn^B<7N94I$W2Po2H)|MB4-EHrBcZT>X&YXsb&? zyhV)9d~Q0fucDv3-O!7T#{@s!m~2H*2VKThkA{S)aX|^CE~toRbm{!9gwD*l%XWvO zu=;Mpi^G~(kQE;reO5Gr;`dH>tOfd^6?=JGvceF2x_jpyXR$C)PB!e#j zZYu=}+r>B&FUIg-Igj5f$zkZ5VPz?c3F#9KZh7~cG5IXOC(3^mhib^%m}OPS5N%~2 zx(2a(ipS!eOa4m!;S0kH-Exs;L#tFqzW+ z#jT{ke!<@dIf+bMJrj8(-HRt(^(;(rOQ@8X?H;zSF zUg4}F@}ZRO$2|8cFjC)$*aVcpGkM<8XwM+DS}NHVWgS?teN!RdY7UmOX3qxRVc>DI zwWH8aI!sdsk`FUkupR!bsc4Z4$^Gnrr>R4T&Q+FMl0(4(rH*>#Y7W*%9}Ni1Zh@dk zhxV=3Mvxu(r2Eqf5bv|8iro?d?HdAza*Ej)UvjE-^2!L7Pjcs{H;rPTG(NX~NedqT zEzlV(r(pWg1tS}qOx!3D1 zbg$TGpWJ|L;?w=jwj2l-_DNT4;-IPZyWz_4Q6vSmZ+)#e4A0ebrQ2qh@O)FcwX38B zNxW5Qf!mmOf+=SB&md84T0LJ+@pT@bsT0cVER0 zz}kR6@<96_oF+t9KD{xFwV8TU>hn%$@%H3R_?E);uS$2#@i7z%T*~7ftH%AH+wldK zWNed+A!a^f%9ASBYTB=kXX^I++EOs2v3c6^PeTqY5!B$Xr78Yddt&R zYSB@m5$dPw@EGR9nm;3x8K_s4Y~0dRhVeCv*8O1X9%lD@lm4QqyoGc+&g0isq~RBam8i@o2~a8hSE3b}Mda!A_Uh ztp!G1=sUR8adLYVcDR|fQ{E(<|+S3Om6Q=CNLMR}I7p?QL{j`XPMh z+hym@w~&pR@@Kyog`U)6uG7bl_&)qt;aD^oeg_$?l?kn|eVBY^_iYLs46>i3f8^ju zHjVutc^D=7vaXBQCE!q~TUuUVFVu40^es0bgI6y+$4YJp9fDCQzZynx%m*Gn-8oP%bo&_OOo6%6YsZ3v zW4MOtOAu$ncd4o4x7A}fMtg8A%b^1INq(=mn}%^jKGI?0UN`bve-}*(WkO_kgo2q& zFRq)k#Oelq~xSlL!s9&O}gbb`+3aFmfzE^m47&8?c)8*|3(ZTi1 z+V`GI;NODU%1@c_7u%7#Uw9NTM?$7Iy0<`OjJvIo)(C39Z_OU{5mZNPj-J2UgxXEH zRiBQJK=+E%i`pPEp6~l3b7XTbG}q=O@stcfFgoJ2kWC9xzyAn65JSV(lJM4rK|R=G zRPsSdv=f13xBC_k$FNBB#g3CvY{Z8j*?u_q9ag(;f0tj`fbrma+czKYz)r^{&osow zu$1-4c#A(5cLr-FNW5GGMm^ts`X>u}IKL#TW?E2U`pYu7n2W++Z-30Ons9YR%9WJ$ zUtq!vQP6PVV7T$~qEF-oq~8u0i4dWoiq_6FaWv*NRNhk@(lR4I9s-=B6>n5?{ zqE{|VtKlScR3g#q2E^L_aG-uC{LoX1Pw0seVmh2tbLhb!@k#r{fY+1c;2z#UoPsW zIL(2MII~>Ldk9CwRibjAH{-x4|Bj#^6g>Obcl}T{3+vy%ImTSl12+ZwiujZv7;M^Z zPKKt>Dj)i)$F{#Ams6 zq~LloC~v)c2KgAocbJg9-K-rvrv@By!`L_)mnOU+l8MZbtX*7oGLGfG=}F&Bg5}l? zzitQhfrr;VbGZ65qASGrxEOZ8Y5f!4p$s|_nl>Jk>9TAcm7{+r!EI}&_Vk5s)G!9ML(-!r9IC~ST4u<61OY;(Jhl&orjcYuEL-=D8g z+^%LQ_JszWpCtEyE#7e2t!DJQyAw|CyOrf<$jE8FHp0EnhLM_t(T34Hh`iRm|8u4r zCHx*cpN>R=uM=^n+3h&^iGRKEC>1TcOPbKvj0l?0I*IOD=v`8l`TVK~ykF#+IUW>@ zERdtT1q}-yj~qzz9>GEN;F^_I9PoVfV+d}sM@?mOX?yN4A}>;_F7Bklwe6JKWCxKS zi<3VtY~q0Y{9=3Kkx)$UE%$wPCyu(cyfg^|CwvG3djlV}9~(Z^(GGL|b%Ng9h8sl3@KqpTL)PEUTUc$OfcOacI!)! z@kd5sY|w`dy_Vagza~S-<}cV*5Xi=h1zXtqp3%FYWuP}K7*)4)>467CiNTt&eD^zlbUC6>Uo4(U^ zz9X0oUgaXnCgaJ&kZ3_--9J)X=y#mUMcI)mzIDo-nE6Nb+xkrd=!svdqp+h3t@np| z^~|{7Q;YkoVNnSQOS7}3f7@_eu_46i+Xz;h$MxGNXTmyd&)(-Qgf6Vz^`=c9zJX(%<*ju%(;kwxGv)g@;C?^Pq9agK$&bMD6BgX5jzGlX8P zd#3K>+W}XZk}KunbUaqbY!P)B#_Z~q{{DPpkci$Ny>~AcduXkJDmPhpyW@JAfd>Oe z%OzBW%xD-EN;YcnY=`BOea@D2!q-a6DjDU^gr~AvSi)2zZdHg++)L@j%D=ZCtLBh# z_|vESHAM_uO8;_=H-q3ORt`%)@fSk}sbO*Fzu@loVvni;3JzJ8uQ2SP<735+LxP)` z*yek7iy1o3Jnu$^R+imG(=L#l_6lBLmLV~!H2%-n2wYA^zEkBR z@M?>|F}~+a=%`T7(nj*JUScvWymt_`uO3V7ilpG7Z=kU$vk}hvWrJ}E-*92g^VARb znHXoDS!pnxj=ohjGVeAHpn9|UievAJFr&QkyqiuxcHJEK(QUzn;HP>W-i^JGik90G zdYOr7-&;@56Zeztlai`x*$l7dq5DOh1Ro3WO!g(_E7E*diggf^9c&MhSjM8_$4kaqOr{5zuyFJ6~pc8g=3)sq?=w{tqTFs~N_ z91W$TbYgw0hmf?bIuB9SeP?-_jUs37=Xr?UH3A7XvSh^OhcCKrAEaPxi@9Bv*tcwFw>f}s})=De%FI% zL)iK3{i#DWYy_|VN)=5Z!->Djm${}H)J&=Px1ubVopwC>uz}D&#Yan2QclBDD*5T= zoF>fiM~jVHwc(6pIA!f^CLAw~rEDD9uUkJSJzgX+fJs5bwSDK1|Lh8uP{uqSkox#%jX9w=*db z_`Q$`MsSpkbVDhuB^*=iO$a}*--wh#e3#iK4CRwbV@OWh`L#y#Gd5rKiLvIZK*z-I zn!g^c7{9Mz_r>5N^lbNkNmuFx$xp(o{mB*$YohV&sv|gHl~eYt<+ZOOPMrm_i0&4_vc`> zT0_L9M;&-q6&szJ&w|X{on2vX7!V9JOIk+mKxlXCXvQ!V-kKSb9FG>P`Nr`R|HDLA zWb|2q>Ov&`+4@XSwGKM-RsIQ=t1vq<9;Q^+jWx;i+KDbYG*0C$xZFoZ+X=Z+iJgPs zF*D4hQ1|2Wl#g8Xu3k)Ebaz~FI1>*C!bX-R*W#;{IZ2^|4Z$x7YZSujVC36-*V}`M zwz>GLhFPOfym0dZ-MRwmIgF)qcEr4ByLRx!6e0AN)4DzbI)0358mzJ>{9>E65%zyZ z@!-6>-=2&C`2R7U3_e2efXdZF2M@i+TKAw%b!+J?h2 zl@Qjt&@S)#1&bDKH&2u$_7Nz#UcYq-+@5>(D`hteYD32=*Lo0s2F2#!q*f0$C&d*- z(+S+Iy8L&YHyOVV4EHLz5kBCaJE$S@r#D5z=*T*ETn{XK6tI)n51>ldM-#qqrCY!! zA%dT(_&(eqnOuq4A~C+H?*rJgaG41CXDNCrr%zV>?m(H?PjcTZ8^Q}8g?%FU(v#XJ zMf1ddgkns0mrr3kyw5XYziUuYn`A2O)51Yp_wwtq1kN2veR<#H8VSp{1-U9sGLg?C z5iEau5ILW(2pLqcFqvq|6#QC+QO`4Tua*sAwcxGA(L-(U8-HD}`#>8GEwbr&9o2^$ z(NebpJ`p;(>#Xayfe!pi8m+pj%ZBpC?bWVMbbLEr7B8$vhkePm&EJDskjQucW~3Jd zvy5Lk0$c{p|0(3$bRPvH()>=1UX9TE3_UZJcn!)T=2;nrNj z*Lh><7_+YrJ9k7IoRJ#D)r{Y7#IpM!ax&AlPJu#P*YxBoG8M;F;sO;ZgGdwna^g!N z6Oy-u)Ey*-;mEqvGnd(otCV$g(bOIsUQpX|ZekQU?3~_8y$)m!?LPlQnDAwvv~RxY zkO9evs_Pf|S-7(3*P}1z*^o2(vsbO87~0B5g7_TU;TRqj_`|x;H{-;Z(rP^?P$KH%m{tj2^qJ;^G}Z28ovVEh!;97baem)_jt47_Ksnnsrwag@m83fqJC&MF2|!O zqcf{z+99MVWPN~~02BoWol%+;3tXLF9i!^A!`k zV-z@#3C!l-Wkcb0U`~F;D3*(LZ+*Otjxqr&r;_0zcy!fo^(1srj4IWs`@tYi++sK9@}i-12I2E)@T;=@Ffrz(wfB7+GMk*kzuaIO!8d9frgQzzQLgel-++pmq{H#zD33| z@-ziJDrHtjVjg1B!u1o^Z2(nd<$cJYfy^vQobY*z1FXC~OSiCaHt|OF{X{Z&KIv|X zxey04iNDRy3hEK|c6j)~Lk>DBo%K6hG`F~C;$9oaAFW%Ujh|7RGMJMWnGsdbgjqW&qaZ28OZFQ_mZ|ZV(*m2 YOX*%JUYr>duJIVgh`hhbHFFOBA9L_+CIA2c From 3d3ad72dcff055d6b853b499e6fc953374397af0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 29 Feb 2024 17:30:28 +0000 Subject: [PATCH 38/40] Add type annotations --- lib/galaxy/tool_util/verify/__init__.py | 51 +++++++++++++++---------- lib/galaxy/util/checkers.py | 6 +-- lib/galaxy/util/image_util.py | 24 ++++++------ 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index ea3ade7d25c7..7adfa3321904 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -15,7 +15,9 @@ Any, Callable, Dict, + List, Optional, + TYPE_CHECKING, ) import numpy @@ -37,6 +39,9 @@ from .asserts import verify_assertions from .test_data import TestDataResolver +if TYPE_CHECKING: + import numpy.typing + log = logging.getLogger(__name__) DEFAULT_TEST_DATA_RESOLVER = TestDataResolver() @@ -442,21 +447,23 @@ def files_contains(file1, file2, attributes=None): raise AssertionError(f"Failed to find '{contains}' in history data. (lines_diff={lines_diff}).") -def _multiobject_intersection_over_union(mask1, mask2, repeat_reverse=True): - iou_list = list() +def _multiobject_intersection_over_union( + mask1: "numpy.typing.NDArray", mask2: "numpy.typing.NDArray", repeat_reverse: bool = True +) -> List[numpy.floating]: + iou_list = [] for label1 in numpy.unique(mask1): cc1 = mask1 == label1 - cc1_iou_list = list() + cc1_iou_list = [] for label2 in numpy.unique(mask2[cc1]): cc2 = mask2 == label2 cc1_iou_list.append(intersection_over_union(cc1, cc2)) iou_list.append(max(cc1_iou_list)) if repeat_reverse: - iou_list += _multiobject_intersection_over_union(mask2, mask1, repeat_reverse=False) + iou_list.extend(_multiobject_intersection_over_union(mask2, mask1, repeat_reverse=False)) return iou_list -def intersection_over_union(mask1, mask2): +def intersection_over_union(mask1: "numpy.typing.NDArray", mask2: "numpy.typing.NDArray") -> numpy.floating: assert mask1.dtype == mask2.dtype assert mask1.ndim == mask2.ndim == 2 assert mask1.shape == mask2.shape @@ -466,15 +473,17 @@ def intersection_over_union(mask1, mask2): return min(_multiobject_intersection_over_union(mask1, mask2)) -def get_image_metric(attributes): +def get_image_metric( + attributes: Dict[str, Any] +) -> Callable[["numpy.typing.NDArray", "numpy.typing.NDArray"], numpy.floating]: metric_name = attributes.get("metric", DEFAULT_METRIC) - attributes = attributes or {} metrics = { - "mae": lambda im1, im2: numpy.abs(im1 - im2).mean(), - "mse": lambda im1, im2: numpy.square((im1 - im2).astype(float)).mean(), - "rms": lambda im1, im2: math.sqrt(numpy.square((im1 - im2).astype(float)).mean()), - "fro": lambda im1, im2: numpy.linalg.norm((im1 - im2).reshape(1, -1), "fro"), - "iou": lambda im1, im2: 1 - intersection_over_union(im1, im2), + "mae": lambda arr1, arr2: numpy.abs(arr1 - arr2).mean(), + # Convert to float before squaring to prevent overflows + "mse": lambda arr1, arr2: numpy.square((arr1 - arr2).astype(float)).mean(), + "rms": lambda arr1, arr2: math.sqrt(numpy.square((arr1 - arr2).astype(float)).mean()), + "fro": lambda arr1, arr2: numpy.linalg.norm((arr1 - arr2).reshape(1, -1), "fro"), + "iou": lambda arr1, arr2: 1 - intersection_over_union(arr1, arr2), } try: return metrics[metric_name] @@ -482,20 +491,22 @@ def get_image_metric(attributes): raise ValueError(f'No such metric: "{metric_name}"') -def files_image_diff(file1, file2, attributes=None): +def files_image_diff(file1: str, file2: str, attributes: Optional[Dict[str, Any]] = None) -> None: """Check the pixel data of 2 image files for differences.""" attributes = attributes or {} - im1 = numpy.array(Image.open(file1)) - im2 = numpy.array(Image.open(file2)) + with Image.open(file1) as im1: + arr1 = numpy.array(im1) + with Image.open(file2) as im2: + arr2 = numpy.array(im2) - if im1.dtype != im2.dtype: - raise AssertionError(f"Image data types did not match ({im1.dtype}, {im2.dtype}).") + if arr1.dtype != arr2.dtype: + raise AssertionError(f"Image data types did not match ({arr1.dtype}, {arr2.dtype}).") - if im1.shape != im2.shape: - raise AssertionError(f"Image dimensions did not match ({im1.shape}, {im2.shape}).") + if arr1.shape != arr2.shape: + raise AssertionError(f"Image dimensions did not match ({arr1.shape}, {arr2.shape}).") - distance = get_image_metric(attributes)(im1, im2) + distance = get_image_metric(attributes)(arr1, arr2) distance_eps = attributes.get("eps", DEFAULT_EPS) if distance > distance_eps: raise AssertionError(f"Image difference {distance} exceeds eps={distance_eps}.") diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index de2e149aa6e4..8e3862bcfe89 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -187,11 +187,9 @@ def iter_zip(file_path: str): yield (z.open(f), f) -def check_image(file_path: str): +def check_image(file_path: str) -> bool: """Simple wrapper around image_type to yield a True/False verdict""" - if image_type(file_path): - return True - return False + return bool(image_type(file_path)) COMPRESSION_CHECK_FUNCTIONS: Dict[str, CompressionChecker] = { diff --git a/lib/galaxy/util/image_util.py b/lib/galaxy/util/image_util.py index d24a75f10725..1b9bf7d99bb5 100644 --- a/lib/galaxy/util/image_util.py +++ b/lib/galaxy/util/image_util.py @@ -2,25 +2,25 @@ import imghdr import logging +from typing import ( + List, + Optional, +) try: - import Image as PIL + from PIL import Image except ImportError: - try: - from PIL import Image as PIL - except ImportError: - PIL = None + PIL = None log = logging.getLogger(__name__) -def image_type(filename): +def image_type(filename: str) -> Optional[str]: fmt = None - if PIL is not None: + if Image is not None: try: - im = PIL.open(filename) - fmt = im.format - im.close() + with Image.open(filename) as im: + fmt = im.format except Exception: # We continue to try with imghdr, so this is a rare case of an # exception we expect to happen frequently, so we're not logging @@ -30,10 +30,10 @@ def image_type(filename): if fmt: return fmt.upper() else: - return False + return None -def check_image_type(filename, types): +def check_image_type(filename: str, types: List[str]) -> bool: fmt = image_type(filename) if fmt in types: return True From 1a3edae577f13a95bcf35c15e6952fe2d6b46a52 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 29 Feb 2024 18:22:42 +0000 Subject: [PATCH 39/40] Add missing package requirements --- packages/data/setup.cfg | 1 + packages/tool_util/setup.cfg | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/data/setup.cfg b/packages/data/setup.cfg index 5e97372d47ec..157324e102a8 100644 --- a/packages/data/setup.cfg +++ b/packages/data/setup.cfg @@ -34,6 +34,7 @@ include_package_data = True install_requires = galaxy-files galaxy-objectstore + galaxy-tool-util galaxy-util[template] alembic alembic-utils diff --git a/packages/tool_util/setup.cfg b/packages/tool_util/setup.cfg index e0753f1af3f0..2aecac71fbe0 100644 --- a/packages/tool_util/setup.cfg +++ b/packages/tool_util/setup.cfg @@ -36,8 +36,10 @@ install_requires = galaxy-util>=22.1 conda-package-streaming lxml!=4.2.2 + numpy MarkupSafe packaging + pillow pydantic>=2,!=2.6.0,!=2.6.1 PyYAML requests From 62c610c250737775475a167acf1dcbc1c356fc63 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sat, 2 Mar 2024 01:58:41 +0000 Subject: [PATCH 40/40] Move numpy and pillow requirements to new extended-assertions extra together pysam, which is also conditionally imported in ``lib/galaxy/tool_util/verify/__init__.py`` . --- lib/galaxy/tool_util/verify/__init__.py | 19 ++++++++++++------- packages/test.sh | 2 +- packages/tool_util/setup.cfg | 6 ++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 7adfa3321904..976c8706b684 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -20,13 +20,18 @@ TYPE_CHECKING, ) -import numpy -from PIL import Image - +try: + import numpy +except ImportError: + pass try: import pysam except ImportError: - pysam = None # type: ignore[assignment] + pass +try: + from PIL import Image +except ImportError: + pass from galaxy.tool_util.parser.util import ( DEFAULT_DELTA, @@ -449,7 +454,7 @@ def files_contains(file1, file2, attributes=None): def _multiobject_intersection_over_union( mask1: "numpy.typing.NDArray", mask2: "numpy.typing.NDArray", repeat_reverse: bool = True -) -> List[numpy.floating]: +) -> List["numpy.floating"]: iou_list = [] for label1 in numpy.unique(mask1): cc1 = mask1 == label1 @@ -463,7 +468,7 @@ def _multiobject_intersection_over_union( return iou_list -def intersection_over_union(mask1: "numpy.typing.NDArray", mask2: "numpy.typing.NDArray") -> numpy.floating: +def intersection_over_union(mask1: "numpy.typing.NDArray", mask2: "numpy.typing.NDArray") -> "numpy.floating": assert mask1.dtype == mask2.dtype assert mask1.ndim == mask2.ndim == 2 assert mask1.shape == mask2.shape @@ -475,7 +480,7 @@ def intersection_over_union(mask1: "numpy.typing.NDArray", mask2: "numpy.typing. def get_image_metric( attributes: Dict[str, Any] -) -> Callable[["numpy.typing.NDArray", "numpy.typing.NDArray"], numpy.floating]: +) -> Callable[["numpy.typing.NDArray", "numpy.typing.NDArray"], "numpy.floating"]: metric_name = attributes.get("metric", DEFAULT_METRIC) metrics = { "mae": lambda arr1, arr2: numpy.abs(arr1 - arr2).mean(), diff --git a/packages/test.sh b/packages/test.sh index 5a64061bd51d..ae69d9b20516 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -51,7 +51,7 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow. if [ "$package_dir" = "util" ]; then pip install -e '.[template,jstree]' elif [ "$package_dir" = "tool_util" ]; then - pip install -e '.[cwl,mulled,edam]' + pip install -e '.[cwl,mulled,edam,extended-assertions]' else pip install -e '.' fi diff --git a/packages/tool_util/setup.cfg b/packages/tool_util/setup.cfg index 2aecac71fbe0..5d6b9b9a378d 100644 --- a/packages/tool_util/setup.cfg +++ b/packages/tool_util/setup.cfg @@ -36,10 +36,8 @@ install_requires = galaxy-util>=22.1 conda-package-streaming lxml!=4.2.2 - numpy MarkupSafe packaging - pillow pydantic>=2,!=2.6.0,!=2.6.1 PyYAML requests @@ -68,6 +66,10 @@ mulled = Whoosh edam = edam-ontology +extended-assertions = + numpy + pysam + pillow [options.packages.find] exclude =