From 9fd124893f0b3bc647559e5f0ffba75d826e08b0 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Mon, 26 Feb 2024 16:06:40 +0100 Subject: [PATCH] WIP update tests --- bioimageio/core/proc_ops.py | 15 +- bioimageio/core/stat_calculators.py | 34 +- bioimageio/core/stat_measures.py | 11 +- tests/build_spec/test_add_weights.py | 3 - tests/build_spec/test_build_spec.py | 11 +- tests/conftest.py | 2 +- .../test_combined_processing.py | 13 +- .../test_device_management.py | 28 +- tests/prediction_pipeline/test_measures.py | 103 ------ .../test_postprocessing.py | 70 ----- .../test_prediction_pipeline.py | 4 +- .../prediction_pipeline/test_preprocessing.py | 212 ------------- tests/prediction_pipeline/test_processing.py | 55 ---- tests/resource_io/test_load_rdf.py | 41 +-- tests/resource_io/test_utils.py | 11 +- tests/test_cli.py | 1 - tests/test_export_package.py | 60 ---- .../test_internal/test_validation_visitors.py | 39 --- tests/test_prediction.py | 8 +- tests/test_proc_ops.py | 295 ++++++++++++++++++ tests/test_resource_tests/test_test_model.py | 54 ++-- tests/test_stat_measures.py | 39 +++ .../weight_converter/keras/test_tensorflow.py | 16 +- tests/weight_converter/torch/test_onnx.py | 4 +- 24 files changed, 478 insertions(+), 651 deletions(-) delete mode 100644 tests/prediction_pipeline/test_measures.py delete mode 100644 tests/prediction_pipeline/test_postprocessing.py delete mode 100644 tests/prediction_pipeline/test_preprocessing.py delete mode 100644 tests/prediction_pipeline/test_processing.py delete mode 100644 tests/test_export_package.py delete mode 100644 tests/test_internal/test_validation_visitors.py create mode 100644 tests/test_proc_ops.py create mode 100644 tests/test_stat_measures.py diff --git a/bioimageio/core/proc_ops.py b/bioimageio/core/proc_ops.py index 535f085c..fd3ef2ee 100644 --- a/bioimageio/core/proc_ops.py +++ b/bioimageio/core/proc_ops.py @@ -3,6 +3,7 @@ from dataclasses import InitVar, dataclass, field from typing import ( Collection, + Generic, Hashable, Literal, Optional, @@ -31,10 +32,12 @@ DatasetMean, DatasetPercentile, DatasetStd, + MeanMeasure, Measure, SampleMean, SamplePercentile, SampleStd, + StdMeasure, ) from bioimageio.spec.model import v0_4, v0_5 @@ -166,7 +169,7 @@ class ScaleLinear(_SimpleOperator): offset: Union[float, xr.DataArray] = 0.0 """additive term""" - def apply(self, input: Tensor, stat: Stat) -> Tensor: + def _apply(self, input: Tensor, stat: Stat) -> Tensor: return input * self.gain + self.offset # @classmethod @@ -315,8 +318,8 @@ def from_proc_descr(cls, descr: Union[v0_4.ScaleRangeDescr, v0_5.ScaleRangeDescr return cls( input=tensor_id, output=tensor_id, - lower_percentile=Percentile(kwargs.min_percentile, axes=axes, tensor_id=ref_tensor), - upper_percentile=Percentile(kwargs.max_percentile, axes=axes, tensor_id=ref_tensor), + lower_percentile=Percentile(n=kwargs.min_percentile, axes=axes, tensor_id=ref_tensor), + upper_percentile=Percentile(n=kwargs.max_percentile, axes=axes, tensor_id=ref_tensor), ) def _apply(self, input: xr.DataArray, stat: Stat) -> xr.DataArray: @@ -363,8 +366,8 @@ def get_descr(self): class ZeroMeanUnitVariance(_SimpleOperator): """normalize to zero mean, unit variance.""" - mean: Union[SampleMean, DatasetMean] - std: Union[SampleStd, DatasetStd] + mean: MeanMeasure + std: StdMeasure eps: float = 1e-6 @@ -372,7 +375,7 @@ def __post_init__(self): assert self.mean.axes == self.std.axes @property - def required_measures(self) -> Collection[Measure]: + def required_measures(self) -> Set[Union[MeanMeasure, StdMeasure]]: return {self.mean, self.std} @classmethod diff --git a/bioimageio/core/stat_calculators.py b/bioimageio/core/stat_calculators.py index 54e601d0..e3ccdc16 100644 --- a/bioimageio/core/stat_calculators.py +++ b/bioimageio/core/stat_calculators.py @@ -437,7 +437,7 @@ def get_measure_calculators( def compute_dataset_measures( - *, measures: Iterable[DatasetMeasure], dataset: Iterable[Sample] + measures: Iterable[DatasetMeasure], dataset: Iterable[Sample] ) -> Dict[DatasetMeasure, MeasureValue]: """compute all dataset `measures` for the given `dataset`""" sample_calculators, calculators = get_measure_calculators(measures) @@ -453,3 +453,35 @@ def compute_dataset_measures( ret.update(calc.finalize().items()) return ret + +def compute_sample_measures(measures: Iterable[SampleMeasure], sample: Sample) -> Dict[SampleMeasure, MeasureValue]: + """compute all sample `measures` for the given `sample`""" + calculators, dataset_calculators = get_measure_calculators(measures) + assert not dataset_calculators + ret: Dict[SampleMeasure, MeasureValue] = {} + + for calc in calculators: + ret.update(calc.compute(sample).items()) + + return ret + + +def compute_measures(measures: Iterable[Measure], dataset: Iterable[Sample]) -> Dict[Measure, MeasureValue]: + """compute all `measures` for the given `dataset` + sample measures are computed for the last sample in `dataset`""" + sample_calculators, dataset_calculators = get_measure_calculators(measures) + ret: Dict[Measure, MeasureValue] = {} + sample = None + for sample in dataset: + for calc in dataset_calculators: + calc.update(sample) + if sample is None: + raise ValueError("empty dataset") + + for calc in dataset_calculators: + ret.update(calc.finalize().items()) + + for calc in sample_calculators: + ret.update(calc.compute(sample).items()) + + return ret diff --git a/bioimageio/core/stat_measures.py b/bioimageio/core/stat_measures.py index 4acb8f31..726c90cf 100644 --- a/bioimageio/core/stat_measures.py +++ b/bioimageio/core/stat_measures.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, TypeVar, Union import xarray as xr @@ -119,3 +119,12 @@ def __post_init__(self): SampleMeasure = Union[SampleMean, SampleStd, SampleVar, SamplePercentile] DatasetMeasure = Union[DatasetMean, DatasetStd, DatasetVar, DatasetPercentile] Measure = Union[SampleMeasure, DatasetMeasure] + +MeanMeasure = Union[SampleMean, DatasetMean] +StdMeasure = Union[SampleStd, DatasetStd] +VarMeasure = Union[SampleVar, DatasetVar] +PercentileMeasure = Union[SamplePercentile, DatasetPercentile] +MeanMeasureT = TypeVar("MeanMeasureT", bound=MeanMeasure) +StdMeasureT = TypeVar("StdMeasureT", bound=StdMeasure) +VarMeasureT = TypeVar("VarMeasureT", bound=VarMeasure) +PercentileMeasureT = TypeVar("PercentileMeasureT", bound=PercentileMeasure) diff --git a/tests/build_spec/test_add_weights.py b/tests/build_spec/test_add_weights.py index 4bba5f87..e3df4b80 100644 --- a/tests/build_spec/test_add_weights.py +++ b/tests/build_spec/test_add_weights.py @@ -1,8 +1,5 @@ import os -from bioimageio.core import export_resource_package, load_description, load_raw_resource_description -from bioimageio.core.resource_tests import test_model as _test_model - def _test_add_weights(model, tmp_path, base_weights, added_weights, **kwargs): from bioimageio.core.build_spec import add_weights diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py index 669eeb8a..b1fa85ab 100644 --- a/tests/build_spec/test_build_spec.py +++ b/tests/build_spec/test_build_spec.py @@ -1,12 +1,11 @@ from typing import Optional -from marshmallow import missing - import bioimageio.spec as spec -from bioimageio.core import load_description, load_raw_resource_description -from bioimageio.core._internal.validation_visitors import resolve_source -from bioimageio.core.resource_io import nodes -from bioimageio.core.resource_tests import test_model as _test_model + +# from bioimageio.core import load_description, load_raw_resource_description +# from bioimageio.core._internal.validation_visitors import resolve_source +# from bioimageio.core.resource_io import nodes +# from bioimageio.core.resource_tests import test_model as _test_model try: import tensorflow diff --git a/tests/conftest.py b/tests/conftest.py index 6a8366ae..0f44a94d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -132,7 +132,7 @@ @fixture(scope="session") -def model_packages(): +def model_packages() -> MappingProxyType[str, FilePath]: return MappingProxyType({name: save_bioimageio_package(MODEL_SOURCES[name]) for name in load_model_packages}) diff --git a/tests/prediction_pipeline/test_combined_processing.py b/tests/prediction_pipeline/test_combined_processing.py index 744e236c..7a590991 100644 --- a/tests/prediction_pipeline/test_combined_processing.py +++ b/tests/prediction_pipeline/test_combined_processing.py @@ -1,11 +1,12 @@ import numpy as np import xarray as xr -from bioimageio.core.resource_io import nodes +def test_postprocessing_dtype(): # TODO: remove? + from bioimageio.core.common import TensorId + from bioimageio.spec.model.v0_5 import BinarizeDescr, BinarizeKwargs, OutputTensorDescr -def test_postprocessing_dtype(): - from bioimageio.core.prediction_pipeline._combined_processing import CombinedProcessing + # from bioimageio.core.prediction_pipeline._combined_processing import CombinedProcessing shape = [3, 32, 32] axes = ("c", "y", "x") @@ -17,12 +18,12 @@ def test_postprocessing_dtype(): for dtype in ("float32", "float64", "uint8", "uint16"): outputs = [ - nodes.OutputTensor( - "out1", + OutputTensorDescr( + id=TensorId("out1"), data_type=dtype, axes=axes, shape=shape, - postprocessing=[nodes.Postprocessing("binarize", dict(threshold=threshold))], + postprocessing=[BinarizeDescr(kwargs=BinarizeKwargs(threshold=threshold))], ) ] com_proc = CombinedProcessing.from_tensor_specs(outputs) diff --git a/tests/prediction_pipeline/test_device_management.py b/tests/prediction_pipeline/test_device_management.py index d8f65eca..bda4af08 100644 --- a/tests/prediction_pipeline/test_device_management.py +++ b/tests/prediction_pipeline/test_device_management.py @@ -1,18 +1,21 @@ +from pathlib import Path + import numpy as np -import pytest import xarray as xr from numpy.testing import assert_array_almost_equal -from bioimageio.core import load_description -from bioimageio.core._internal.pytest_utils import skip_on -from bioimageio.core.resource_io.nodes import Model +from bioimageio.core.utils.testing import skip_on +from bioimageio.spec import load_description +from bioimageio.spec.model.v0_4 import ModelDescr as ModelDescr04 +from bioimageio.spec.model.v0_5 import ModelDescr, WeightsFormat +from bioimageio.spec.utils import load_array class TooFewDevicesException(Exception): pass -def _test_device_management(model_package, weight_format): +def _test_device_management(model_package: Path, weight_format: WeightsFormat): import torch if torch.cuda.device_count() == 0: @@ -21,13 +24,18 @@ def _test_device_management(model_package, weight_format): from bioimageio.core.prediction_pipeline import create_prediction_pipeline bio_model = load_description(model_package) - assert isinstance(bio_model, Model) + assert isinstance(bio_model, (ModelDescr, ModelDescr04)) pred_pipe = create_prediction_pipeline(bioimageio_model=bio_model, weight_format=weight_format, devices=["cuda:0"]) - inputs = [ - xr.DataArray(np.load(str(test_tensor)), dims=tuple(spec.axes)) - for test_tensor, spec in zip(bio_model.test_inputs, bio_model.inputs) - ] + if isinstance(bio_model, ModelDescr04): + inputs = [ + xr.DataArray(np.load(str(test_tensor)), dims=tuple(spec.axes)) + for test_tensor, spec in zip(bio_model.test_inputs, bio_model.inputs) + ] + else: + inputs = [ + xr.DataArray(load_array(ipt.test_tensor), dims=tuple(a.id for a in ipt.axes)) for ipt in bio_model.inputs + ] with pred_pipe as pp: outputs = pp.forward(*inputs) diff --git a/tests/prediction_pipeline/test_measures.py b/tests/prediction_pipeline/test_measures.py deleted file mode 100644 index 37916424..00000000 --- a/tests/prediction_pipeline/test_measures.py +++ /dev/null @@ -1,103 +0,0 @@ -import dataclasses -from itertools import product - -import numpy as np -import numpy.testing -import pytest -import xarray as xr - -from bioimageio.core import stat_measures -from bioimageio.core.prediction_pipeline._measure_groups import get_measure_groups -from bioimageio.core.prediction_pipeline._utils import PER_DATASET, PER_SAMPLE -from bioimageio.core.stat_measures import Mean, Percentile, Std, Var - - -@pytest.mark.parametrize("name_axes", product(["mean", "var", "std"], [None, ("x", "y")])) -def test_individual_normal_measure(name_axes): - name, axes = name_axes - measure = getattr(stat_measures, name.title())(axes=axes) - data = xr.DataArray(np.random.random((5, 6, 3)), dims=("x", "y", "c")) - - expected = getattr(data, name)(dim=axes) - actual = measure.compute(data) - xr.testing.assert_allclose(expected, actual) - - -@pytest.mark.parametrize("axes_n", product([None, ("x", "y")], [0, 10, 50, 100])) -def test_individual_percentile_measure(axes_n): - axes, n = axes_n - measure = stat_measures.Percentile(axes=axes, n=n) - data = xr.DataArray(np.random.random((5, 6, 3)), dims=("x", "y", "c")) - - expected = data.quantile(q=n / 100, dim=axes) - actual = measure.compute(data) - xr.testing.assert_allclose(expected, actual) - - -@pytest.mark.parametrize( - "measures_mode", - product( - [ - {"t1": {Mean()}, "t2": {Mean(), Std()}}, - {"t1": {Mean(), Var(), Std()}, "t2": {Std(axes=("x", "y"))}}, - {"t1": {Mean(axes=("x", "y"))}, "t2": {Mean(), Std(axes=("x", "y"))}}, - { - "t1": {Percentile(n=10), Percentile(n=35), Percentile(n=10, axes=("x", "y"))}, - "t2": {Percentile(n=10, axes=("x", "y")), Percentile(n=35, axes=("x", "y")), Percentile(n=10)}, - }, - ], - [PER_SAMPLE, PER_DATASET], - ), -) -def test_measure_groups(measures_mode): - measures, mode = measures_mode - - def get_sample(): - return { - "t1": xr.DataArray(np.random.random((2, 500, 600, 3)), dims=("b", "x", "y", "c")), - "t2": xr.DataArray(np.random.random((1, 500, 600)), dims=("c", "x", "y")), - } - - sample = get_sample() - dataset_seq = [sample, get_sample()] - dataset_full = {tn: xr.concat([s[tn] for s in dataset_seq], dim="dataset") for tn in sample.keys()} - - # compute independently - expected = {} - for tn, ms in measures.items(): - for m in ms: - if mode == PER_SAMPLE: - expected[(tn, m)] = m.compute(sample[tn]) - elif mode == PER_DATASET: - if m.axes is None: - m_d = m - else: - m_d = dataclasses.replace(m, axes=("dataset",) + m.axes) - - expected[(tn, m)] = m_d.compute(dataset_full[tn]) - else: - raise NotImplementedError(mode) - - groups = get_measure_groups({mode: measures})[mode] - actual = {} - for g in groups: - if mode == PER_SAMPLE: - res = g.compute(sample) - elif mode == PER_DATASET: - for s in dataset_seq: - g.update_with_sample(s) - - res = g.finalize() - else: - raise NotImplementedError(mode) - - for tn, vs in res.items(): - for m, v in vs.items(): - actual[(tn, m)] = v - - # discard additionally computed measures by groups - actual = {k: v for k, v in actual.items() if k in expected} - - for k in expected.keys(): - assert k in actual - numpy.testing.assert_array_almost_equal(expected[k].data, actual[k].data, decimal=2) diff --git a/tests/prediction_pipeline/test_postprocessing.py b/tests/prediction_pipeline/test_postprocessing.py deleted file mode 100644 index 52c3e151..00000000 --- a/tests/prediction_pipeline/test_postprocessing.py +++ /dev/null @@ -1,70 +0,0 @@ -import numpy as np -import pytest -import xarray as xr - -from bioimageio.core.prediction_pipeline._measure_groups import compute_measures - - -def test_binarize(): - from bioimageio.core.prediction_pipeline._processing import Binarize - - shape = (3, 32, 32) - axes = ("c", "y", "x") - np_data = np.random.rand(*shape) - data = xr.DataArray(np_data, dims=axes) - - threshold = 0.5 - exp = xr.DataArray(np_data > threshold, dims=axes) - - binarize = Binarize("data_name", threshold=threshold) - res = binarize(data) - xr.testing.assert_allclose(res, exp) - - -@pytest.mark.parametrize("axes", [None, tuple("cy"), tuple("cyx"), tuple("x")]) -def test_scale_mean_variance(axes): - from bioimageio.core.prediction_pipeline._processing import ScaleMeanVariance - - shape = (3, 32, 46) - ipt_axes = ("c", "y", "x") - np_data = np.random.rand(*shape) - ipt_data = xr.DataArray(np_data, dims=ipt_axes) - ref_data = xr.DataArray((np_data * 2) + 3, dims=ipt_axes) - - scale_mean_variance = ScaleMeanVariance("data_name", reference_tensor="ref_name", axes=axes) - required = scale_mean_variance.get_required_measures() - computed = compute_measures(required, sample={"data_name": ipt_data, "ref_name": ref_data}) - scale_mean_variance.set_computed_measures(computed) - - res = scale_mean_variance(ipt_data) - xr.testing.assert_allclose(res, ref_data) - - -@pytest.mark.parametrize("axes", [None, tuple("cy"), tuple("y"), tuple("yx")]) -def test_scale_mean_variance_per_channel(axes): - from bioimageio.core.prediction_pipeline._processing import ScaleMeanVariance - - shape = (3, 32, 46) - ipt_axes = ("c", "y", "x") - np_data = np.random.rand(*shape) - ipt_data = xr.DataArray(np_data, dims=ipt_axes) - - # set different mean, std per channel - np_ref_data = np.stack([d * i + i for i, d in enumerate(np_data, start=2)]) - print(np_ref_data.shape) - ref_data = xr.DataArray(np_ref_data, dims=ipt_axes) - - scale_mean_variance = ScaleMeanVariance("data_name", reference_tensor="ref_name", axes=axes) - required = scale_mean_variance.get_required_measures() - computed = compute_measures(required, sample={"data_name": ipt_data, "ref_name": ref_data}) - scale_mean_variance.set_computed_measures(computed) - - res = scale_mean_variance(ipt_data) - - if axes is not None and "c" not in axes: - # mean,std per channel should match exactly - xr.testing.assert_allclose(res, ref_data) - else: - # mean,std across channels should not match - with pytest.raises(AssertionError): - xr.testing.assert_allclose(res, ref_data) diff --git a/tests/prediction_pipeline/test_prediction_pipeline.py b/tests/prediction_pipeline/test_prediction_pipeline.py index 3a2c57aa..2c196401 100644 --- a/tests/prediction_pipeline/test_prediction_pipeline.py +++ b/tests/prediction_pipeline/test_prediction_pipeline.py @@ -2,8 +2,8 @@ import xarray as xr from numpy.testing import assert_array_almost_equal -from bioimageio.core import load_description -from bioimageio.core.resource_io.nodes import Model +# from bioimageio.core import load_description +# from bioimageio.core.resource_io.nodes import Model def _test_prediction_pipeline(model_package, weight_format): diff --git a/tests/prediction_pipeline/test_preprocessing.py b/tests/prediction_pipeline/test_preprocessing.py deleted file mode 100644 index fb8efa06..00000000 --- a/tests/prediction_pipeline/test_preprocessing.py +++ /dev/null @@ -1,212 +0,0 @@ -import numpy as np -import xarray as xr - -from bioimageio.core.prediction_pipeline._measure_groups import compute_measures -from bioimageio.core.prediction_pipeline._utils import PER_SAMPLE - - -def test_scale_linear(): - from bioimageio.core.prediction_pipeline._processing import ScaleLinear - - preprocessing = ScaleLinear("data_name", offset=[1, 2, 42], gain=[1, 2, 3], axes="yx") - data = xr.DataArray(np.arange(6).reshape((1, 2, 3)), dims=("x", "y", "c")) - expected = xr.DataArray(np.array([[[1, 4, 48], [4, 10, 57]]]), dims=("x", "y", "c")) - result = preprocessing.apply(data) - xr.testing.assert_allclose(expected, result) - - -def test_scale_linear_no_channel(): - from bioimageio.core.prediction_pipeline._processing import ScaleLinear - - preprocessing = ScaleLinear("data_name", offset=1, gain=2, axes="yx") - data = xr.DataArray(np.arange(6).reshape(2, 3), dims=("x", "y")) - expected = xr.DataArray(np.array([[1, 3, 5], [7, 9, 11]]), dims=("x", "y")) - result = preprocessing.apply(data) - xr.testing.assert_allclose(expected, result) - - -def test_zero_mean_unit_variance_preprocessing(): - from bioimageio.core.prediction_pipeline._processing import ZeroMeanUnitVariance - - data = xr.DataArray(np.arange(9).reshape(3, 3), dims=("x", "y")) - - preprocessing = ZeroMeanUnitVariance("data_name", mode=PER_SAMPLE) - required = preprocessing.get_required_measures() - computed = compute_measures(required, sample={"data_name": data}) - preprocessing.set_computed_measures(computed) - - expected = xr.DataArray( - np.array( - [ - [-1.54919274, -1.16189455, -0.77459637], - [-0.38729818, 0.0, 0.38729818], - [0.77459637, 1.16189455, 1.54919274], - ] - ), - dims=("x", "y"), - ) - result = preprocessing(data) - xr.testing.assert_allclose(expected, result) - - -def test_zero_mean_unit_variance_preprocessing_fixed(): - from bioimageio.core.prediction_pipeline._processing import ZeroMeanUnitVariance - - preprocessing = ZeroMeanUnitVariance( - "data_name", mode="fixed", axes=["y"], mean=[1, 4, 7], std=[0.81650, 0.81650, 0.81650] - ) - data = xr.DataArray(np.arange(9).reshape((1, 1, 3, 3)), dims=("b", "c", "x", "y")) - expected = xr.DataArray( - np.array([[-1.224743, 0.0, 1.224743], [-1.224743, 0.0, 1.224743], [-1.224743, 0.0, 1.224743]])[None, None], - dims=("b", "c", "x", "y"), - ) - result = preprocessing(data) - xr.testing.assert_allclose(expected, result) - - -def test_zero_mean_unit_across_axes(): - from bioimageio.core.prediction_pipeline._processing import ZeroMeanUnitVariance - - data = xr.DataArray(np.arange(18).reshape((2, 3, 3)), dims=("c", "x", "y")) - - axes = ("x", "y") - preprocessing = ZeroMeanUnitVariance("data_name", axes=axes, mode=PER_SAMPLE) - required = preprocessing.get_required_measures() - computed = compute_measures(required, sample={"data_name": data}) - preprocessing.set_computed_measures(computed) - - expected = xr.DataArray( - np.array( - [ - [-1.54919274, -1.16189455, -0.77459637], - [-0.38729818, 0.0, 0.38729818], - [0.77459637, 1.16189455, 1.54919274], - ] - ), - dims=("x", "y"), - ) - result = preprocessing(data) - xr.testing.assert_allclose(expected, result[dict(c=0)]) - - -def test_zero_mean_unit_variance_fixed(): - from bioimageio.core.prediction_pipeline._processing import ZeroMeanUnitVariance - - np_data = np.arange(9).reshape(3, 3) - mean = np_data.mean() - std = np_data.mean() - eps = 1.0e-7 - preprocessing = ZeroMeanUnitVariance("data_name", mode="fixed", mean=mean, std=std, eps=eps) - - data = xr.DataArray(np_data, dims=("x", "y")) - expected = xr.DataArray((np_data - mean) / (std + eps), dims=("x", "y")) - result = preprocessing(data) - xr.testing.assert_allclose(expected, result) - - -def test_binarize(): - from bioimageio.core.prediction_pipeline._processing import Binarize - - preprocessing = Binarize("data_name", threshold=14) - data = xr.DataArray(np.arange(30).reshape((2, 3, 5)), dims=("x", "y", "c")) - expected = xr.zeros_like(data) - expected[{"x": slice(1, None)}] = 1 - result = preprocessing(data) - xr.testing.assert_allclose(expected, result) - - -def test_clip_preprocessing(): - from bioimageio.core.prediction_pipeline._processing import Clip - - preprocessing = Clip("data_name", min=3, max=5) - data = xr.DataArray(np.arange(9).reshape(3, 3), dims=("x", "y")) - expected = xr.DataArray(np.array([[3, 3, 3], [3, 4, 5], [5, 5, 5]]), dims=("x", "y")) - result = preprocessing(data) - xr.testing.assert_equal(expected, result) - - -def test_combination_of_preprocessing_steps_with_dims_specified(): - from bioimageio.core.prediction_pipeline._processing import ZeroMeanUnitVariance - - data = xr.DataArray(np.arange(18).reshape((2, 3, 3)), dims=("c", "x", "y")) - axes = ("x", "y") - preprocessing = ZeroMeanUnitVariance("data_name", axes=axes, mode=PER_SAMPLE) - required = preprocessing.get_required_measures() - computed = compute_measures(required, sample={"data_name": data}) - preprocessing.set_computed_measures(computed) - - expected = xr.DataArray( - np.array( - [ - [-1.54919274, -1.16189455, -0.77459637], - [-0.38729818, 0.0, 0.38729818], - [0.77459637, 1.16189455, 1.54919274], - ] - ), - dims=("x", "y"), - ) - - result = preprocessing(data) - xr.testing.assert_allclose(expected, result[dict(c=0)]) - - -def test_scale_range(): - from bioimageio.core.prediction_pipeline._processing import ScaleRange - - preprocessing = ScaleRange("data_name") - np_data = np.arange(9).reshape(3, 3).astype("float32") - data = xr.DataArray(np_data, dims=("x", "y")) - required = preprocessing.get_required_measures() - computed = compute_measures(required, sample={"data_name": data}) - preprocessing.set_computed_measures(computed) - - eps = 1.0e-6 - mi, ma = np_data.min(), np_data.max() - exp_data = (np_data - mi) / (ma - mi + eps) - expected = xr.DataArray(exp_data, dims=("x", "y")) - - result = preprocessing(data) - # NOTE xarray.testing.assert_allclose compares irrelavant properties here and fails although the result is correct - np.testing.assert_allclose(expected, result) - - -def test_scale_range_axes(): - from bioimageio.core.prediction_pipeline._processing import ScaleRange - - min_percentile = 1.0 - max_percentile = 99.0 - preprocessing = ScaleRange( - "data_name", axes=("x", "y"), min_percentile=min_percentile, max_percentile=max_percentile - ) - - np_data = np.arange(18).reshape((2, 3, 3)).astype("float32") - data = xr.DataArray(np_data, dims=("c", "x", "y")) - - required = preprocessing.get_required_measures() - computed = compute_measures(required, sample={"data_name": data}) - preprocessing.set_computed_measures(computed) - - eps = 1.0e-6 - p_low = np.percentile(np_data, min_percentile, axis=(1, 2), keepdims=True) - p_up = np.percentile(np_data, max_percentile, axis=(1, 2), keepdims=True) - exp_data = (np_data - p_low) / (p_up - p_low + eps) - expected = xr.DataArray(exp_data, dims=("c", "x", "y")) - - result = preprocessing(data) - # NOTE xarray.testing.assert_allclose compares irrelavant properties here and fails although the result is correct - np.testing.assert_allclose(expected, result) - - -def test_sigmoid(): - from bioimageio.core.prediction_pipeline._processing import Sigmoid - - shape = (3, 32, 32) - axes = ("c", "y", "x") - np_data = np.random.rand(*shape) - data = xr.DataArray(np_data, dims=axes) - - sigmoid = Sigmoid("data_name") - res = sigmoid(data) - - exp = xr.DataArray(1.0 / (1 + np.exp(-np_data)), dims=axes) - xr.testing.assert_allclose(res, exp) diff --git a/tests/prediction_pipeline/test_processing.py b/tests/prediction_pipeline/test_processing.py deleted file mode 100644 index 819982a2..00000000 --- a/tests/prediction_pipeline/test_processing.py +++ /dev/null @@ -1,55 +0,0 @@ -import dataclasses - -import numpy as np -import pytest -import xarray as xr - -from bioimageio.core.prediction_pipeline._processing import IMPLEMENTED_PROCESSING -from bioimageio.core.prediction_pipeline._utils import FIXED - -try: - from typing import get_args -except ImportError: - from typing_extensions import get_args # type: ignore - - -def test_assert_dtype(): - from bioimageio.core.prediction_pipeline._processing import AssertDtype - - proc = AssertDtype("test_tensor", dtype="uint8") - tensor = xr.DataArray(np.zeros((1,), dtype="uint8"), dims=tuple("c")) - out = proc(tensor) - assert out is tensor - - tensor = tensor.astype("uint16") - with pytest.raises(AssertionError): - out = proc(tensor) - assert out is tensor - - -@pytest.mark.parametrize( - "proc", - list(IMPLEMENTED_PROCESSING["pre"].values()) + list(IMPLEMENTED_PROCESSING["post"].values()), -) -def test_no_req_measures_for_mode_fixed(proc): - # check if mode=fixed is valid for this proc - for f in dataclasses.fields(proc): - if f.name == "mode": - break - else: - raise AttributeError("Processing is missing mode attribute") - # mode is always annotated as literals (or literals of literals) - valid_modes = get_args(f.type) - for inner in get_args(f.type): - valid_modes += get_args(inner) - - if FIXED not in valid_modes: - return - - # we might be missing required kwargs. These have marshmallow.missing value as default - # and raise a TypeError is in __post_init__() - proc.__post_init__ = lambda self: None # ignore missing kwargs - - proc_instance = proc(tensor_name="tensor_name", mode=FIXED) - req_measures = proc_instance.get_required_measures() - assert not req_measures diff --git a/tests/resource_io/test_load_rdf.py b/tests/resource_io/test_load_rdf.py index 873d9b26..0d86d1a2 100644 --- a/tests/resource_io/test_load_rdf.py +++ b/tests/resource_io/test_load_rdf.py @@ -4,43 +4,18 @@ import pytest -from bioimageio.core._internal.validation_visitors import resolve_source +def test_load_model_with_abs_path_source(unet2d_nuclei_broad_model: Path): + from bioimageio.spec import load_description -def test_load_non_existing_rdf(): - from bioimageio.core import load_description - - spec_path = Path("some/none/existing/path/to/spec.model.yaml") - - with pytest.raises(FileNotFoundError): - load_description(spec_path) - - -def test_load_raw_model(any_model): - from bioimageio.core import load_raw_resource_description - - raw_model = load_raw_resource_description(any_model) - assert raw_model - - -def test_load_model(any_model): - from bioimageio.core import load_description - - model = load_description(any_model) - assert model - - -def test_load_model_with_abs_path_source(unet2d_nuclei_broad_model): - from bioimageio.core.resource_io import load_description, load_raw_resource_description - - raw_rd = load_raw_resource_description(unet2d_nuclei_broad_model) - path_source = (raw_rd.root_path / "rdf.yaml").absolute() + raw_rd = load_description(unet2d_nuclei_broad_model) + path_source = (raw_rd.root / "rdf.yaml").absolute() assert path_source.is_absolute() model = load_description(path_source) assert model -def test_load_model_with_rel_path_source(unet2d_nuclei_broad_model): +def test_load_model_with_rel_path_source(unet2d_nuclei_broad_model: Path): from bioimageio.core.resource_io import load_description, load_raw_resource_description raw_rd = load_raw_resource_description(unet2d_nuclei_broad_model) @@ -50,7 +25,7 @@ def test_load_model_with_rel_path_source(unet2d_nuclei_broad_model): assert model -def test_load_model_with_abs_str_source(unet2d_nuclei_broad_model): +def test_load_model_with_abs_str_source(unet2d_nuclei_broad_model: Path): from bioimageio.core.resource_io import load_description, load_raw_resource_description raw_rd = load_raw_resource_description(unet2d_nuclei_broad_model) @@ -60,7 +35,7 @@ def test_load_model_with_abs_str_source(unet2d_nuclei_broad_model): assert model -def test_load_model_with_rel_str_source(unet2d_nuclei_broad_model): +def test_load_model_with_rel_str_source(unet2d_nuclei_broad_model: Path): from bioimageio.core.resource_io import load_description, load_raw_resource_description raw_rd = load_raw_resource_description(unet2d_nuclei_broad_model) @@ -70,7 +45,7 @@ def test_load_model_with_rel_str_source(unet2d_nuclei_broad_model): assert model -def test_load_remote_rdf(unet2d_nuclei_broad_model): +def test_load_remote_rdf(unet2d_nuclei_broad_model: Path): # remote model is a pytorch model, needing unet2d_nuclei_broad_model skips the test when needed _ = unet2d_nuclei_broad_model from bioimageio.core import load_description diff --git a/tests/resource_io/test_utils.py b/tests/resource_io/test_utils.py index d1e570cc..ff834edc 100644 --- a/tests/resource_io/test_utils.py +++ b/tests/resource_io/test_utils.py @@ -2,12 +2,13 @@ from pathlib import Path import pytest -from bioimageio.spec.shared import raw_nodes -from bioimageio.spec.shared.raw_nodes import RawNode -from bioimageio.core._internal import validation_visitors -from bioimageio.core._internal.validation_visitors import Sha256NodeChecker -from bioimageio.core.resource_io import nodes +# from bioimageio.spec.shared import raw_nodes +# from bioimageio.spec.shared.raw_nodes import RawNode + +# from bioimageio.core._internal import validation_visitors +# from bioimageio.core._internal.validation_visitors import Sha256NodeChecker +# from bioimageio.core.resource_io import nodes def test_resolve_import_path(tmpdir): diff --git a/tests/test_cli.py b/tests/test_cli.py index 473a1b1a..2ed9f894 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,7 +3,6 @@ from typing import Sequence import numpy as np -import pytest from bioimageio.core import load_description diff --git a/tests/test_export_package.py b/tests/test_export_package.py deleted file mode 100644 index a04aa994..00000000 --- a/tests/test_export_package.py +++ /dev/null @@ -1,60 +0,0 @@ -import shutil -from pathlib import Path -from tempfile import TemporaryDirectory -from zipfile import ZipFile - -from marshmallow import missing - -from bioimageio.spec.model import raw_nodes - - -def test_export_package(any_onnx_model): - from bioimageio.core import export_resource_package, load_raw_resource_description - - package_path = export_resource_package(any_onnx_model, weights_priority_order=["onnx"]) - assert isinstance(package_path, Path), package_path - assert package_path.exists(), package_path - - raw_model = load_raw_resource_description(package_path) - assert isinstance(raw_model, raw_nodes.Model) - - -def test_package_with_folder(unet2d_nuclei_broad_model): - from bioimageio.core import export_resource_package, load_raw_resource_description - - with TemporaryDirectory() as tmp_dir: - tmp_dir = Path(tmp_dir) - - # extract package (to not cache to BIOIMAGEIO_CACHE) - package_folder = tmp_dir / "package" - with ZipFile(unet2d_nuclei_broad_model) as zf: - zf.extractall(package_folder) - - # load package - model = load_raw_resource_description(package_folder / "rdf.yaml") - assert isinstance(model, raw_nodes.Model) - - # alter package to have its documentation in a nested folder - doc = model.documentation - assert doc is not missing - doc = doc.relative_to(model.root_path) - assert not doc.is_absolute() - new_doc = Path("nested") / "folder" / doc - (package_folder / new_doc).parent.mkdir(parents=True) - shutil.move(package_folder / doc, package_folder / new_doc) - model.documentation = new_doc - - # export altered package - altered_package = tmp_dir / "altered_package.zip" - altered_package = export_resource_package(model, output_path=altered_package, weights_priority_order=["onnx"]) - - # extract altered package (to not cache to BIOIMAGEIO_CACHE) - altered_package_folder = tmp_dir / "altered_package" - with ZipFile(altered_package) as zf: - zf.extractall(altered_package_folder) - - # load altered package - reloaded_model = load_raw_resource_description(altered_package_folder / "rdf.yaml") - assert isinstance(reloaded_model, raw_nodes.Model) - assert reloaded_model.documentation.as_posix().endswith(new_doc.as_posix()) - assert reloaded_model.documentation.exists() diff --git a/tests/test_internal/test_validation_visitors.py b/tests/test_internal/test_validation_visitors.py deleted file mode 100644 index 7988f658..00000000 --- a/tests/test_internal/test_validation_visitors.py +++ /dev/null @@ -1,39 +0,0 @@ -from functools import singledispatchmethod - -from bioimageio.core._internal.validation_visitors import Note, ValidationVisitor -from bioimageio.spec._internal.base_nodes import Node -from bioimageio.spec.summary import ErrorEntry - - -def test_traversing_nodes(): - class MyVisitor(ValidationVisitor): - @singledispatchmethod - def visit(self, obj: type, note: Note = Note()): - super().visit(obj, note) - - @visit.register - def _visit_int(self, nr: int, note: Note = Note()): - super().visit(nr, note) - self.errors.append(ErrorEntry(loc=note.loc, msg=f"nr: {nr}", type="got-int")) - - class NestedNode(Node): - leaf: int - - class MyNode(Node): - nested: NestedNode - - tree = { - "a": MyNode(nested=NestedNode(leaf=1)), - "b": [NestedNode(leaf=2), NestedNode(leaf=3)], - "c": (NestedNode(leaf=4),), - "d": {"deep": MyNode(nested=NestedNode(leaf=5))}, - } - visitor = MyVisitor() - visitor.visit(tree) - assert len(visitor.errors) == [ - ErrorEntry(loc=("a", "nested", "leaf"), msg="nr: 1", type="got-int"), - ErrorEntry(loc=("b", 0, "leaf"), msg="nr: 2", type="got-int"), - ErrorEntry(loc=("b", 1, "leaf"), msg="nr: 3", type="got-int"), - ErrorEntry(loc=("c", 0, "leaf"), msg="nr: 4", type="got-int"), - ErrorEntry(loc=("d", "deep", "nested", "leaf"), msg="nr: 5", type="got-int"), - ] diff --git a/tests/test_prediction.py b/tests/test_prediction.py index b73d0f82..a0e34b08 100644 --- a/tests/test_prediction.py +++ b/tests/test_prediction.py @@ -4,15 +4,15 @@ import numpy as np from numpy.testing import assert_array_almost_equal -from bioimageio.core import load_description -from bioimageio.core.resource_io.nodes import Model +from bioimageio.spec import load_description +from bioimageio.spec.model.v0_5 import ModelDescr -def test_predict_image(any_model, tmpdir): +def test_predict_image(any_model: Path, tmpdir: Path): from bioimageio.core.prediction import predict_image spec = load_description(any_model) - assert isinstance(spec, Model) + assert isinstance(spec, ModelDescr) inputs = spec.test_inputs outputs = [Path(tmpdir) / f"out{i}.npy" for i in range(len(spec.test_outputs))] diff --git a/tests/test_proc_ops.py b/tests/test_proc_ops.py new file mode 100644 index 00000000..f029517e --- /dev/null +++ b/tests/test_proc_ops.py @@ -0,0 +1,295 @@ +from typing import Iterable, Optional, Tuple, Type, TypeVar + +import numpy as np +import pytest +import xarray as xr +from typing_extensions import TypeGuard + +from bioimageio.core.common import AxisId, Sample, TensorId +from bioimageio.core.stat_calculators import compute_measures +from bioimageio.core.stat_measures import SampleMean, SamplePercentile, SampleStd + + +@pytest.fixture(scope="module") +def tid(): + return TensorId("data123") + + +def test_scale_linear(tid: TensorId): + from bioimageio.core.proc_ops import ScaleLinear + + offset = xr.DataArray([1, 2, 42], dims=("c")) + gain = xr.DataArray([1, 2, 3], dims=("c")) + data = xr.DataArray(np.arange(6).reshape((1, 2, 3)), dims=("x", "y", "c")) + sample = Sample(data={tid: data}) + + op = ScaleLinear(input=tid, output=tid, offset=offset, gain=gain) + op(sample) + + expected = xr.DataArray(np.array([[[1, 4, 48], [4, 10, 57]]]), dims=("x", "y", "c")) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +def test_scale_linear_no_channel(tid: TensorId): + from bioimageio.core.proc_ops import ScaleLinear + + op = ScaleLinear(tid, tid, offset=1, gain=2) + data = xr.DataArray(np.arange(6).reshape(2, 3), dims=("x", "y")) + sample = Sample(data={tid: data}) + op(sample) + + expected = xr.DataArray(np.array([[1, 3, 5], [7, 9, 11]]), dims=("x", "y")) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +T = TypeVar("T") + + +def is_iterable(val: Iterable[T], inner: Type[T]) -> TypeGuard[Iterable[T]]: + """Determines whether all objects in the list are strings""" + return all(isinstance(x, inner) for x in val) + + +def test_zero_mean_unit_variance(tid: TensorId): + from bioimageio.core.proc_ops import ZeroMeanUnitVariance + + data = xr.DataArray(np.arange(9).reshape(3, 3), dims=("x", "y")) + sample = Sample(data={tid: data}) + m = SampleMean(tid) + std = SampleStd(tid) + op = ZeroMeanUnitVariance(tid, tid, m, std) + req = op.required_measures + sample.stat = compute_measures(req, [sample]) + op(sample) + + expected = xr.DataArray( + np.array( + [ + [-1.54919274, -1.16189455, -0.77459637], + [-0.38729818, 0.0, 0.38729818], + [0.77459637, 1.16189455, 1.54919274], + ] + ), + dims=("x", "y"), + ) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +def test_zero_mean_unit_variance_fixed(tid: TensorId): + from bioimageio.core.proc_ops import FixedZeroMeanUnitVariance + + op = FixedZeroMeanUnitVariance( + tid, tid, mean=xr.DataArray([1, 4, 7], dims=("y")), std=xr.DataArray([0.81650, 0.81650, 0.81650], dims=("y")) + ) + data = xr.DataArray(np.arange(9).reshape((1, 1, 3, 3)), dims=("b", "c", "x", "y")) + expected = xr.DataArray( + np.array([[-1.224743, 0.0, 1.224743], [-1.224743, 0.0, 1.224743], [-1.224743, 0.0, 1.224743]])[None, None], + dims=("b", "c", "x", "y"), + ) + sample = Sample(data={tid: data}) + op(sample) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +def test_zero_mean_unit_across_axes(tid: TensorId): + from bioimageio.core.proc_ops import ZeroMeanUnitVariance + + data = xr.DataArray(np.arange(18).reshape((2, 3, 3)), dims=("c", "x", "y")) + + op = ZeroMeanUnitVariance(tid, tid, SampleMean(tid, (AxisId("c"),)), SampleStd(tid, (AxisId("c"),))) + sample = Sample(data={tid: data}) + sample.stat = compute_measures(op.required_measures, [sample]) + + expected = xr.DataArray( + np.array( + [ + [-1.54919274, -1.16189455, -0.77459637], + [-0.38729818, 0.0, 0.38729818], + [0.77459637, 1.16189455, 1.54919274], + ] + ), + dims=("x", "y"), + ) + op(sample) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +def test_zero_mean_unit_variance_fixed2(tid: TensorId): + from bioimageio.core.proc_ops import FixedZeroMeanUnitVariance + + np_data = np.arange(9).reshape(3, 3) + mean = float(np_data.mean()) + std = float(np_data.mean()) + eps = 1.0e-7 + op = FixedZeroMeanUnitVariance(tid, tid, mean=mean, std=std, eps=eps) + + data = xr.DataArray(np_data, dims=("x", "y")) + sample = Sample(data={tid: data}) + expected = xr.DataArray((np_data - mean) / (std + eps), dims=("x", "y")) + op(sample) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +def test_binarize(tid: TensorId): + from bioimageio.core.proc_ops import Binarize + + op = Binarize(tid, tid, threshold=14) + data = xr.DataArray(np.arange(30).reshape((2, 3, 5)), dims=("x", "y", "c")) + sample = Sample(data={tid: data}) + expected = xr.zeros_like(data) + expected[{"x": slice(1, None)}] = 1 + op(sample) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +def test_binarize2(tid: TensorId): + from bioimageio.core.proc_ops import Binarize + + shape = (3, 32, 32) + axes = ("c", "y", "x") + np_data = np.random.rand(*shape) + data = xr.DataArray(np_data, dims=axes) + + threshold = 0.5 + exp = xr.DataArray(np_data > threshold, dims=axes) + + sample = Sample(data={tid: data}) + binarize = Binarize(tid, tid, threshold=threshold) + binarize(sample) + xr.testing.assert_allclose(exp, sample.data[tid]) + + +def test_clip(tid: TensorId): + from bioimageio.core.proc_ops import Clip + + op = Clip(tid, tid, min=3, max=5) + data = xr.DataArray(np.arange(9).reshape(3, 3), dims=("x", "y")) + sample = Sample(data={tid: data}) + + expected = xr.DataArray(np.array([[3, 3, 3], [3, 4, 5], [5, 5, 5]]), dims=("x", "y")) + op(sample) + xr.testing.assert_equal(expected, sample.data[tid]) + + +def test_combination_of_op_steps_with_dims_specified(tid: TensorId): + from bioimageio.core.proc_ops import ZeroMeanUnitVariance + + data = xr.DataArray(np.arange(18).reshape((2, 3, 3)), dims=("c", "x", "y")) + sample = Sample(data={tid: data}) + op = ZeroMeanUnitVariance(tid, tid, SampleMean(tid, (AxisId("c"),)), SampleStd(tid, (AxisId("c"),))) + sample.stat = compute_measures(op.required_measures, [sample]) + + expected = xr.DataArray( + np.array( + [ + [-1.54919274, -1.16189455, -0.77459637], + [-0.38729818, 0.0, 0.38729818], + [0.77459637, 1.16189455, 1.54919274], + ] + ), + dims=("x", "y"), + ) + + op(sample) + xr.testing.assert_allclose(expected, sample.data[tid]) + + +@pytest.mark.parametrize("axes", [None, tuple(map(AxisId, "cy")), tuple(map(AxisId, "cyx")), tuple(map(AxisId, "x"))]) +def test_scale_mean_variance(tid: TensorId, axes: Optional[Tuple[AxisId, ...]]): + from bioimageio.core.proc_ops import ScaleMeanVariance + + shape = (3, 32, 46) + ipt_axes = ("c", "y", "x") + np_data = np.random.rand(*shape) + ipt_data = xr.DataArray(np_data, dims=ipt_axes) + ref_data = xr.DataArray((np_data * 2) + 3, dims=ipt_axes) + + op = ScaleMeanVariance(tid, tid, reference_tensor=TensorId("ref_name"), axes=axes) + sample = Sample(data={tid: ipt_data, TensorId("ref_name"): ref_data}) + sample.stat = compute_measures(op.required_measures, [sample]) + op(sample) + xr.testing.assert_allclose(ref_data, sample.data[tid]) + + +@pytest.mark.parametrize("axes", [None, tuple(map(AxisId, "cy")), tuple(map(AxisId, "y")), tuple(map(AxisId, "yx"))]) +def test_scale_mean_variance_per_channel(tid: TensorId, axes: Optional[Tuple[AxisId, ...]]): + from bioimageio.core.proc_ops import ScaleMeanVariance + + shape = (3, 32, 46) + ipt_axes = ("c", "y", "x") + np_data = np.random.rand(*shape) + ipt_data = xr.DataArray(np_data, dims=ipt_axes) + + # set different mean, std per channel + np_ref_data = np.stack([d * i + i for i, d in enumerate(np_data, start=2)]) + ref_data = xr.DataArray(np_ref_data, dims=ipt_axes) + + op = ScaleMeanVariance(tid, tid, reference_tensor=TensorId("ref_name"), axes=axes) + sample = Sample(data={tid: ipt_data, TensorId("ref_name"): ref_data}) + sample.stat = compute_measures(op.required_measures, [sample]) + op(sample) + + if axes is not None and AxisId("c") not in axes: + # mean,std per channel should match exactly + xr.testing.assert_allclose(ref_data, sample.data[tid]) + else: + # mean,std across channels should not match + with pytest.raises(AssertionError): + xr.testing.assert_allclose(ref_data, sample.data[tid]) + + +def test_scale_range(tid: TensorId): + from bioimageio.core.proc_ops import ScaleRange + + op = ScaleRange(tid, tid) + np_data = np.arange(9).reshape(3, 3).astype("float32") + data = xr.DataArray(np_data, dims=("x", "y")) + sample = Sample(data={tid: data}) + sample.stat = compute_measures(op.required_measures, [sample]) + + eps = 1.0e-6 + mi, ma = np_data.min(), np_data.max() + exp_data = (np_data - mi) / (ma - mi + eps) + expected = xr.DataArray(exp_data, dims=("x", "y")) + + op(sample) + # NOTE xarray.testing.assert_allclose compares irrelavant properties here and fails although the result is correct + np.testing.assert_allclose(expected, sample.data[tid]) + + +def test_scale_range_axes(tid: TensorId): + from bioimageio.core.proc_ops import ScaleRange + + lower_percentile = SamplePercentile(tid, 1, axes=(AxisId("c"),)) + upper_percentile = SamplePercentile(tid, 100, axes=(AxisId("c"),)) + op = ScaleRange(tid, tid, lower_percentile, upper_percentile) + + np_data = np.arange(18).reshape((2, 3, 3)).astype("float32") + data = xr.DataArray(np_data, dims=("c", "x", "y")) + sample = Sample(data={tid: data}) + sample.stat = compute_measures(op.required_measures, [sample]) + + eps = 1.0e-6 + p_low = np.percentile(np_data, lower_percentile.n, axis=(1, 2), keepdims=True) + p_up = np.percentile(np_data, upper_percentile.n, axis=(1, 2), keepdims=True) + exp_data = (np_data - p_low) / (p_up - p_low + eps) + expected = xr.DataArray(exp_data, dims=("c", "x", "y")) + + op(sample) + # NOTE xarray.testing.assert_allclose compares irrelavant properties here and fails although the result is correct + np.testing.assert_allclose(expected, sample.data[tid]) + + +def test_sigmoid(tid: TensorId): + from bioimageio.core.proc_ops import Sigmoid + + shape = (3, 32, 32) + axes = ("c", "y", "x") + np_data = np.random.rand(*shape) + data = xr.DataArray(np_data, dims=axes) + sample = Sample(data={tid: data}) + sigmoid = Sigmoid(tid, tid) + sigmoid(sample) + + exp = xr.DataArray(1.0 / (1 + np.exp(-np_data)), dims=axes) + xr.testing.assert_allclose(exp, sample.data[tid]) diff --git a/tests/test_resource_tests/test_test_model.py b/tests/test_resource_tests/test_test_model.py index 9498e8ab..f83baf52 100644 --- a/tests/test_resource_tests/test_test_model.py +++ b/tests/test_resource_tests/test_test_model.py @@ -1,74 +1,72 @@ -import pathlib +from pathlib import Path -import pytest +from bioimageio.spec import InvalidDescr -def test_error_for_wrong_shape(stardist_wrong_shape): +def test_error_for_wrong_shape(stardist_wrong_shape: Path): from bioimageio.core.resource_tests import test_model - summary = test_model(stardist_wrong_shape)[-1] + summary = test_model(stardist_wrong_shape) expected_error_message = ( "Shape (1, 512, 512, 33) of test output 0 'output' does not match output shape description: " "ImplicitOutputShape(reference_tensor='input', " "scale=[1.0, 1.0, 1.0, 0.0], offset=[1.0, 1.0, 1.0, 33.0])." ) - assert summary["error"] == expected_error_message + assert summary.details[0].errors[0].msg == expected_error_message -def test_error_for_wrong_shape2(stardist_wrong_shape2): +def test_error_for_wrong_shape2(stardist_wrong_shape2: Path): from bioimageio.core.resource_tests import test_model - summary = test_model(stardist_wrong_shape2)[-1] + summary = test_model(stardist_wrong_shape2) expected_error_message = ( "Shape (1, 512, 512, 1) of test input 0 'input' does not match input shape description: " "ParametrizedInputShape(min=[1, 80, 80, 1], step=[0, 17, 17, 0])." ) - assert summary["error"] == expected_error_message + assert summary.details[0].errors[0].msg == expected_error_message -def test_test_model(any_model): +def test_test_model(any_model: Path): from bioimageio.core.resource_tests import test_model summary = test_model(any_model) - assert all([s["status"] for s in summary]) + assert summary.status == "passed" -def test_test_resource(any_model): +def test_test_resource(any_model: Path): from bioimageio.core.resource_tests import test_description summary = test_description(any_model) - assert all([s["status"] for s in summary]) + assert summary.status == "passed" -def test_validation_section_warning(unet2d_nuclei_broad_model, tmp_path: pathlib.Path): +def test_validation_section_warning(unet2d_nuclei_broad_model: str, tmp_path: Path): from bioimageio.core import load_description from bioimageio.core.resource_tests import test_description model = load_description(unet2d_nuclei_broad_model) - - summary = test_description(model)[2] - assert summary["name"] == "Test documentation completeness." - assert summary["warnings"] == {"documentation": "No '# Validation' (sub)section found."} - assert summary["status"] == "passed" + assert not isinstance(model, InvalidDescr) + summary = test_description(model) + assert summary.name == "Test documentation completeness." + assert summary.warnings == {"documentation": "No '# Validation' (sub)section found."} + assert summary.status == "passed" doc_with_validation = tmp_path / "doc.md" - doc_with_validation.write_text("# Validation\nThis is a section about how to validate the model on new data") + _ = doc_with_validation.write_text("# Validation\nThis is a section about how to validate the model on new data") model.documentation = doc_with_validation - summary = test_description(model)[2] - assert summary["name"] == "Test documentation completeness." - assert summary["warnings"] == {} - assert summary["status"] == "passed" + summary = test_description(model) + assert summary.name == "Test documentation completeness." + assert summary.warnings == {} + assert summary.status == "passed" -def test_issue289(unet2d_nuclei_broad_model): +def test_issue289(unet2d_nuclei_broad_model: str): """test for failure case from https://github.com/bioimage-io/core-bioimage-io-python/issues/289""" # remote model is a pytorch model, needing unet2d_nuclei_broad_model skips the test when needed _ = unet2d_nuclei_broad_model - import bioimageio.core from bioimageio.core.resource_tests import test_model doi = "10.5281/zenodo.6287342" - model_resource = bioimageio.core.load_description(doi) - test_result = test_model(model_resource) - assert all([t["status"] == "passed" for t in test_result]) + summary = test_model(doi) + assert summary.status == "passed" diff --git a/tests/test_stat_measures.py b/tests/test_stat_measures.py new file mode 100644 index 00000000..7845da89 --- /dev/null +++ b/tests/test_stat_measures.py @@ -0,0 +1,39 @@ +from itertools import product +from typing import Optional, Tuple + +import numpy as np +import pytest +import xarray as xr + +from bioimageio.core import stat_measures +from bioimageio.core.common import AxisId, Sample, TensorId +from bioimageio.core.stat_calculators import SamplePercentilesCalculator, get_measure_calculators +from bioimageio.core.stat_measures import SamplePercentile + + +@pytest.mark.parametrize("name, axes", product(["mean", "var", "std"], [None, (AxisId("x"), AxisId("y"))])) +def test_individual_normal_measure(name: str, axes: Optional[Tuple[AxisId, AxisId]]): + measure = getattr(stat_measures, name.title() + "Measure")(axes=axes) + data = xr.DataArray(np.random.random((5, 6, 3)), dims=("x", "y", "c")) + + expected = getattr(data, name)(dim=axes) + actual = measure.compute(data) + xr.testing.assert_allclose(expected, actual) + + +@pytest.mark.parametrize("axes", [None, (AxisId("x"), AxisId("y"))]) +def test_individual_percentile_measure(axes: Optional[Tuple[AxisId, ...]]): + ns = [0, 10, 50, 100] + tid = TensorId("tensor") + + measures = [SamplePercentile(tensor_id=tid, axes=axes, n=n) for n in ns] + calcs, _ = get_measure_calculators(measures) + assert len(calcs) == 1 + calc = calcs[0] + assert isinstance(calc, SamplePercentilesCalculator) + + data = xr.DataArray(np.random.random((5, 6, 3)), dims=("x", "y", "c")) + actual = calc.compute(Sample(data={tid: data})) + for m in measures: + expected = data.quantile(q=m.n / 100, dim=m.axes) + xr.testing.assert_allclose(expected, actual[m]) diff --git a/tests/weight_converter/keras/test_tensorflow.py b/tests/weight_converter/keras/test_tensorflow.py index 712263fa..5cc7f297 100644 --- a/tests/weight_converter/keras/test_tensorflow.py +++ b/tests/weight_converter/keras/test_tensorflow.py @@ -1,22 +1,30 @@ import zipfile +from pathlib import Path +from bioimageio.spec import load_description +from bioimageio.spec.model.v0_5 import ModelDescr -def test_tensorflow_converter(any_keras_model, tmp_path): + +def test_tensorflow_converter(any_keras_model: Path, tmp_path: Path): from bioimageio.core.weight_converter.keras import convert_weights_to_tensorflow_saved_model_bundle out_path = tmp_path / "weights" - ret_val = convert_weights_to_tensorflow_saved_model_bundle(any_keras_model, out_path) + model = load_description(any_keras_model) + assert isinstance(model, ModelDescr), model.validation_summary.format() + ret_val = convert_weights_to_tensorflow_saved_model_bundle(model, out_path) assert out_path.exists() assert (out_path / "variables").exists() assert (out_path / "saved_model.pb").exists() assert ret_val == 0 # check for correctness is done in converter and returns 0 if it passes -def test_tensorflow_converter_zipped(any_keras_model, tmp_path): +def test_tensorflow_converter_zipped(any_keras_model: Path, tmp_path: Path): from bioimageio.core.weight_converter.keras import convert_weights_to_tensorflow_saved_model_bundle out_path = tmp_path / "weights.zip" - ret_val = convert_weights_to_tensorflow_saved_model_bundle(any_keras_model, out_path) + model = load_description(any_keras_model) + assert isinstance(model, ModelDescr), model.validation_summary.format() + ret_val = convert_weights_to_tensorflow_saved_model_bundle(model, out_path) assert out_path.exists() assert ret_val == 0 # check for correctness is done in converter and returns 0 if it passes diff --git a/tests/weight_converter/torch/test_onnx.py b/tests/weight_converter/torch/test_onnx.py index 5a26c916..bc757806 100644 --- a/tests/weight_converter/torch/test_onnx.py +++ b/tests/weight_converter/torch/test_onnx.py @@ -1,9 +1,11 @@ import os +from pathlib import Path + import pytest # todo: test with 'any_torch_model' -def test_onnx_converter(convert_to_onnx, tmp_path): +def test_onnx_converter(convert_to_onnx: Path, tmp_path, Path): from bioimageio.core.weight_converter.torch.onnx import convert_weights_to_onnx out_path = tmp_path / "weights.onnx"