diff --git a/poetry.lock b/poetry.lock index 4cf95c8a2..7f3d00e81 100644 --- a/poetry.lock +++ b/poetry.lock @@ -273,7 +273,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.770" +version = "0.800" description = "Optional static typing for Python" category = "dev" optional = false @@ -683,7 +683,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "6b9e47f9a57731afa7d3035cb0cd7ea6fa837f3b3b12e31a619f994fc9e1a6d2" +content-hash = "da7dd403252286fde6ae2765a5c9477fcb88068465af1a0b65b94dd3ccd624e6" [metadata.files] appdirs = [ @@ -912,20 +912,28 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] mypy = [ - {file = "mypy-0.770-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:a34b577cdf6313bf24755f7a0e3f3c326d5c1f4fe7422d1d06498eb25ad0c600"}, - {file = "mypy-0.770-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:86c857510a9b7c3104cf4cde1568f4921762c8f9842e987bc03ed4f160925754"}, - {file = "mypy-0.770-cp35-cp35m-win_amd64.whl", hash = "sha256:a8ffcd53cb5dfc131850851cc09f1c44689c2812d0beb954d8138d4f5fc17f65"}, - {file = "mypy-0.770-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:7687f6455ec3ed7649d1ae574136835a4272b65b3ddcf01ab8704ac65616c5ce"}, - {file = "mypy-0.770-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3beff56b453b6ef94ecb2996bea101a08f1f8a9771d3cbf4988a61e4d9973761"}, - {file = "mypy-0.770-cp36-cp36m-win_amd64.whl", hash = "sha256:15b948e1302682e3682f11f50208b726a246ab4e6c1b39f9264a8796bb416aa2"}, - {file = "mypy-0.770-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:b90928f2d9eb2f33162405f32dde9f6dcead63a0971ca8a1b50eb4ca3e35ceb8"}, - {file = "mypy-0.770-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:c56ffe22faa2e51054c5f7a3bc70a370939c2ed4de308c690e7949230c995913"}, - {file = "mypy-0.770-cp37-cp37m-win_amd64.whl", hash = "sha256:8dfb69fbf9f3aeed18afffb15e319ca7f8da9642336348ddd6cab2713ddcf8f9"}, - {file = "mypy-0.770-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:219a3116ecd015f8dca7b5d2c366c973509dfb9a8fc97ef044a36e3da66144a1"}, - {file = "mypy-0.770-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7ec45a70d40ede1ec7ad7f95b3c94c9cf4c186a32f6bacb1795b60abd2f9ef27"}, - {file = "mypy-0.770-cp38-cp38-win_amd64.whl", hash = "sha256:f91c7ae919bbc3f96cd5e5b2e786b2b108343d1d7972ea130f7de27fdd547cf3"}, - {file = "mypy-0.770-py3-none-any.whl", hash = "sha256:3b1fc683fb204c6b4403a1ef23f0b1fac8e4477091585e0c8c54cbdf7d7bb164"}, - {file = "mypy-0.770.tar.gz", hash = "sha256:8a627507ef9b307b46a1fea9513d5c98680ba09591253082b4c48697ba05a4ae"}, + {file = "mypy-0.800-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:e1c84c65ff6d69fb42958ece5b1255394714e0aac4df5ffe151bc4fe19c7600a"}, + {file = "mypy-0.800-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:947126195bfe4709c360e89b40114c6746ae248f04d379dca6f6ab677aa07641"}, + {file = "mypy-0.800-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:b95068a3ce3b50332c40e31a955653be245666a4bc7819d3c8898aa9fb9ea496"}, + {file = "mypy-0.800-cp35-cp35m-win_amd64.whl", hash = "sha256:ca7ad5aed210841f1e77f5f2f7d725b62c78fa77519312042c719ed2ab937876"}, + {file = "mypy-0.800-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e32b7b282c4ed4e378bba8b8dfa08e1cfa6f6574067ef22f86bee5b1039de0c9"}, + {file = "mypy-0.800-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e497a544391f733eca922fdcb326d19e894789cd4ff61d48b4b195776476c5cf"}, + {file = "mypy-0.800-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:5615785d3e2f4f03ab7697983d82c4b98af5c321614f51b8f1034eb9ebe48363"}, + {file = "mypy-0.800-cp36-cp36m-win_amd64.whl", hash = "sha256:2b216eacca0ec0ee124af9429bfd858d5619a0725ee5f88057e6e076f9eb1a7b"}, + {file = "mypy-0.800-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e3b8432f8df19e3c11235c4563a7250666dc9aa7cdda58d21b4177b20256ca9f"}, + {file = "mypy-0.800-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d16c54b0dffb861dc6318a8730952265876d90c5101085a4bc56913e8521ba19"}, + {file = "mypy-0.800-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:0d2fc8beb99cd88f2d7e20d69131353053fbecea17904ee6f0348759302c52fa"}, + {file = "mypy-0.800-cp37-cp37m-win_amd64.whl", hash = "sha256:aa9d4901f3ee1a986a3a79fe079ffbf7f999478c281376f48faa31daaa814e86"}, + {file = "mypy-0.800-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:319ee5c248a7c3f94477f92a729b7ab06bf8a6d04447ef3aa8c9ba2aa47c6dcf"}, + {file = "mypy-0.800-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:74f5aa50d0866bc6fb8e213441c41e466c86678c800700b87b012ed11c0a13e0"}, + {file = "mypy-0.800-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:a301da58d566aca05f8f449403c710c50a9860782148332322decf73a603280b"}, + {file = "mypy-0.800-cp38-cp38-win_amd64.whl", hash = "sha256:b9150db14a48a8fa114189bfe49baccdff89da8c6639c2717750c7ae62316738"}, + {file = "mypy-0.800-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5fdf935a46aa20aa937f2478480ebf4be9186e98e49cc3843af9a5795a49a25"}, + {file = "mypy-0.800-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6f8425fecd2ba6007e526209bb985ce7f49ed0d2ac1cc1a44f243380a06a84fb"}, + {file = "mypy-0.800-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:5ff616787122774f510caeb7b980542a7cc2222be3f00837a304ea85cd56e488"}, + {file = "mypy-0.800-cp39-cp39-win_amd64.whl", hash = "sha256:90b6f46dc2181d74f80617deca611925d7e63007cf416397358aa42efb593e07"}, + {file = "mypy-0.800-py3-none-any.whl", hash = "sha256:3e0c159a7853e3521e3f582adb1f3eac66d0b0639d434278e2867af3a8c62653"}, + {file = "mypy-0.800.tar.gz", hash = "sha256:e0202e37756ed09daf4b0ba64ad2c245d357659e014c3f51d8cd0681ba66940a"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, diff --git a/pyproject.toml b/pyproject.toml index 5f0749d20..8d261cb54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ pylint = "^2.6.0" black = "^20.8b1" pytest = "^6.2.1" setuptools-scm = "^3.3.3" -mypy = "^0.770" +mypy = "^0.800" [build-system] requires = ["poetry>=0.12"] diff --git a/tests/test_bounding_box.py b/tests/test_bounding_box.py index 33c9cd31c..559a54162 100644 --- a/tests/test_bounding_box.py +++ b/tests/test_bounding_box.py @@ -2,7 +2,7 @@ import pytest -def test_align_with_mag_ceiled(): +def test_align_with_mag_ceiled() -> None: assert BoundingBox((1, 1, 1), (10, 10, 10)).align_with_mag( Mag(2), ceil=True @@ -18,7 +18,7 @@ def test_align_with_mag_ceiled(): ) == BoundingBox(topleft=(0, 2, 2), size=(10, 10, 10)) -def test_align_with_mag_floored(): +def test_align_with_mag_floored() -> None: assert BoundingBox((1, 1, 1), (10, 10, 10)).align_with_mag(Mag(2)) == BoundingBox( topleft=(2, 2, 2), size=(8, 8, 8) @@ -34,7 +34,7 @@ def test_align_with_mag_floored(): ) -def test_in_mag(): +def test_in_mag() -> None: with pytest.raises(AssertionError): BoundingBox((1, 2, 3), (9, 9, 9)).in_mag(Mag(2)) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 827a2ce3d..8088aa701 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -2,6 +2,8 @@ import json import os from os.path import dirname, join +from typing import Tuple, cast + import pytest import numpy as np @@ -9,12 +11,19 @@ from wkw.wkw import WKWException -from wkcuber.api.Dataset import WKDataset, TiffDataset, TiledTiffDataset +from wkcuber.api.Dataset import ( + WKDataset, + TiffDataset, + TiledTiffDataset, + AbstractDataset, +) from os import path, makedirs from wkcuber.api.Layer import Layer +from wkcuber.api.MagDataset import TiledTiffMagDataset from wkcuber.api.Properties.DatasetProperties import TiffProperties, WKProperties from wkcuber.api.TiffData.TiffMag import TiffReader +from wkcuber.api.View import View from wkcuber.api.bounding_box import BoundingBox from wkcuber.compress import compress_mag_inplace from wkcuber.mag import Mag @@ -23,12 +32,12 @@ expected_error_msg = "The test did not throw an exception even though it should. " -def delete_dir(relative_path): +def delete_dir(relative_path: str) -> None: if path.exists(relative_path) and path.isdir(relative_path): rmtree(relative_path) -def chunk_job(args): +def chunk_job(args: Tuple[View, tuple]) -> None: view, additional_args = args # increment the color value of each voxel @@ -39,7 +48,7 @@ def chunk_job(args): view.write(data) -def advanced_chunk_job(args): +def advanced_chunk_job(args: Tuple[View, tuple]) -> None: view, additional_args = args # write different data for each chunk (depending on the global_offset of the chunk) @@ -48,7 +57,7 @@ def advanced_chunk_job(args): view.write(data) -def for_each_chunking_with_wrong_chunk_size(view): +def for_each_chunking_with_wrong_chunk_size(view: View) -> None: with get_executor_for_args(None) as executor: try: view.for_each_chunk( @@ -86,7 +95,7 @@ def for_each_chunking_with_wrong_chunk_size(view): pass -def for_each_chunking_advanced(ds, view): +def for_each_chunking_advanced(ds: AbstractDataset, view: View) -> None: chunk_size = (64, 64, 64) with get_executor_for_args(None) as executor: view.for_each_chunk( @@ -116,7 +125,7 @@ def for_each_chunking_advanced(ds, view): ) -def get_multichanneled_data(dtype): +def get_multichanneled_data(dtype: type) -> np.ndarray: data = np.zeros((3, 250, 200, 10), dtype=dtype) for h in range(10): for i in range(250): @@ -127,7 +136,7 @@ def get_multichanneled_data(dtype): return data -def test_create_wk_dataset_with_layer_and_mag(): +def test_create_wk_dataset_with_layer_and_mag() -> None: delete_dir("./testoutput/wk_dataset") ds = WKDataset.create("./testoutput/wk_dataset", scale=(1, 1, 1)) @@ -143,7 +152,7 @@ def test_create_wk_dataset_with_layer_and_mag(): assert len(ds.properties.data_layers["color"].wkw_magnifications) == 2 -def test_create_wk_dataset_with_explicit_header_fields(): +def test_create_wk_dataset_with_explicit_header_fields() -> None: delete_dir("./testoutput/wk_dataset_advanced") ds = WKDataset.create("./testoutput/wk_dataset_advanced", scale=(1, 1, 1)) @@ -169,7 +178,7 @@ def test_create_wk_dataset_with_explicit_header_fields(): assert ds.properties.data_layers["color"].wkw_magnifications[1].mag == Mag("2-2-1") -def test_create_tiff_dataset_with_layer_and_mag(): +def test_create_tiff_dataset_with_layer_and_mag() -> None: # This test would be the same for WKDataset delete_dir("./testoutput/tiff_dataset") @@ -187,21 +196,21 @@ def test_create_tiff_dataset_with_layer_and_mag(): assert len(ds.properties.data_layers["color"].wkw_magnifications) == 2 -def test_open_wk_dataset(): +def test_open_wk_dataset() -> None: ds = WKDataset("./testdata/simple_wk_dataset") assert len(ds.properties.data_layers) == 1 assert len(ds.properties.data_layers["color"].wkw_magnifications) == 1 -def test_open_tiff_dataset(): +def test_open_tiff_dataset() -> None: ds = TiffDataset("./testdata/simple_tiff_dataset") assert len(ds.properties.data_layers) == 1 assert len(ds.properties.data_layers["color"].wkw_magnifications) == 1 -def test_view_read_with_open(): +def test_view_read_with_open() -> None: wk_view = WKDataset("./testdata/simple_wk_dataset/").get_view( "color", "1", size=(16, 16, 16) @@ -218,7 +227,7 @@ def test_view_read_with_open(): assert not wk_view._is_opened -def test_tiff_mag_read_with_open(): +def test_tiff_mag_read_with_open() -> None: tiff_dataset = TiffDataset("./testdata/simple_tiff_dataset/") layer = tiff_dataset.get_layer("color") @@ -228,7 +237,7 @@ def test_tiff_mag_read_with_open(): assert data.shape == (1, 10, 10, 10) # single channel -def test_view_read_without_open(): +def test_view_read_without_open() -> None: # This test would be the same for TiffDataset wk_view = WKDataset("./testdata/simple_wk_dataset/").get_view( @@ -244,7 +253,7 @@ def test_view_read_without_open(): assert not wk_view._is_opened -def test_view_wk_write(): +def test_view_wk_write() -> None: delete_dir("./testoutput/simple_wk_dataset/") copytree("./testdata/simple_wk_dataset/", "./testoutput/simple_wk_dataset/") @@ -262,7 +271,7 @@ def test_view_wk_write(): assert np.array_equal(data, write_data) -def test_view_tiff_write(): +def test_view_tiff_write() -> None: delete_dir("./testoutput/simple_tiff_dataset/") copytree("./testdata/simple_tiff_dataset/", "./testoutput/simple_tiff_dataset/") @@ -281,7 +290,7 @@ def test_view_tiff_write(): assert np.array_equal(data, np.expand_dims(write_data, 0)) -def test_view_tiff_write_out_of_bounds(): +def test_view_tiff_write_out_of_bounds() -> None: new_dataset_path = "./testoutput/tiff_view_dataset_out_of_bounds/" delete_dir(new_dataset_path) @@ -303,7 +312,7 @@ def test_view_tiff_write_out_of_bounds(): pass -def test_view_wk_write_out_of_bounds(): +def test_view_wk_write_out_of_bounds() -> None: new_dataset_path = "./testoutput/wk_view_dataset_out_of_bounds/" delete_dir(new_dataset_path) @@ -323,7 +332,7 @@ def test_view_wk_write_out_of_bounds(): pass -def test_wk_view_out_of_bounds(): +def test_wk_view_out_of_bounds() -> None: try: # The size of the mag is (24, 24, 24). Trying to get an bigger view should throw an error WKDataset("./testdata/simple_wk_dataset/").get_view( @@ -336,7 +345,7 @@ def test_wk_view_out_of_bounds(): pass -def test_tiff_view_out_of_bounds(): +def test_tiff_view_out_of_bounds() -> None: try: # The size of the mag is (24, 24, 24). Trying to get an bigger view should throw an error TiffDataset("./testdata/simple_tiff_dataset/").get_view( @@ -349,7 +358,7 @@ def test_tiff_view_out_of_bounds(): pass -def test_tiff_write_out_of_bounds(): +def test_tiff_write_out_of_bounds() -> None: new_dataset_path = "./testoutput/simple_tiff_dataset_out_of_bounds/" delete_dir(new_dataset_path) @@ -365,7 +374,7 @@ def test_tiff_write_out_of_bounds(): assert ds.properties.data_layers["color"].get_bounding_box_size() == (300, 300, 15) -def test_wk_write_out_of_bounds(): +def test_wk_write_out_of_bounds() -> None: new_dataset_path = "./testoutput/simple_wk_dataset_out_of_bounds/" delete_dir(new_dataset_path) @@ -381,7 +390,7 @@ def test_wk_write_out_of_bounds(): assert ds.properties.data_layers["color"].get_bounding_box_size() == (24, 24, 48) -def test_wk_write_out_of_bounds_mag2(): +def test_wk_write_out_of_bounds_mag2() -> None: new_dataset_path = "./testoutput/simple_wk_dataset_out_of_bounds/" delete_dir(new_dataset_path) @@ -399,7 +408,7 @@ def test_wk_write_out_of_bounds_mag2(): assert ds.properties.data_layers["color"].get_bounding_box_size() == (120, 24, 58) -def test_update_new_bounding_box_offset(): +def test_update_new_bounding_box_offset() -> None: # This test would be the same for WKDataset delete_dir("./testoutput/tiff_dataset") @@ -422,7 +431,7 @@ def test_update_new_bounding_box_offset(): assert ds.properties.data_layers["color"].bounding_box["topLeft"] == (5, 5, 10) -def test_other_file_extensions_for_tiff_dataset(): +def test_other_file_extensions_for_tiff_dataset() -> None: # The TiffDataset also works with other file extensions (in this case .png) # It also works with .jpg but this format uses lossy compression @@ -439,7 +448,7 @@ def test_other_file_extensions_for_tiff_dataset(): assert np.array_equal(mag.read(size=(10, 10, 10)), np.expand_dims(write_data, 0)) -def test_tiff_write_multi_channel_uint8(): +def test_tiff_write_multi_channel_uint8() -> None: dataset_path = "./testoutput/tiff_multichannel/" delete_dir(dataset_path) @@ -454,7 +463,7 @@ def test_tiff_write_multi_channel_uint8(): assert np.array_equal(data, mag.read(size=(250, 200, 10))) -def test_wk_write_multi_channel_uint8(): +def test_wk_write_multi_channel_uint8() -> None: dataset_path = "./testoutput/wk_multichannel/" delete_dir(dataset_path) @@ -469,7 +478,7 @@ def test_wk_write_multi_channel_uint8(): assert np.array_equal(data, mag.read(size=(250, 200, 10))) -def test_tiff_write_multi_channel_uint16(): +def test_tiff_write_multi_channel_uint16() -> None: dataset_path = "./testoutput/tiff_multichannel/" delete_dir(dataset_path) @@ -489,7 +498,7 @@ def test_tiff_write_multi_channel_uint16(): assert np.array_equal(data, written_data) -def test_wk_write_multi_channel_uint16(): +def test_wk_write_multi_channel_uint16() -> None: dataset_path = "./testoutput/wk_multichannel/" delete_dir(dataset_path) @@ -507,7 +516,7 @@ def test_wk_write_multi_channel_uint16(): assert np.array_equal(data, written_data) -def test_wkw_empty_read(): +def test_wkw_empty_read() -> None: filename = "./testoutput/empty_wk_dataset" delete_dir(filename) @@ -521,7 +530,7 @@ def test_wkw_empty_read(): assert data.shape == (1, 0, 0, 0) -def test_tiff_empty_read(): +def test_tiff_empty_read() -> None: filename = "./testoutput/empty_tiff_dataset" delete_dir(filename) @@ -535,7 +544,7 @@ def test_tiff_empty_read(): assert data.shape == (1, 0, 0, 0) -def test_tiff_read_padded_data(): +def test_tiff_read_padded_data() -> None: filename = "./testoutput/empty_tiff_dataset" delete_dir(filename) @@ -551,7 +560,7 @@ def test_tiff_read_padded_data(): assert np.array_equal(data, np.zeros((3, 10, 10, 10))) -def test_wk_read_padded_data(): +def test_wk_read_padded_data() -> None: filename = "./testoutput/empty_wk_dataset" delete_dir(filename) @@ -567,7 +576,7 @@ def test_wk_read_padded_data(): assert np.array_equal(data, np.zeros((3, 10, 10, 10))) -def test_read_and_write_of_properties(): +def test_read_and_write_of_properties() -> None: destination_path = "./testoutput/read_write_properties/" delete_dir(destination_path) source_file_name = "./testdata/simple_tiff_dataset/datasource-properties.json" @@ -581,7 +590,7 @@ def test_read_and_write_of_properties(): filecmp.cmp(source_file_name, destination_file_name) -def test_num_channel_mismatch_assertion(): +def test_num_channel_mismatch_assertion() -> None: delete_dir("./testoutput/wk_dataset") ds = WKDataset.create("./testoutput/wk_dataset", scale=(1, 1, 1)) @@ -601,7 +610,7 @@ def test_num_channel_mismatch_assertion(): pass -def test_get_or_add_layer(): +def test_get_or_add_layer() -> None: # This test would be the same for TiffDataset delete_dir("./testoutput/wk_dataset") @@ -637,7 +646,7 @@ def test_get_or_add_layer(): pass -def test_get_or_add_mag_for_wk(): +def test_get_or_add_mag_for_wk() -> None: delete_dir("./testoutput/wk_dataset") layer = WKDataset.create("./testoutput/wk_dataset", scale=(1, 1, 1)).add_layer( @@ -667,7 +676,7 @@ def test_get_or_add_mag_for_wk(): pass -def test_get_or_add_mag_for_tiff(): +def test_get_or_add_mag_for_tiff() -> None: delete_dir("./testoutput/wk_dataset") layer = TiffDataset.create("./testoutput/wk_dataset", scale=(1, 1, 1)).add_layer( @@ -687,7 +696,7 @@ def test_get_or_add_mag_for_tiff(): assert mag.name == "1" -def test_tiled_tiff_read_and_write_multichannel(): +def test_tiled_tiff_read_and_write_multichannel() -> None: delete_dir("./testoutput/TiledTiffDataset") tiled_tiff_ds = TiledTiffDataset.create( "./testoutput/TiledTiffDataset", @@ -708,7 +717,7 @@ def test_tiled_tiff_read_and_write_multichannel(): assert np.array_equal(data, written_data) -def test_tiled_tiff_read_and_write(): +def test_tiled_tiff_read_and_write() -> None: delete_dir("./testoutput/tiled_tiff_dataset") tiled_tiff_ds = TiledTiffDataset.create( "./testoutput/tiled_tiff_dataset", @@ -742,7 +751,7 @@ def test_tiled_tiff_read_and_write(): ) -def test_open_dataset_without_num_channels_in_properties(): +def test_open_dataset_without_num_channels_in_properties() -> None: delete_dir("./testoutput/old_wk_dataset/") copytree("./testdata/old_wk_dataset/", "./testoutput/old_wk_dataset/") @@ -763,7 +772,7 @@ def test_open_dataset_without_num_channels_in_properties(): assert data["dataLayers"][0].get("num_channels") == 1 -def test_advanced_pattern(): +def test_advanced_pattern() -> None: delete_dir("./testoutput/tiff_dataset_advanced_pattern") ds = TiledTiffDataset.create( "./testoutput/tiff_dataset_advanced_pattern", @@ -778,7 +787,7 @@ def test_advanced_pattern(): assert np.array_equal(mag.read(size=(10, 10, 10)), np.expand_dims(data, 0)) -def test_invalid_pattern(): +def test_invalid_pattern() -> None: delete_dir("./testoutput/tiff_invalid_dataset") try: @@ -808,7 +817,7 @@ def test_invalid_pattern(): pass -def test_largest_segment_id_requirement(): +def test_largest_segment_id_requirement() -> None: path = "./testoutput/largest_segment_id" delete_dir(path) ds = WKDataset.create(path, scale=(10, 10, 10)) @@ -828,7 +837,7 @@ def test_largest_segment_id_requirement(): ) -def test_properties_with_segmentation(): +def test_properties_with_segmentation() -> None: input_json_path = "./testdata/complex_property_ds/datasource-properties.json" output_json_path = "./testoutput/complex_property_ds/datasource-properties.json" properties = WKProperties._from_json(input_json_path) @@ -862,7 +871,7 @@ def test_properties_with_segmentation(): assert input_data == output_data -def test_chunking_wk(): +def test_chunking_wk() -> None: delete_dir("./testoutput/chunking_dataset_wk/") copytree("./testdata/simple_wk_dataset/", "./testoutput/chunking_dataset_wk/") @@ -883,7 +892,7 @@ def test_chunking_wk(): assert np.array_equal(original_data + 50, view.read(size=view.size)) -def test_chunking_wk_advanced(): +def test_chunking_wk_advanced() -> None: delete_dir("./testoutput/chunking_dataset_wk_advanced/") copytree( "./testdata/simple_wk_dataset/", "./testoutput/chunking_dataset_wk_advanced/" @@ -896,7 +905,7 @@ def test_chunking_wk_advanced(): for_each_chunking_advanced(ds, view) -def test_chunking_wk_wrong_chunk_size(): +def test_chunking_wk_wrong_chunk_size() -> None: delete_dir("./testoutput/chunking_dataset_wk_with_wrong_chunk_size/") copytree( "./testdata/simple_wk_dataset/", @@ -910,7 +919,7 @@ def test_chunking_wk_wrong_chunk_size(): for_each_chunking_with_wrong_chunk_size(view) -def test_chunking_tiff(): +def test_chunking_tiff() -> None: delete_dir("./testoutput/chunking_dataset_tiff/") copytree("./testdata/simple_tiff_dataset/", "./testoutput/chunking_dataset_tiff/") @@ -932,7 +941,7 @@ def test_chunking_tiff(): assert np.array_equal(original_data + 50, new_data) -def test_chunking_tiff_wrong_chunk_size(): +def test_chunking_tiff_wrong_chunk_size() -> None: delete_dir("./testoutput/chunking_dataset_tiff_with_wrong_chunk_size/") copytree( "./testdata/simple_tiff_dataset/", @@ -946,7 +955,7 @@ def test_chunking_tiff_wrong_chunk_size(): for_each_chunking_with_wrong_chunk_size(view) -def test_chunking_tiled_tiff_wrong_chunk_size(): +def test_chunking_tiled_tiff_wrong_chunk_size() -> None: delete_dir("./testoutput/chunking_dataset_tiled_tiff_with_wrong_chunk_size/") ds = TiledTiffDataset.create( @@ -961,7 +970,7 @@ def test_chunking_tiled_tiff_wrong_chunk_size(): for_each_chunking_with_wrong_chunk_size(view) -def test_chunking_tiled_tiff_advanced(): +def test_chunking_tiled_tiff_advanced() -> None: delete_dir("./testoutput/chunking_dataset_tiled_tiff_advanced/") copytree( "./testdata/simple_wk_dataset/", @@ -976,7 +985,7 @@ def test_chunking_tiled_tiff_advanced(): for_each_chunking_advanced(ds, view) -def test_tiled_tiff_inverse_pattern(): +def test_tiled_tiff_inverse_pattern() -> None: delete_dir("./testoutput/tiled_tiff_dataset_inverse") tiled_tiff_ds = TiledTiffDataset.create( "./testoutput/tiled_tiff_dataset_inverse", @@ -985,7 +994,10 @@ def test_tiled_tiff_inverse_pattern(): pattern="{zzz}/{xxx}/{yyy}.tif", ) - mag = tiled_tiff_ds.add_layer("color", Layer.COLOR_TYPE).add_mag("1") + mag = cast( + TiledTiffMagDataset, + tiled_tiff_ds.add_layer("color", Layer.COLOR_TYPE).add_mag("1"), + ) data = np.zeros((250, 200, 10), dtype=np.uint8) for h in range(10): @@ -1014,7 +1026,7 @@ def test_tiled_tiff_inverse_pattern(): ) -def test_view_write_without_open(): +def test_view_write_without_open() -> None: # This test would be the same for TiffDataset delete_dir("./testoutput/wk_dataset_write_without_open") @@ -1034,15 +1046,17 @@ def test_view_write_without_open(): assert not wk_view._is_opened -def test_typing_of_get_mag(): +def test_typing_of_get_mag() -> None: ds = WKDataset("./testdata/simple_wk_dataset") layer = ds.get_layer("color") assert layer.get_mag("1") == layer.get_mag(1) assert layer.get_mag("1") == layer.get_mag((1, 1, 1)) + assert layer.get_mag("1") == layer.get_mag([1, 1, 1]) + assert layer.get_mag("1") == layer.get_mag(np.array([1, 1, 1])) assert layer.get_mag("1") == layer.get_mag(Mag(1)) -def test_wk_dataset_get_or_create(): +def test_wk_dataset_get_or_create() -> None: delete_dir("./testoutput/wk_dataset_get_or_create") # dataset does not exists yet @@ -1069,7 +1083,7 @@ def test_wk_dataset_get_or_create(): pass -def test_tiff_dataset_get_or_create(): +def test_tiff_dataset_get_or_create() -> None: delete_dir("./testoutput/tiff_dataset_get_or_create") # dataset does not exists yet @@ -1107,7 +1121,7 @@ def test_tiff_dataset_get_or_create(): pass -def test_tiled_tiff_dataset_get_or_create(): +def test_tiled_tiff_dataset_get_or_create() -> None: delete_dir("./testoutput/tiled_tiff_dataset_get_or_create") # dataset does not exists yet @@ -1163,7 +1177,7 @@ def test_tiled_tiff_dataset_get_or_create(): pass -def test_changing_layer_bounding_box(): +def test_changing_layer_bounding_box() -> None: delete_dir("./testoutput/test_changing_layer_bounding_box/") copytree( "./testdata/simple_tiff_dataset/", @@ -1212,7 +1226,7 @@ def test_changing_layer_bounding_box(): assert np.array_equal(original_data[:, 10:, 10:, :], new_data) -def test_view_offsets(): +def test_view_offsets() -> None: delete_dir("./testoutput/wk_offset_tests") ds = WKDataset.create("./testoutput/wk_offset_tests", scale=(1, 1, 1)) @@ -1284,7 +1298,7 @@ def test_view_offsets(): pass -def test_adding_layer_with_invalid_dtype_per_layer(): +def test_adding_layer_with_invalid_dtype_per_layer() -> None: delete_dir("./testoutput/invalid_dtype") ds = WKDataset.create("./testoutput/invalid_dtype", scale=(1, 1, 1)) @@ -1301,7 +1315,7 @@ def test_adding_layer_with_invalid_dtype_per_layer(): ) # "int"/"int64" works with 4 channels -def test_adding_layer_with_valid_dtype_per_layer(): +def test_adding_layer_with_valid_dtype_per_layer() -> None: delete_dir("./testoutput/valid_dtype") ds = WKDataset.create("./testoutput/valid_dtype", scale=(1, 1, 1)) @@ -1311,7 +1325,7 @@ def test_adding_layer_with_valid_dtype_per_layer(): ds.add_layer("color4", Layer.COLOR_TYPE, dtype_per_channel="uint8", num_channels=3) -def test_writing_subset_of_compressed_data_multi_channel(): +def test_writing_subset_of_compressed_data_multi_channel() -> None: delete_dir("./testoutput/compressed_data/") # create uncompressed dataset @@ -1350,7 +1364,7 @@ def test_writing_subset_of_compressed_data_multi_channel(): ) # the old data is still there -def test_writing_subset_of_compressed_data_single_channel(): +def test_writing_subset_of_compressed_data_single_channel() -> None: delete_dir("./testoutput/compressed_data/") # create uncompressed dataset @@ -1387,7 +1401,7 @@ def test_writing_subset_of_compressed_data_single_channel(): ) # the old data is still there -def test_writing_subset_of_compressed_data(): +def test_writing_subset_of_compressed_data() -> None: delete_dir("./testoutput/compressed_data/") # create uncompressed dataset @@ -1417,7 +1431,7 @@ def test_writing_subset_of_compressed_data(): ) -def test_writing_subset_of_chunked_compressed_data(): +def test_writing_subset_of_chunked_compressed_data() -> None: delete_dir("./testoutput/compressed_data/") # create uncompressed dataset @@ -1463,7 +1477,7 @@ def test_writing_subset_of_chunked_compressed_data(): ) # the old data is still there -def test_add_symlink_layer(): +def test_add_symlink_layer() -> None: delete_dir("./testoutput/wk_dataset_with_symlink") delete_dir("./testoutput/simple_wk_dataset_copy") copytree("./testdata/simple_wk_dataset/", "./testoutput/simple_wk_dataset_copy/") @@ -1491,7 +1505,7 @@ def test_add_symlink_layer(): assert np.array_equal(original_mag.read(size=(10, 10, 10)), write_data) -def test_search_dataset_also_for_long_layer_name(): +def test_search_dataset_also_for_long_layer_name() -> None: delete_dir("./testoutput/long_layer_name") ds = WKDataset.create("./testoutput/long_layer_name", scale=(1, 1, 1)) @@ -1527,7 +1541,7 @@ def test_search_dataset_also_for_long_layer_name(): layer.delete_mag("2") -def test_outdated_dtype_parameter(): +def test_outdated_dtype_parameter() -> None: delete_dir("./testoutput/outdated_dtype") ds = WKDataset.create("./testoutput/outdated_dtype", scale=(1, 1, 1)) diff --git a/tests/test_downsampling.py b/tests/test_downsampling.py index eaed27e5a..dbdae00ba 100644 --- a/tests/test_downsampling.py +++ b/tests/test_downsampling.py @@ -1,5 +1,5 @@ import logging -from typing import Tuple +from typing import Tuple, cast import numpy as np from wkcuber.downsampling import ( @@ -24,16 +24,16 @@ def read_wkw( wkw_info: WkwDatasetInfo, offset: Tuple[int, int, int], size: Tuple[int, int, int] -): +) -> np.array: with open_wkw(wkw_info) as wkw_dataset: return wkw_dataset.read(offset, size) -def test_downsample_cube(): +def test_downsample_cube() -> None: buffer = np.zeros((CUBE_EDGE_LEN,) * 3, dtype=np.uint8) buffer[:, :, :] = np.arange(0, CUBE_EDGE_LEN) - output = downsample_cube(buffer, (2, 2, 2), InterpolationModes.MODE) + output = downsample_cube(buffer, [2, 2, 2], InterpolationModes.MODE) assert output.shape == (CUBE_EDGE_LEN // 2,) * 3 assert buffer[0, 0, 0] == 0 @@ -41,7 +41,7 @@ def test_downsample_cube(): assert np.all(output[:, :, :] == np.arange(0, CUBE_EDGE_LEN, 2)) -def test_downsample_mode(): +def test_downsample_mode() -> None: a = np.array([[1, 3, 4, 2, 2, 7], [5, 2, 2, 1, 4, 1], [3, 3, 2, 2, 1, 1]]) @@ -51,7 +51,7 @@ def test_downsample_mode(): assert np.all(result == expected_result) -def test_downsample_median(): +def test_downsample_median() -> None: a = np.array([[1, 3, 4, 2, 2, 7], [5, 2, 2, 1, 4, 1], [3, 3, 2, 2, 1, 1]]) @@ -61,7 +61,7 @@ def test_downsample_median(): assert np.all(result == expected_result) -def test_non_linear_filter_reshape(): +def test_non_linear_filter_reshape() -> None: a = np.array([[[1, 3], [1, 4]], [[4, 2], [3, 1]]], dtype=np.uint8) a_filtered = non_linear_filter_3d(a, [2, 2, 2], _mode) @@ -77,7 +77,7 @@ def test_non_linear_filter_reshape(): assert np.all(expected_result == a_filtered) -def test_cube_addresses(): +def test_cube_addresses() -> None: addresses = cube_addresses(source_info) assert len(addresses) == 5 * 5 * 1 @@ -85,7 +85,7 @@ def test_cube_addresses(): assert max(addresses) == (4, 4, 0) -def downsample_test_helper(use_compress): +def downsample_test_helper(use_compress: bool) -> None: try: shutil.rmtree(target_info.dataset_path) except: @@ -94,15 +94,15 @@ def downsample_test_helper(use_compress): offset = (1, 2, 0) source_buffer = read_wkw( source_info, - tuple(a * WKW_CUBE_SIZE * 2 for a in offset), - (CUBE_EDGE_LEN * 2,) * 3, + cast(Tuple[int, int, int], tuple(a * WKW_CUBE_SIZE * 2 for a in offset)), + (CUBE_EDGE_LEN * 2, CUBE_EDGE_LEN * 2, CUBE_EDGE_LEN * 2), )[0] assert np.any(source_buffer != 0) downsample_args = ( source_info, target_info, - (2, 2, 2), + [2, 2, 2], InterpolationModes.MAX, offset, CUBE_EDGE_LEN, @@ -118,7 +118,9 @@ def downsample_test_helper(use_compress): target_info.header.block_type = block_type target_buffer = read_wkw( - target_info, tuple(a * WKW_CUBE_SIZE for a in offset), (CUBE_EDGE_LEN,) * 3 + target_info, + cast(Tuple[int, int, int], tuple(a * WKW_CUBE_SIZE for a in offset)), + (CUBE_EDGE_LEN, CUBE_EDGE_LEN, CUBE_EDGE_LEN), )[0] assert np.any(target_buffer != 0) @@ -128,15 +130,15 @@ def downsample_test_helper(use_compress): ) -def test_downsample_cube_job(): +def test_downsample_cube_job() -> None: downsample_test_helper(False) -def test_compressed_downsample_cube_job(): +def test_compressed_downsample_cube_job() -> None: downsample_test_helper(True) -def test_downsample_multi_channel(): +def test_downsample_multi_channel() -> None: offset = (0, 0, 0) num_channels = 3 size = (32, 32, 10) @@ -171,9 +173,9 @@ def test_downsample_multi_channel(): downsample_args = ( source_info, target_info, - (2, 2, 2), + [2, 2, 2], InterpolationModes.MAX, - tuple(a * WKW_CUBE_SIZE for a in offset), + offset, CUBE_EDGE_LEN, False, True, @@ -191,28 +193,28 @@ def test_downsample_multi_channel(): target_buffer = read_wkw( target_info, - tuple(a * WKW_CUBE_SIZE for a in offset), - list(map(lambda x: x // 2, size)), + offset, + cast(Tuple[int, int, int], tuple([x // 2 for x in size])), ) assert np.any(target_buffer != 0) assert np.all(target_buffer == joined_buffer) -def test_anisotropic_mag_calculation(): +def test_anisotropic_mag_calculation() -> None: mag_tests = [ - [(10.5, 10.5, 24), Mag(1), Mag((2, 2, 1))], - [(10.5, 10.5, 21), Mag(1), Mag((2, 2, 1))], - [(10.5, 24, 10.5), Mag(1), Mag((2, 1, 2))], - [(24, 10.5, 10.5), Mag(1), Mag((1, 2, 2))], - [(10.5, 10.5, 10.5), Mag(1), Mag((2, 2, 2))], - [(10.5, 10.5, 24), Mag((2, 2, 1)), Mag((4, 4, 1))], - [(10.5, 10.5, 21), Mag((2, 2, 1)), Mag((4, 4, 2))], - [(10.5, 24, 10.5), Mag((2, 1, 2)), Mag((4, 1, 4))], - [(24, 10.5, 10.5), Mag((1, 2, 2)), Mag((1, 4, 4))], - [(10.5, 10.5, 10.5), Mag(2), Mag(4)], - [(320, 320, 200), Mag(1), Mag((1, 1, 2))], - [(320, 320, 200), Mag((1, 1, 2)), Mag((2, 2, 4))], + ((10.5, 10.5, 24), Mag(1), Mag((2, 2, 1))), + ((10.5, 10.5, 21), Mag(1), Mag((2, 2, 1))), + ((10.5, 24, 10.5), Mag(1), Mag((2, 1, 2))), + ((24, 10.5, 10.5), Mag(1), Mag((1, 2, 2))), + ((10.5, 10.5, 10.5), Mag(1), Mag((2, 2, 2))), + ((10.5, 10.5, 24), Mag((2, 2, 1)), Mag((4, 4, 1))), + ((10.5, 10.5, 21), Mag((2, 2, 1)), Mag((4, 4, 2))), + ((10.5, 24, 10.5), Mag((2, 1, 2)), Mag((4, 1, 4))), + ((24, 10.5, 10.5), Mag((1, 2, 2)), Mag((1, 4, 4))), + ((10.5, 10.5, 10.5), Mag(2), Mag(4)), + ((320, 320, 200), Mag(1), Mag((1, 1, 2))), + ((320, 320, 200), Mag((1, 1, 2)), Mag((2, 2, 4))), ] for i in range(len(mag_tests)): diff --git a/tests/test_export_wkw_as_tiff.py b/tests/test_export_wkw_as_tiff.py index 280a195d8..58ccf3345 100644 --- a/tests/test_export_wkw_as_tiff.py +++ b/tests/test_export_wkw_as_tiff.py @@ -11,7 +11,7 @@ source_path = os.path.join("testdata", ds_name) -def test_export_tiff_stack(): +def test_export_tiff_stack() -> None: destination_path = os.path.join("testoutput", ds_name + "_tiff") bbox = BoundingBox((100, 100, 10), (100, 500, 50)) bbox_dict = bbox.as_config() @@ -64,7 +64,7 @@ def test_export_tiff_stack(): ) -def test_export_tiff_stack_tile_size(): +def test_export_tiff_stack_tile_size() -> None: destination_path = os.path.join("testoutput", ds_name + "_tile_size") args_list = [ "--source_path", @@ -128,7 +128,7 @@ def test_export_tiff_stack_tile_size(): ) -def test_export_tiff_stack_tiles_per_dimension(): +def test_export_tiff_stack_tiles_per_dimension() -> None: destination_path = os.path.join("testoutput", ds_name + "_tiles_per_dimension") args_list = [ "--source_path", diff --git a/tests/test_mag.py b/tests/test_mag.py index 7d62ac2d2..99d07b00a 100644 --- a/tests/test_mag.py +++ b/tests/test_mag.py @@ -3,12 +3,12 @@ from wkcuber.metadata import detect_resolutions -def test_detect_resolutions(): +def test_detect_resolutions() -> None: resolutions = sorted(list(detect_resolutions("testdata/WT1_wkw", "color"))) assert [mag.to_layer_name() for mag in resolutions] == ["1", "2-2-1"] -def test_mag_constructor(): +def test_mag_constructor() -> None: mag = Mag(16) assert mag.to_array() == [16, 16, 16] diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 977b3f170..b7eaadefd 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -13,7 +13,7 @@ ) -def test_element_class_conversion(): +def test_element_class_conversion() -> None: test_wkw_path = os.path.join("testoutput", "test_metadata") prediction_layer_name = "prediction" prediction_wkw_info = WkwDatasetInfo( @@ -25,7 +25,7 @@ def test_element_class_conversion(): write_webknossos_metadata( test_wkw_path, "test_metadata", - [11.24, 11.24, 28], + (11.24, 11.24, 28), compute_max_id=True, exact_bounding_box={"topLeft": [0, 0, 0], "width": 4, "height": 4, "depth": 4}, ) @@ -40,8 +40,11 @@ def test_element_class_conversion(): def check_element_class_of_layer( - test_wkw_path, layer_name, expected_element_class, expected_dtype -): + test_wkw_path: str, + layer_name: str, + expected_element_class: str, + expected_dtype: type, +) -> None: datasource_properties = read_datasource_properties(test_wkw_path) layer_to_check = None for layer in datasource_properties["dataLayers"]: @@ -56,7 +59,9 @@ def check_element_class_of_layer( assert converted_dtype == expected_dtype -def write_custom_layer(target_path, layer_name, dtype, num_channels): +def write_custom_layer( + target_path: str, layer_name: str, dtype: type, num_channels: int +) -> None: data = ( np.arange(4 * 4 * 4 * num_channels) .reshape((num_channels, 4, 4, 4)) @@ -70,7 +75,7 @@ def write_custom_layer(target_path, layer_name, dtype, num_channels): dataset.write(off=(0, 0, 0), data=data) -def test_mapping_detection(): +def test_mapping_detection() -> None: # NOTE: the mappings do not match do the actual wkw data. Therefore do not use them expected_mappings = [ "test_mapping_1.json", diff --git a/tests/test_nifti_conversion.py b/tests/test_nifti_conversion.py index cf362cb1f..0f6038c80 100644 --- a/tests/test_nifti_conversion.py +++ b/tests/test_nifti_conversion.py @@ -2,7 +2,7 @@ from wkcuber.convert_nifti import to_target_datatype -def test_to_target_datatype_segmentation_layer(): +def test_to_target_datatype_segmentation_layer() -> None: data = np.array( [ [ diff --git a/tests/test_pad_or_crop.py b/tests/test_pad_or_crop.py index ed14b9368..9c3151088 100644 --- a/tests/test_pad_or_crop.py +++ b/tests/test_pad_or_crop.py @@ -2,7 +2,7 @@ from wkcuber.utils import pad_or_crop_to_size_and_topleft -def test_pad_or_crop_to_size_and_topleft(): +def test_pad_or_crop_to_size_and_topleft() -> None: target_topleft = np.array((0, 50, 36, 1)) target_size = np.array((1, 156, 112, 30)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 301793d0e..131964cd3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,12 +8,12 @@ BLOCK_LEN = 32 -def delete_dir(relative_path): +def delete_dir(relative_path: str) -> None: if os.path.exists(relative_path) and os.path.isdir(relative_path): rmtree(relative_path) -def test_get_chunks(): +def test_get_chunks() -> None: source = list(range(0, 48)) target = list(get_chunks(source, 8)) @@ -21,7 +21,7 @@ def test_get_chunks(): assert target[0] == list(range(0, 8)) -def test_get_regular_chunks(): +def test_get_regular_chunks() -> None: target = list(get_regular_chunks(4, 44, 8)) assert len(target) == 6 @@ -29,7 +29,7 @@ def test_get_regular_chunks(): assert list(target[-1]) == list(range(40, 48)) -def test_get_regular_chunks_max_inclusive(): +def test_get_regular_chunks_max_inclusive() -> None: target = list(get_regular_chunks(4, 44, 1)) assert len(target) == 41 @@ -38,7 +38,7 @@ def test_get_regular_chunks_max_inclusive(): assert list(target[-1]) == list(range(44, 45)) -def test_buffered_slice_writer(): +def test_buffered_slice_writer() -> None: test_img = np.arange(24 * 24).reshape(24, 24).astype(np.uint16) + 1 dtype = test_img.dtype bbox = {"topleft": (0, 0, 0), "size": (24, 24, 35)} diff --git a/typecheck.sh b/typecheck.sh index 896d4059f..e87d587dc 100755 --- a/typecheck.sh +++ b/typecheck.sh @@ -1,3 +1,7 @@ #!/bin/bash set -eEuo pipefail -python -m mypy wkcuber --disallow-untyped-defs --show-error-codes --strict-equality --namespace-packages +echo "Typecheck wkcuber module:" +python -m mypy -p wkcuber --disallow-untyped-defs --show-error-codes --strict-equality --namespace-packages + +echo "Typecheck tests:" +python -m mypy -p tests --disallow-untyped-defs --show-error-codes --strict-equality --namespace-packages diff --git a/wkcuber/api/Dataset.py b/wkcuber/api/Dataset.py index aaded747f..4bcd21d4f 100644 --- a/wkcuber/api/Dataset.py +++ b/wkcuber/api/Dataset.py @@ -4,7 +4,7 @@ from os import makedirs, path from os.path import join, normpath, basename from pathlib import Path -from typing import Type, Tuple, Union, Dict, Any, Optional, cast +from typing import Type, Tuple, Union, Dict, Any, Optional, cast, TypeVar, Generic import numpy as np import os @@ -76,16 +76,19 @@ def dtype_per_channel_to_dtype_per_layer( ) -class AbstractDataset(ABC): +LayerT = TypeVar("LayerT", bound=Layer) + + +class AbstractDataset(Generic[LayerT]): @abstractmethod def __init__(self, dataset_path: Union[str, Path]) -> None: properties: Properties = self._get_properties_type()._from_json( join(dataset_path, Properties.FILE_NAME) ) - self.layers: Dict[str, Layer] = {} + self.layers: Dict[str, LayerT] = {} self.path = Path(properties.path).parent self.properties = properties - self.data_format = "abstract" + self._data_format = "abstract" # construct self.layer for layer_name in self.properties.data_layers: @@ -126,7 +129,7 @@ def create_with_properties(cls, properties: Properties) -> "AbstractDataset": def get_properties(self) -> Properties: return self.properties - def get_layer(self, layer_name: str) -> Layer: + def get_layer(self, layer_name: str) -> LayerT: if layer_name not in self.layers.keys(): raise IndexError( "The layer {} is not a layer of this dataset".format(layer_name) @@ -141,7 +144,7 @@ def add_layer( dtype_per_channel: Union[str, np.dtype] = None, num_channels: int = None, **kwargs: Any, - ) -> Layer: + ) -> LayerT: if "dtype" in kwargs: raise ValueError( f"Called Dataset.add_layer with 'dtype'={kwargs['dtype']}. This parameter is deprecated. Use 'dtype_per_layer' or 'dtype_per_channel' instead." @@ -187,7 +190,7 @@ def add_layer( layer_name, category, dtype_per_layer, - self.data_format, + self._data_format, num_channels, **kwargs, ) @@ -204,7 +207,7 @@ def get_or_add_layer( dtype_per_channel: Union[str, np.dtype] = None, num_channels: int = None, **kwargs: Any, - ) -> Layer: + ) -> LayerT: if "dtype" in kwargs: raise ValueError( f"Called Dataset.get_or_add_layer with 'dtype'={kwargs['dtype']}. This parameter is deprecated. Use 'dtype_per_layer' or 'dtype_per_channel' instead." @@ -261,7 +264,7 @@ def delete_layer(self, layer_name: str) -> None: # delete files on disk rmtree(join(self.path, layer_name)) - def add_symlink_layer(self, foreign_layer_path: Union[str, Path]) -> Layer: + def add_symlink_layer(self, foreign_layer_path: Union[str, Path]) -> LayerT: foreign_layer_path = os.path.abspath(foreign_layer_path) layer_name = os.path.basename(os.path.normpath(foreign_layer_path)) if layer_name in self.layers.keys(): @@ -292,7 +295,7 @@ def add_symlink_layer(self, foreign_layer_path: Union[str, Path]) -> Layer: def get_view( self, layer_name: str, - mag: Union[str, Mag], + mag: Union[int, str, list, tuple, np.ndarray, Mag], size: Tuple[int, int, int], offset: Tuple[int, int, int] = None, is_bounded: bool = True, @@ -304,7 +307,7 @@ def get_view( def _create_layer( self, layer_name: str, dtype_per_channel: np.dtype, num_channels: int - ) -> Layer: + ) -> LayerT: raise NotImplementedError @abstractmethod @@ -316,7 +319,7 @@ def _get_type(self) -> Type["AbstractDataset"]: pass -class WKDataset(AbstractDataset): +class WKDataset(AbstractDataset[WKLayer]): @classmethod def create( cls, dataset_path: Union[str, Path], scale: Tuple[float, float, float] @@ -342,7 +345,7 @@ def get_or_create( def __init__(self, dataset_path: Union[str, Path]) -> None: super().__init__(dataset_path) - self.data_format = "wkw" + self._data_format = "wkw" assert isinstance(self.properties, WKProperties) def to_tiff_dataset(self, new_dataset_path: Union[str, Path]) -> "TiffDataset": @@ -350,7 +353,7 @@ def to_tiff_dataset(self, new_dataset_path: Union[str, Path]) -> "TiffDataset": def _create_layer( self, layer_name: str, dtype_per_channel: np.dtype, num_channels: int - ) -> Layer: + ) -> WKLayer: return WKLayer(layer_name, self, dtype_per_channel, num_channels) def _get_properties_type(self) -> Type[WKProperties]: @@ -360,7 +363,7 @@ def _get_type(self) -> Type["WKDataset"]: return WKDataset -class TiffDataset(AbstractDataset): +class TiffDataset(AbstractDataset[TiffLayer]): properties: TiffProperties @classmethod @@ -416,7 +419,7 @@ def to_wk_dataset(self, new_dataset_path: Union[str, Path]) -> WKDataset: def _create_layer( self, layer_name: str, dtype_per_channel: np.dtype, num_channels: int - ) -> Layer: + ) -> TiffLayer: return TiffLayer(layer_name, self, dtype_per_channel, num_channels) def _get_properties_type(self) -> Type[TiffProperties]: @@ -426,7 +429,7 @@ def _get_type(self) -> Type["TiffDataset"]: return TiffDataset -class TiledTiffDataset(AbstractDataset): +class TiledTiffDataset(AbstractDataset[TiledTiffLayer]): properties: TiffProperties @classmethod @@ -490,7 +493,7 @@ def __init__(self, dataset_path: Union[str, Path]) -> None: def _create_layer( self, layer_name: str, dtype_per_channel: np.dtype, num_channels: int - ) -> Layer: + ) -> TiledTiffLayer: return TiledTiffLayer(layer_name, self, dtype_per_channel, num_channels) def _get_properties_type(self) -> Type[TiffProperties]: diff --git a/wkcuber/api/Layer.py b/wkcuber/api/Layer.py index bd9449cf4..ccd88efdb 100644 --- a/wkcuber/api/Layer.py +++ b/wkcuber/api/Layer.py @@ -1,7 +1,8 @@ +from abc import ABC, abstractmethod from shutil import rmtree from os.path import join from os import makedirs -from typing import Tuple, Type, Union, Dict, Any, TYPE_CHECKING +from typing import Tuple, Type, Union, Dict, Any, TYPE_CHECKING, TypeVar, Generic import numpy as np @@ -15,12 +16,16 @@ TiffMagDataset, TiledTiffMagDataset, find_mag_path_on_disk, + GenericTiffMagDataset, ) from wkcuber.mag import Mag from wkcuber.utils import DEFAULT_WKW_FILE_LEN -class Layer: +MagT = TypeVar("MagT", bound=MagDataset) + + +class Layer(Generic[MagT]): COLOR_TYPE = "color" SEGMENTATION_TYPE = "segmentation" @@ -36,21 +41,26 @@ def __init__( self.dataset = dataset self.dtype_per_channel = dtype_per_channel self.num_channels = num_channels - self.mags: Dict[str, Any] = {} + self.mags: Dict[str, MagT] = {} full_path = join(dataset.path, name) makedirs(full_path, exist_ok=True) - def get_mag(self, mag: Union[str, Mag]) -> MagDataset: + def get_mag(self, mag: Union[int, str, list, tuple, np.ndarray, Mag]) -> MagT: mag = Mag(mag).to_layer_name() if mag not in self.mags.keys(): raise IndexError("The mag {} is not a mag of this layer".format(mag)) return self.mags[mag] - def get_or_add_mag(self, mag: Union[str, Mag], **kwargs: Any) -> MagDataset: + def add_mag(self, mag: Union[int, str, list, tuple, np.ndarray, Mag]) -> MagT: pass - def delete_mag(self, mag: Union[str, Mag]) -> None: + def get_or_add_mag( + self, mag: Union[int, str, list, tuple, np.ndarray, Mag] + ) -> MagT: + pass + + def delete_mag(self, mag: Union[int, str, list, tuple, np.ndarray, Mag]) -> None: mag = Mag(mag).to_layer_name() if mag not in self.mags.keys(): raise IndexError( @@ -63,12 +73,16 @@ def delete_mag(self, mag: Union[str, Mag]) -> None: full_path = find_mag_path_on_disk(self.dataset.path, self.name, mag) rmtree(full_path) - def _create_dir_for_mag(self, mag: Union[str, Mag]) -> None: + def _create_dir_for_mag( + self, mag: Union[int, str, list, tuple, np.ndarray, Mag] + ) -> None: mag = Mag(mag).to_layer_name() full_path = join(self.dataset.path, self.name, mag) makedirs(full_path, exist_ok=True) - def _assert_mag_does_not_exist_yet(self, mag: Union[str, Mag]) -> None: + def _assert_mag_does_not_exist_yet( + self, mag: Union[int, str, list, tuple, np.ndarray, Mag] + ) -> None: mag = Mag(mag).to_layer_name() if mag in self.mags.keys(): raise IndexError( @@ -103,42 +117,40 @@ def setup_mag(self, mag: str) -> None: pass -class WKLayer(Layer): +class WKLayer(Layer[WKMagDataset]): mags: Dict[str, WKMagDataset] def add_mag( self, - mag: Union[str, Mag], - block_len: int = None, - file_len: int = None, - block_type: int = None, - ) -> MagDataset: - if block_len is None: - block_len = 32 - if file_len is None: - file_len = DEFAULT_WKW_FILE_LEN - if block_type is None: - block_type = wkw.Header.BLOCK_TYPE_RAW - + mag: Union[int, str, list, tuple, np.ndarray, Mag], + block_len: int = 32, + file_len: int = DEFAULT_WKW_FILE_LEN, + block_type: int = wkw.Header.BLOCK_TYPE_RAW, + ) -> WKMagDataset: # normalize the name of the mag mag = Mag(mag).to_layer_name() self._assert_mag_does_not_exist_yet(mag) self._create_dir_for_mag(mag) - self.mags[mag] = WKMagDataset.create(self, mag, block_len, file_len, block_type) + self.mags[mag] = WKMagDataset( + self, mag, block_len, file_len, block_type, create=True + ) self.dataset.properties._add_mag( self.name, mag, cube_length=block_len * file_len ) return self.mags[mag] - def get_or_add_mag(self, mag: Union[str, Mag], **kwargs: Any) -> MagDataset: + def get_or_add_mag( + self, + mag: Union[int, str, list, tuple, np.ndarray, Mag], + block_len: int = 32, + file_len: int = DEFAULT_WKW_FILE_LEN, + block_type: int = wkw.Header.BLOCK_TYPE_RAW, + ) -> WKMagDataset: # normalize the name of the mag mag = Mag(mag).to_layer_name() - block_len: int = kwargs.get("block_len", None) - file_len: int = kwargs.get("file_len", None) - block_type: int = kwargs.get("block_type", None) if mag in self.mags.keys(): assert ( @@ -152,7 +164,9 @@ def get_or_add_mag(self, mag: Union[str, Mag], **kwargs: Any) -> MagDataset: ), f"Cannot get_or_add_mag: The mag {mag} already exists, but the block types do not match" return self.get_mag(mag) else: - return self.add_mag(mag, block_len, file_len, block_type) + return self.add_mag( + mag, block_len=block_len, file_len=file_len, block_type=block_type + ) def setup_mag(self, mag: str) -> None: # This method is used to initialize the mag when opening the Dataset. This does not create e.g. the wk_header. @@ -175,24 +189,29 @@ def setup_mag(self, mag: str) -> None: ) -class TiffLayer(Layer): +TiffMagT = TypeVar("TiffMagT", bound=GenericTiffMagDataset) + + +class GenericTiffLayer(Layer[TiffMagT], ABC): dataset: "TiffDataset" - def add_mag(self, mag: Union[str, Mag]) -> MagDataset: + def add_mag(self, mag: Union[int, str, list, tuple, np.ndarray, Mag]) -> TiffMagT: # normalize the name of the mag mag = Mag(mag).to_layer_name() self._assert_mag_does_not_exist_yet(mag) self._create_dir_for_mag(mag) - self.mags[mag] = self._get_mag_dataset_class().create( + self.mags[mag] = self._get_mag_dataset_class()( self, mag, self.dataset.properties.pattern ) self.dataset.properties._add_mag(self.name, mag) return self.mags[mag] - def get_or_add_mag(self, mag: Union[str, Mag], **kwargs: Any) -> MagDataset: + def get_or_add_mag( + self, mag: Union[int, str, list, tuple, np.ndarray, Mag] + ) -> TiffMagT: # normalize the name of the mag mag = Mag(mag).to_layer_name() @@ -214,10 +233,16 @@ def setup_mag(self, mag: str) -> None: ) self.dataset.properties._add_mag(self.name, mag) + @abstractmethod + def _get_mag_dataset_class(self) -> Type[TiffMagT]: + pass + + +class TiffLayer(GenericTiffLayer[TiffMagDataset]): def _get_mag_dataset_class(self) -> Type[TiffMagDataset]: return TiffMagDataset -class TiledTiffLayer(TiffLayer): +class TiledTiffLayer(GenericTiffLayer[TiledTiffMagDataset]): def _get_mag_dataset_class(self) -> Type[TiledTiffMagDataset]: return TiledTiffMagDataset diff --git a/wkcuber/api/MagDataset.py b/wkcuber/api/MagDataset.py index 71e766894..6620921e1 100644 --- a/wkcuber/api/MagDataset.py +++ b/wkcuber/api/MagDataset.py @@ -1,7 +1,7 @@ import os from os.path import join from pathlib import Path -from typing import Type, Tuple, Union, cast, TYPE_CHECKING +from typing import Type, Tuple, Union, cast, TYPE_CHECKING, TypeVar, Generic, Any from wkw import wkw import numpy as np @@ -9,7 +9,13 @@ import wkcuber.api as api if TYPE_CHECKING: - from wkcuber.api.Layer import TiffLayer, WKLayer, Layer + from wkcuber.api.Layer import ( + WKLayer, + Layer, + GenericTiffLayer, + TiffLayer, + TiledTiffLayer, + ) from wkcuber.api.View import WKView, TiffView, View from wkcuber.api.TiffData.TiffMag import TiffMagHeader from wkcuber.mag import Mag @@ -165,11 +171,16 @@ def __init__( block_len: int, file_len: int, block_type: int, + create: bool = False, ) -> None: self.block_len = block_len self.file_len = file_len self.block_type = block_type super().__init__(layer, name) + if create: + wkw.Dataset.create( + join(layer.dataset.path, layer.name, self.name), self.header + ) def get_header(self) -> wkw.Header: return wkw.Header( @@ -181,25 +192,17 @@ def get_header(self) -> wkw.Header: block_type=self.block_type, ) - @classmethod - def create( - cls, layer: "WKLayer", name: str, block_len: int, file_len: int, block_type: int - ) -> "WKMagDataset": - mag_dataset = cls(layer, name, block_len, file_len, block_type) - wkw.Dataset.create( - join(layer.dataset.path, layer.name, mag_dataset.name), mag_dataset.header - ) - - return mag_dataset - def _get_view_type(self) -> Type[WKView]: return WKView -class TiffMagDataset(MagDataset): - layer: "TiffLayer" +TiffLayerT = TypeVar("TiffLayerT", bound="GenericTiffLayer") + - def __init__(self, layer: "TiffLayer", name: str, pattern: str) -> None: +class GenericTiffMagDataset(MagDataset, Generic[TiffLayerT]): + layer: TiffLayerT + + def __init__(self, layer: TiffLayerT, name: str, pattern: str) -> None: self.pattern = pattern super().__init__(layer, name) @@ -211,16 +214,15 @@ def get_header(self) -> TiffMagHeader: tile_size=self.layer.dataset.properties.tile_size, ) - @classmethod - def create(cls, layer: "TiffLayer", name: str, pattern: str) -> "TiffMagDataset": - mag_dataset = cls(layer, name, pattern) - return mag_dataset - def _get_view_type(self) -> Type[TiffView]: return TiffView -class TiledTiffMagDataset(TiffMagDataset): +class TiffMagDataset(GenericTiffMagDataset["TiffLayer"]): + pass + + +class TiledTiffMagDataset(GenericTiffMagDataset["TiledTiffLayer"]): def get_tile(self, x_index: int, y_index: int, z_index: int) -> np.array: tile_size = self.layer.dataset.properties.tile_size assert tile_size is not None diff --git a/wkcuber/api/View.py b/wkcuber/api/View.py index 3deaf6622..0102c2f72 100644 --- a/wkcuber/api/View.py +++ b/wkcuber/api/View.py @@ -132,7 +132,7 @@ def assert_bounds( def for_each_chunk( self, - work_on_chunk: Callable[[List[Any]], None], + work_on_chunk: Callable[[Tuple["View", Tuple[Any, ...]]], None], job_args_per_chunk: Any, chunk_size: Tuple[int, int, int], executor: Union[ClusterExecutor, cluster_tools.WrappedProcessPoolExecutor], diff --git a/wkcuber/convert_nifti.py b/wkcuber/convert_nifti.py index 8f02ad32e..5cde6e8cb 100644 --- a/wkcuber/convert_nifti.py +++ b/wkcuber/convert_nifti.py @@ -215,27 +215,27 @@ def convert_nifti( tiff_ds = TiffDataset.get_or_create( target_path, scale=cast(Tuple[float, float, float], scale or (1, 1, 1)) ) - layer = tiff_ds.get_or_add_layer( + tiff_layer = tiff_ds.get_or_add_layer( layer_name, category_type, dtype_per_layer=np.dtype(dtype), **max_cell_id_args, ) - mag = layer.get_or_add_mag("1") + tiff_mag = tiff_layer.get_or_add_mag("1") - mag.write(cube_data.squeeze()) + tiff_mag.write(cube_data.squeeze()) else: wk_ds = WKDataset.get_or_create( target_path, scale=cast(Tuple[float, float, float], scale or (1, 1, 1)) ) - layer = wk_ds.get_or_add_layer( + wk_layer = wk_ds.get_or_add_layer( layer_name, category_type, dtype_per_layer=np.dtype(dtype), **max_cell_id_args, ) - mag = layer.get_or_add_mag("1", file_len=file_len) - mag.write(cube_data) + wk_mag = wk_layer.get_or_add_mag("1", file_len=file_len) + wk_mag.write(cube_data) logging.debug( "Converting of {} took {:.8f}s".format(