diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be577837e..6d8bf0cc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.7, 3.8] steps: - uses: actions/checkout@v1 @@ -25,10 +25,10 @@ jobs: run: | pip install poetry poetry install - + - name: Decompress test data run: tar -xzvf testdata/WT1_wkw.tar.gz - + - name: Check formatting run: poetry run black --check . @@ -38,10 +38,10 @@ jobs: - name: Check typing run: | ./typecheck.sh - + - name: Python tests run: poetry run pytest tests - + - name: Smoke test docker run: | docker run --rm \ @@ -59,19 +59,19 @@ jobs: - name: Test tile cubing run: tests/scripts/tile_cubing.sh - + - name: Test simple tiff cubing run: tests/scripts/simple_tiff_cubing.sh - + - name: Test simple tiff cubing (no compression) run: tests/scripts/simple_tiff_cubing_no_compression.sh - + - name: Test metadata generation run: tests/scripts/meta_generation.sh - + - name: Test KNOSSOS conversion run: tests/scripts/knossos_conversion.sh - + - name: Decompress reference magnification data run: | mkdir -p testdata/tiff_mag_2_reference @@ -98,7 +98,7 @@ jobs: DOCKER_PASS: ${{ secrets.DOCKER_PASS }} run: | echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - + - name: Push docker images run: | docker push scalableminds/webknossos-cuber:$GITHUB_SHA diff --git a/poetry.lock b/poetry.lock index 131a46407..4cf95c8a2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -53,7 +53,6 @@ python-versions = ">=3.6" [package.dependencies] appdirs = "*" click = ">=7.1.2" -dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} mypy-extensions = ">=0.4.3" pathspec = ">=0.6,<1" regex = ">=2020.1.8" @@ -138,14 +137,6 @@ python-versions = "*" [package.dependencies] six = "*" -[[package]] -name = "dataclasses" -version = "0.6" -description = "A backport of the dataclasses module for Python 3.6" -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "decorator" version = "4.4.2" @@ -162,6 +153,20 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "imagecodecs" +version = "2020.12.24" +description = "Image transformation, compression, and decompression codecs" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.15.1" + +[package.extras] +all = ["matplotlib (>=3.2)", "tifffile (>=2020.12.8)"] + [[package]] name = "imageio" version = "2.9.0" @@ -536,7 +541,7 @@ test = ["pytest (!=3.7.3)", "pytest-cov", "pytest-localserver", "flake8", "codec [[package]] name = "scikit-learn" -version = "0.24.0" +version = "0.24.1" description = "A set of python modules for machine learning and data mining" category = "main" optional = false @@ -556,14 +561,14 @@ tests = ["matplotlib (>=2.1.1)", "scikit-image (>=0.13)", "pandas (>=0.25.0)", " [[package]] name = "scipy" -version = "1.5.4" +version = "1.6.0" description = "SciPy: Scientific Library for Python" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -numpy = ">=1.14.5" +numpy = ">=1.16.5" [[package]] name = "setuptools-scm" @@ -592,6 +597,20 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "tifffile" +version = "2020.12.8" +description = "Read and write TIFF(r) files" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.15.1" + +[package.extras] +all = ["imagecodecs (>=2020.5.30)", "matplotlib (>=3.2)", "lxml"] + [[package]] name = "toml" version = "0.10.2" @@ -663,8 +682,8 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" -python-versions = "^3.6" -content-hash = "41e718ffb14e47aab8f706e43750554af5fa4eca12535029fbaefff65f431685" +python-versions = "^3.7" +content-hash = "6b9e47f9a57731afa7d3035cb0cd7ea6fa837f3b3b12e31a619f994fc9e1a6d2" [metadata.files] appdirs = [ @@ -751,10 +770,6 @@ cycler = [ {file = "cycler-0.10.0-py2.py3-none-any.whl", hash = "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d"}, {file = "cycler-0.10.0.tar.gz", hash = "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"}, ] -dataclasses = [ - {file = "dataclasses-0.6-py3-none-any.whl", hash = "sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f"}, - {file = "dataclasses-0.6.tar.gz", hash = "sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"}, -] decorator = [ {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, {file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"}, @@ -763,6 +778,31 @@ idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] +imagecodecs = [ + {file = "imagecodecs-2020.12.24-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:cbab499b936fba7ea6785d5e29302f65f6e6775eb85ff3789d43a4f32e05ddef"}, + {file = "imagecodecs-2020.12.24-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb66f56c4d806d60271dae38b65e640f81acae14c0bdc3394661f7df65655854"}, + {file = "imagecodecs-2020.12.24-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:947824b6e12e2305378394b370b99222f444f05d70d4764f13108c6e53c674dc"}, + {file = "imagecodecs-2020.12.24-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:40e94929fcb1d7e16484d508f67bb523e703fe25960decfe61d639de88971e19"}, + {file = "imagecodecs-2020.12.24-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:3f3b9332cda9b7e0e72b2b36ac1456b99541277c77cb041b7d7327712b5b67ba"}, + {file = "imagecodecs-2020.12.24-cp37-cp37m-win32.whl", hash = "sha256:145900dd5db4b13400d5742c1b63da2c6553edf42a0487d1c1d1e63702b5440a"}, + {file = "imagecodecs-2020.12.24-cp37-cp37m-win_amd64.whl", hash = "sha256:b2b90c00eaf81e7772e56c359c7eab4067ac3728d4b7d44953464df227a032c1"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:955e0350d38d6f34260608b8cc95a6dcd954f7b1ac88332e1f61ec90444bbcc2"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5227e5fe30813099f8f87fe20efc065d08d6a073921566f8a5828e6475111aca"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c233b7191675474122da096fd7a60b0301b23181873ea2cef5b8d880473d0fbb"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:a5cba1ca4ad1c008839514980770d841616f965f90ebe123a6805aca95b5c257"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:cabd3bd74549c079a38d9842ff110ddcb29cba2613ee2a664c0ac31d9f5f79f5"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-win32.whl", hash = "sha256:4c392a308a3cc50f4461f377f511c4d244397510356d38c30e7587e85b2a4eec"}, + {file = "imagecodecs-2020.12.24-cp38-cp38-win_amd64.whl", hash = "sha256:63f66fda54530706ecdd268e3bb1916aca7ec9d4695209167a677ed75f61640f"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:6e8eaa870834ccc0a77eda22a1c9e800ea91aa174184052f3e56876c6ba2bb3f"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c5ac2276f3ee170a94508c4d68e60dbda414ab8957f5152b6fd083abaef44186"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:1df5db330ab8be64a1faee702df8bef68bd3fa4cddd400c03cca4b5af52431c7"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:df131cee7a490513b6c64853c382615b70ce7c8522da16af00cd3275a96967f4"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:d23f3c1f776ff377bf26f1a11ed6968b895ca6ab856088900bd6cc5dd5b60135"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-win32.whl", hash = "sha256:5e35b7648ec4ddda40c58db42138edd113a9d7d1914483e295d51e0a948b8035"}, + {file = "imagecodecs-2020.12.24-cp39-cp39-win_amd64.whl", hash = "sha256:b5fdfa68c4b46e1ef4c80747e3caec57c4cc434704019dcf5ddc265e957dcd86"}, + {file = "imagecodecs-2020.12.24-pp37-pypy37_pp73-win32.whl", hash = "sha256:4edde922c1fb4184d53d46fd597388ffa62a3f67c6afbd6508b8d0041a55eeab"}, + {file = "imagecodecs-2020.12.24.tar.gz", hash = "sha256:bcfb909c38fc0460f24ab75f58c643f2c50a10d0fa49ac07d5ea7b780765e7bd"}, +] imageio = [ {file = "imageio-2.9.0-py3-none-any.whl", hash = "sha256:3604d751f03002e8e0e7650aa71d8d9148144a87daf17cb1f3228e80747f2e6b"}, {file = "imageio-2.9.0.tar.gz", hash = "sha256:52ddbaeca2dccf53ba2d6dec5676ca7bc3b2403ef8b37f7da78b7654bb3e10f0"}, @@ -1135,58 +1175,48 @@ scikit-image = [ {file = "scikit_image-0.16.2-cp38-cp38-win_amd64.whl", hash = "sha256:e18d73cc8893e2268b172c29f9aab530faf8cd3b7c11ae0bee3e763d719d35c5"}, ] scikit-learn = [ - {file = "scikit-learn-0.24.0.tar.gz", hash = "sha256:076369634ee72b5a5941440661e2f306ff4ac30903802dc52031c7e9199ac640"}, - {file = "scikit_learn-0.24.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:890d7d588f65acb0c4f6c083347c9076916bda5e6bd8400f06244b1afc1009af"}, - {file = "scikit_learn-0.24.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:e534f5f3796db6781c87e9835dcd51b7854c8c5a379c9210b93605965c1941fd"}, - {file = "scikit_learn-0.24.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:d7fe05fcb44eadd6d6c874c768f085f5de1239db3a3b7be4d3d23d12e4120589"}, - {file = "scikit_learn-0.24.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7f654befc5ad413690cc58f3f34a3e906caf825195ce0fda00a8e9565e1403e6"}, - {file = "scikit_learn-0.24.0-cp36-cp36m-win32.whl", hash = "sha256:afeb06dc69847927634e58579b9cdc72e1390b79497336b2324b1b173f33bd47"}, - {file = "scikit_learn-0.24.0-cp36-cp36m-win_amd64.whl", hash = "sha256:26f66b3726b54dfb76ea51c5d9c2431ed17ebc066cb4527662b9e851a3e7ba61"}, - {file = "scikit_learn-0.24.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c08b27cb78ee8d2dc781a7affed09859441f5b624f9f92da59ac0791c8774dfc"}, - {file = "scikit_learn-0.24.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:905d8934d1e27a686698864a5863ff2c0e13a2ae1adb78a8a848aacc8a49927d"}, - {file = "scikit_learn-0.24.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d819d625832fb2969911a243e009cfa135cb8ef1e150866e417d6e9d75290087"}, - {file = "scikit_learn-0.24.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:18f7131e62265bf2691ed1d0303c640313894ccfe4278427478c6b2f45094b53"}, - {file = "scikit_learn-0.24.0-cp37-cp37m-win32.whl", hash = "sha256:b0d13fd56d26cf3de0314a4fd48037108c638fe126d813f5c1222bb0f08b6a76"}, - {file = "scikit_learn-0.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c912247e42114f389858ae05d63f4359d4e667ea72aaabee191aee9ad3f9774a"}, - {file = "scikit_learn-0.24.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:758619e49cd7c17282e6cc60d5cc73c02c072b47c9a10010bb3bb47e0d976e50"}, - {file = "scikit_learn-0.24.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:66f27bf21202a850bcd7b6303916e4907f6e22ec59a14974ede4955aed5c7ed0"}, - {file = "scikit_learn-0.24.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:5e6e3c042cea83f2e20a45e563b8eabc1f8f72446251fe23ebefdf111a173a33"}, - {file = "scikit_learn-0.24.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2a5348585aa793bc8cc5a72f8e9067c9380834b0aadbd55f924843b071f13282"}, - {file = "scikit_learn-0.24.0-cp38-cp38-win32.whl", hash = "sha256:743b6edd98c98991be46c08e6b21df3861d5ae915f91d59f988384d93f7263e7"}, - {file = "scikit_learn-0.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:2951f87d35e72f007701c6e028aa230f6df6212a3194677c0c950486066a454d"}, - {file = "scikit_learn-0.24.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:44e452ea8491225c5783d49577aad0f36202dfd52aec7f82c0fdfe5fbd5f7400"}, - {file = "scikit_learn-0.24.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:800aaf63f8838c00e85db2267dd226f89858594843fd03932a9eda95746d2c40"}, - {file = "scikit_learn-0.24.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:3eeff086f7329521d27249a082ea3c48c085cedb110db5f65968ab55c3ba2e09"}, - {file = "scikit_learn-0.24.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:4395e91b3548005f4a645018435b5a94f8cce232b5b70753020e606c6a750656"}, - {file = "scikit_learn-0.24.0-cp39-cp39-win32.whl", hash = "sha256:80ca024154b84b6ac4cfc86930ba13fdc348a209753bf2c16129db6f9eb8a80b"}, - {file = "scikit_learn-0.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:490436b44b3a1957cb625e871764b0aa330b34cc416aea4abc6c38ca63d0d682"}, + {file = "scikit-learn-0.24.1.tar.gz", hash = "sha256:a0334a1802e64d656022c3bfab56a73fbd6bf4b1298343f3688af2151810bbdf"}, + {file = "scikit_learn-0.24.1-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:9bed8a1ef133c8e2f13966a542cb8125eac7f4b67dcd234197c827ba9c7dd3e0"}, + {file = "scikit_learn-0.24.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:9dfa564ef27e8e674aa1cc74378416d580ac4ede1136c13dd555a87996e13422"}, + {file = "scikit_learn-0.24.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:9c6097b6a9b2bafc5e0f31f659e6ab5e131383209c30c9e978c5b8abdac5ed2a"}, + {file = "scikit_learn-0.24.1-cp36-cp36m-win32.whl", hash = "sha256:7b04691eb2f41d2c68dbda8d1bd3cb4ef421bdc43aaa56aeb6c762224552dfb6"}, + {file = "scikit_learn-0.24.1-cp36-cp36m-win_amd64.whl", hash = "sha256:1adf483e91007a87171d7ce58c34b058eb5dab01b5fee6052f15841778a8ecd8"}, + {file = "scikit_learn-0.24.1-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:ddb52d088889f5596bc4d1de981f2eca106b58243b6679e4782f3ba5096fd645"}, + {file = "scikit_learn-0.24.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:99349d77f54e11f962d608d94dfda08f0c9e5720d97132233ebdf35be2858b2d"}, + {file = "scikit_learn-0.24.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:83b21ff053b1ff1c018a2d24db6dd3ea339b1acfbaa4d9c881731f43748d8b3b"}, + {file = "scikit_learn-0.24.1-cp37-cp37m-win32.whl", hash = "sha256:c3deb3b19dd9806acf00cf0d400e84562c227723013c33abefbbc3cf906596e9"}, + {file = "scikit_learn-0.24.1-cp37-cp37m-win_amd64.whl", hash = "sha256:d54dbaadeb1425b7d6a66bf44bee2bb2b899fe3e8850b8e94cfb9c904dcb46d0"}, + {file = "scikit_learn-0.24.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:3c4f07f47c04e81b134424d53c3f5e16dfd7f494e44fd7584ba9ce9de2c5e6c1"}, + {file = "scikit_learn-0.24.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:826b92bf45b8ad80444814e5f4ac032156dd481e48d7da33d611f8fe96d5f08b"}, + {file = "scikit_learn-0.24.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:259ec35201e82e2db1ae2496f229e63f46d7f1695ae68eef9350b00dc74ba52f"}, + {file = "scikit_learn-0.24.1-cp38-cp38-win32.whl", hash = "sha256:8772b99d683be8f67fcc04789032f1b949022a0e6880ee7b75a7ec97dbbb5d0b"}, + {file = "scikit_learn-0.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:ed9d65594948678827f4ff0e7ae23344e2f2b4cabbca057ccaed3118fdc392ca"}, + {file = "scikit_learn-0.24.1-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:8aa1b3ac46b80eaa552b637eeadbbce3be5931e4b5002b964698e33a1b589e1e"}, + {file = "scikit_learn-0.24.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:895dbf2030aa7337649e36a83a007df3c9811396b4e2fa672a851160f36ce90c"}, + {file = "scikit_learn-0.24.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:9a24d1ccec2a34d4cd3f2a1f86409f3f5954cc23d4d2270ba0d03cf018aa4780"}, + {file = "scikit_learn-0.24.1-cp39-cp39-win32.whl", hash = "sha256:fab31f48282ebf54dd69f6663cd2d9800096bad1bb67bbc9c9ac84eb77b41972"}, + {file = "scikit_learn-0.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:4562dcf4793e61c5d0f89836d07bc37521c3a1889da8f651e2c326463c4bd697"}, ] scipy = [ - {file = "scipy-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47"}, - {file = "scipy-1.5.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d"}, - {file = "scipy-1.5.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9"}, - {file = "scipy-1.5.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06"}, - {file = "scipy-1.5.4-cp36-cp36m-win32.whl", hash = "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340"}, - {file = "scipy-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389"}, - {file = "scipy-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62"}, - {file = "scipy-1.5.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012"}, - {file = "scipy-1.5.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554"}, - {file = "scipy-1.5.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c"}, - {file = "scipy-1.5.4-cp37-cp37m-win32.whl", hash = "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54"}, - {file = "scipy-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e"}, - {file = "scipy-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938"}, - {file = "scipy-1.5.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce"}, - {file = "scipy-1.5.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce"}, - {file = "scipy-1.5.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42"}, - {file = "scipy-1.5.4-cp38-cp38-win32.whl", hash = "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443"}, - {file = "scipy-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437"}, - {file = "scipy-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4"}, - {file = "scipy-1.5.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e"}, - {file = "scipy-1.5.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660"}, - {file = "scipy-1.5.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57"}, - {file = "scipy-1.5.4-cp39-cp39-win32.whl", hash = "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474"}, - {file = "scipy-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2"}, - {file = "scipy-1.5.4.tar.gz", hash = "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b"}, + {file = "scipy-1.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3d4303e3e21d07d9557b26a1707bb9fc065510ee8501c9bf22a0157249a82fd0"}, + {file = "scipy-1.6.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:1bc5b446600c4ff7ab36bade47180673141322f0febaa555f1c433fe04f2a0e3"}, + {file = "scipy-1.6.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8840a9adb4ede3751f49761653d3ebf664f25195fdd42ada394ffea8903dd51d"}, + {file = "scipy-1.6.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:8629135ee00cc2182ac8be8e75643b9f02235942443732c2ed69ab48edcb6614"}, + {file = "scipy-1.6.0-cp37-cp37m-win32.whl", hash = "sha256:58731bbe0103e96b89b2f41516699db9b63066e4317e31b8402891571f6d358f"}, + {file = "scipy-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:876badc33eec20709d4e042a09834f5953ebdac4088d45a4f3a1f18b56885718"}, + {file = "scipy-1.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c0911f3180de343643f369dc5cfedad6ba9f939c2d516bddea4a6871eb000722"}, + {file = "scipy-1.6.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:b8af26839ae343655f3ca377a5d5e5466f1d3b3ac7432a43449154fe958ae0e0"}, + {file = "scipy-1.6.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:4f1d9cc977ac6a4a63c124045c1e8bf67ec37098f67c699887a93736961a00ae"}, + {file = "scipy-1.6.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:eb7928275f3560d47e5538e15e9f32b3d64cd30ea8f85f3e82987425476f53f6"}, + {file = "scipy-1.6.0-cp38-cp38-win32.whl", hash = "sha256:31ab217b5c27ab429d07428a76002b33662f98986095bbce5d55e0788f7e8b15"}, + {file = "scipy-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:2f1c2ebca6fd867160e70102200b1bd07b3b2d31a3e6af3c58d688c15d0d07b7"}, + {file = "scipy-1.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:155225621df90fcd151e25d51c50217e412de717475999ebb76e17e310176981"}, + {file = "scipy-1.6.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:f68d5761a2d2376e2b194c8e9192bbf7c51306ca176f1a0889990a52ef0d551f"}, + {file = "scipy-1.6.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d902d3a5ad7f28874c0a82db95246d24ca07ad932741df668595fe00a4819870"}, + {file = "scipy-1.6.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:aef3a2dbc436bbe8f6e0b635f0b5fe5ed024b522eee4637dbbe0b974129ca734"}, + {file = "scipy-1.6.0-cp39-cp39-win32.whl", hash = "sha256:cdbc47628184a0ebeb5c08f1892614e1bd4a51f6e0d609c6eed253823a960f5b"}, + {file = "scipy-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:313785c4dab65060f9648112d025f6d2fec69a8a889c714328882d678a95f053"}, + {file = "scipy-1.6.0.tar.gz", hash = "sha256:cb6dc9f82dfd95f6b9032a8d7ea70efeeb15d5b5fd6ed4e8537bb3c673580566"}, ] setuptools-scm = [ {file = "setuptools_scm-3.5.0-py2.7.egg", hash = "sha256:fa6511072840d7eaad3ea36cc8849f437fefd85b32499a7f6026cba16ebbf63a"}, @@ -1205,6 +1235,10 @@ threadpoolctl = [ {file = "threadpoolctl-2.1.0-py3-none-any.whl", hash = "sha256:38b74ca20ff3bb42caca8b00055111d74159ee95c4370882bbff2b93d24da725"}, {file = "threadpoolctl-2.1.0.tar.gz", hash = "sha256:ddc57c96a38beb63db45d6c159b5ab07b6bced12c45a1f07b2b92f272aebfa6b"}, ] +tifffile = [ + {file = "tifffile-2020.12.8-py3-none-any.whl", hash = "sha256:78e7fe7d27fdd3fbd67cc95aaa8265c43b942170bd2473b1d587dcc849822892"}, + {file = "tifffile-2020.12.8.tar.gz", hash = "sha256:6c65c3997a21cad40349921e557a383fd5f0ebd728f5e91fa6c8f8f9e45c4bbd"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pyproject.toml b/pyproject.toml index c0cb77b9b..5f0749d20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" license = "AGPL-3.0" [tool.poetry.dependencies] -python = "^3.6" +python = "^3.7" scipy = "^1.4.0" numpy = "^1.17.4" pillow = "^6.2.1" @@ -19,6 +19,8 @@ psutil = "^5.6.7" nibabel = "^2.5.1" scikit-image = "^0.16.2" scikit-learn = "^0.24.0" +tifffile = "^2020.11.26" +imagecodecs = "^2020.5.30" [tool.poetry.dev-dependencies] pylint = "^2.6.0" diff --git a/tests/test_utils.py b/tests/test_utils.py index 0226c82d4..301793d0e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,10 +3,16 @@ import wkw from wkcuber.mag import Mag import os +from shutil import rmtree BLOCK_LEN = 32 +def delete_dir(relative_path): + if os.path.exists(relative_path) and os.path.isdir(relative_path): + rmtree(relative_path) + + def test_get_chunks(): source = list(range(0, 48)) target = list(get_chunks(source, 8)) @@ -42,6 +48,8 @@ def test_buffered_slice_writer(): mag = Mag(1) dataset_path = os.path.join(dataset_dir, layer_name, mag.to_layer_name()) + delete_dir(dataset_dir) + with BufferedSliceWriter(dataset_dir, layer_name, dtype, origin, mag=mag) as writer: for i in range(13): writer.write_slice(i, test_img) diff --git a/wkcuber/cubing.py b/wkcuber/cubing.py index 7371b6151..cf4d2d036 100644 --- a/wkcuber/cubing.py +++ b/wkcuber/cubing.py @@ -109,9 +109,9 @@ def find_source_filenames(source_path: str) -> List[str]: return natsorted(source_files) -def read_image_file(file_name: str, dtype: type) -> np.ndarray: +def read_image_file(file_name: str, dtype: type, z_slice: int) -> np.ndarray: try: - return image_reader.read_array(file_name, dtype) + return image_reader.read_array(file_name, dtype, z_slice) except Exception as exc: logging.error("Reading of file={} failed with {}".format(file_name, exc)) raise exc @@ -176,7 +176,7 @@ def cubing_job( for z, file_name in zip(z_batch, source_file_batch): # Image shape will be (x, y, channel_count, z=1) image = read_image_file( - file_name, target_wkw_info.header.voxel_type + file_name, target_wkw_info.header.voxel_type, z ) if not pad: @@ -237,13 +237,19 @@ def cubing( batch_size: int, args: Namespace, ) -> dict: - source_files = find_source_filenames(source_path) # All images are assumed to have equal dimensions num_x, num_y = image_reader.read_dimensions(source_files[0]) num_channels = image_reader.read_channel_count(source_files[0]) - num_z = len(source_files) + num_z_slices_per_file = image_reader.read_z_slices_per_file(source_files[0]) + assert ( + num_z_slices_per_file == 1 or len(source_files) == 1 + ), "Multi page TIFF support only for single files" + if num_z_slices_per_file > 1: + num_z = num_z_slices_per_file + else: + num_z = len(source_files) target_mag = Mag(args.target_mag) target_wkw_info = WkwDatasetInfo( @@ -277,6 +283,11 @@ def cubing( # Prepare z batches max_z = min(num_z + start_z, z + BLOCK_LEN) z_batch = list(range(z, max_z)) + # Prepare source files array + if len(source_files) > 1: + source_files_array = source_files[z - start_z : max_z - start_z] + else: + source_files_array = source_files * (max_z - z) # Prepare job job_args.append( ( @@ -284,7 +295,7 @@ def cubing( z_batch, target_mag, interpolation_mode, - source_files[z - start_z : max_z - start_z], + source_files_array, batch_size, (num_x, num_y), args.pad, diff --git a/wkcuber/image_readers.py b/wkcuber/image_readers.py index 4a28b11a2..847892972 100644 --- a/wkcuber/image_readers.py +++ b/wkcuber/image_readers.py @@ -7,13 +7,30 @@ from .vendor.dm3 import DM3 from .vendor.dm4 import DM4File, DM4TagHeader +from tifffile import TiffFile # Disable PIL's maximum image limit. Image.MAX_IMAGE_PIXELS = None -class PillowImageReader: - def read_array(self, file_name: str, dtype: np.dtype) -> np.ndarray: +class ImageReader: + def read_array(self, file_name: str, dtype: np.dtype, z_slice: int) -> np.ndarray: + pass + + def read_dimensions(self, file_name: str) -> Tuple[int, int]: + pass + + def read_channel_count(self, file_name: str) -> int: + pass + + def read_z_slices_per_file( + self, file_name: str # pylint: disable=unused-argument + ) -> int: + return 1 + + +class PillowImageReader(ImageReader): + def read_array(self, file_name: str, dtype: np.dtype, z_slice: int) -> np.ndarray: this_layer = np.array(Image.open(file_name), dtype) this_layer = this_layer.swapaxes(0, 1) this_layer = this_layer.reshape(this_layer.shape + (1,)) @@ -38,8 +55,8 @@ def to_target_datatype(data: np.ndarray, target_dtype: np.dtype) -> np.ndarray: return (data / factor).astype(target_dtype) -class Dm3ImageReader: - def read_array(self, file_name: str, dtype: np.dtype) -> np.ndarray: +class Dm3ImageReader(ImageReader): + def read_array(self, file_name: str, dtype: np.dtype, z_slice: int) -> np.ndarray: dm3_file = DM3(file_name) this_layer = to_target_datatype(dm3_file.imagedata, dtype) this_layer = this_layer.swapaxes(0, 1) @@ -55,7 +72,7 @@ def read_channel_count(self, _file_name: str) -> int: return 1 -class Dm4ImageReader: +class Dm4ImageReader(ImageReader): def _read_tags(self, dm4file: DM4File) -> Tuple[DM4File.DM4TagDir, DM4TagHeader]: tags = dm4file.read_directory() image_data_tag = ( @@ -78,7 +95,7 @@ def _read_dimensions( ) return width, height - def read_array(self, file_name: str, dtype: np.dtype) -> np.ndarray: + def read_array(self, file_name: str, dtype: np.dtype, z_slice: int) -> np.ndarray: dm4file = DM4File.open(file_name) image_data_tag, image_tag = self._read_tags(dm4file) width, height = self._read_dimensions(dm4file, image_data_tag) @@ -94,7 +111,6 @@ def read_array(self, file_name: str, dtype: np.dtype) -> np.ndarray: return data def read_dimensions(self, file_name: str) -> Tuple[int, int]: - dm4file = DM4File.open(file_name) image_data_tag, _ = self._read_tags(dm4file) dimensions = self._read_dimensions(dm4file, image_data_tag) @@ -107,13 +123,71 @@ def read_channel_count(self, _file_name: str) -> int: return 1 -class ImageReader: +def find_count_of_axis(tif_file: TiffFile, axis: str) -> int: + assert len(tif_file.series) == 1, "only single tif series are supported" + tif_series = tif_file.series[0] + index = tif_series.axes.find(axis) + if index == -1: + return 1 + else: + return tif_series.shape[index] # pylint: disable=unsubscriptable-object + + +class TiffImageReader(ImageReader): + def read_array(self, file_name: str, dtype: np.dtype, z_slice: int) -> np.ndarray: + with TiffFile(file_name) as tif_file: + num_channels = self.read_channel_count(file_name) + if len(tif_file.pages) > num_channels: + data = np.array( + list( + map( + lambda x: x.asarray(), + tif_file.pages[ + z_slice * num_channels : z_slice * num_channels + + num_channels + ], + ) + ), + dtype, + ) + else: + data = np.array( + list(map(lambda x: x.asarray(), tif_file.pages[0:num_channels])), + dtype, + ) + # transpose data to shape(x, y, channel_count) + data = np.transpose( + data, + ( + tif_file.pages[0].axes.find("X") + 1, + tif_file.pages[0].axes.find("Y") + 1, + 0, + ), + ) + data = data.reshape(data.shape + (1,)) + return data + + def read_dimensions(self, file_name: str) -> Tuple[int, int]: + with TiffFile(file_name) as tif_file: + return find_count_of_axis(tif_file, "X"), find_count_of_axis(tif_file, "Y") + + def read_channel_count(self, file_name: str) -> int: + with TiffFile(file_name) as tif_file: + return find_count_of_axis(tif_file, "C") + + def read_z_slices_per_file(self, file_name: str) -> int: + with TiffFile(file_name) as tif_file: + return find_count_of_axis(tif_file, "Z") + + +class ImageReaderManager: def __init__(self) -> None: self.readers: Dict[ - str, Union[PillowImageReader, Dm3ImageReader, Dm4ImageReader] + str, + Union[TiffImageReader, PillowImageReader, Dm3ImageReader, Dm4ImageReader], ] = { - ".tif": PillowImageReader(), - ".tiff": PillowImageReader(), + ".tif": TiffImageReader(), + ".tiff": TiffImageReader(), ".jpg": PillowImageReader(), ".jpeg": PillowImageReader(), ".png": PillowImageReader(), @@ -121,11 +195,11 @@ def __init__(self) -> None: ".dm4": Dm4ImageReader(), } - def read_array(self, file_name: str, dtype: np.dtype) -> np.ndarray: + def read_array(self, file_name: str, dtype: np.dtype, z_slice: int) -> np.ndarray: _, ext = path.splitext(file_name) # Image shape will be (x, y, channel_count, z=1) or (x, y, z=1) - image = self.readers[ext].read_array(file_name, dtype) + image = self.readers[ext].read_array(file_name, dtype, z_slice) # Standardize the image shape to (x, y, channel_count, z=1) if image.ndim == 3: image = image.reshape(image.shape + (1,)) @@ -140,5 +214,9 @@ def read_channel_count(self, file_name: str) -> int: _, ext = path.splitext(file_name) return self.readers[ext].read_channel_count(file_name) + def read_z_slices_per_file(self, file_name: str) -> int: + _, ext = path.splitext(file_name) + return self.readers[ext].read_z_slices_per_file(file_name) + -image_reader = ImageReader() +image_reader = ImageReaderManager() diff --git a/wkcuber/tile_cubing.py b/wkcuber/tile_cubing.py index 165699daf..a64dabb90 100644 --- a/wkcuber/tile_cubing.py +++ b/wkcuber/tile_cubing.py @@ -223,7 +223,7 @@ def tile_cubing_job( if file_name: # read the image image = read_image_file( - file_name, target_wkw_info.header.voxel_type + file_name, target_wkw_info.header.voxel_type, z ) slices.append(image) else: