diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50de01b11..1c64ad6e6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.6, 3.7] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v1 @@ -30,13 +30,13 @@ jobs: run: tar -xzvf testdata/WT1_wkw.tar.gz - name: Check formatting - run: black --check . + run: poetry run black --check . - name: Lint code - run: python -m pylint -j4 wkcuber + run: poetry run pylint -j4 wkcuber - name: Python tests - run: py.test tests + run: poetry run pytest tests - name: Smoke test docker run: | diff --git a/README.md b/README.md index d05b42520..a86a32e9c 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ -# webKnossos cuber +# webKnossos cuber (wkcuber) +[![PyPI version](https://img.shields.io/pypi/v/wkcuber)](https://pypi.python.org/pypi/wkcuber) +[![Supported Python Versions](https://img.shields.io/pypi/pyversions/wkcuber.svg)](https://pypi.python.org/pypi/wkcuber) +[![Build Status](https://img.shields.io/github/workflow/status/scalableminds/wkcuber/CI/master)](https://github.com/scalableminds/wkcuber/actions?query=workflow%3A%22CI%22) +[![Code Style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![CircleCI Status](https://circleci.com/gh/scalableminds/webknossos-cuber.svg?&style=shield)](https://circleci.com/gh/scalableminds/webknossos-cuber) - -Easily create [WKW](https://github.com/scalableminds/webknossos-wrap) datasets for [webKnossos](https://webknossos.org). +Python library for creating and working with [webKnossos](https://webknossos.org) [WKW](https://github.com/scalableminds/webknossos-wrap) datasets. WKW is a container format for efficiently storing large, scale 3D image data as found in (electron) microscopy. The tools are modular components to allow easy integration into existing pipelines and workflows. -Created with [Python3](https://www.python.org/). - ## Features * `wkcuber`: Convert image stacks to fully ready WKW datasets (includes downsampling, compressing and metadata generation) @@ -32,7 +32,9 @@ Created with [Python3](https://www.python.org/). * NIFTI files ## Installation -### Python3 with pip +### Python 3 with pip from PyPi +- `wkcuber` requires at least Python 3.6+ + ``` # Make sure to have lz4 installed: # Mac: brew install lz4 @@ -99,11 +101,32 @@ python -m wkcuber.check_equality /data/source /data/target Most tasks can be configured to be executed in a parallelized manner. Via `--distribution_strategy` you can pass `multiprocessing` or `slurm`. The first can be further configured with `--jobs` and the latter via `--job_resources='{"mem": "10M"}'`. Use `--help` to get more information. -## Test data credits +## Development +Make sure to install all the required dependencies using Poetry: +``` +pip install poetry +poetry install +``` + +Please, format, lint, and unit test your code changes before merging them. +``` +poetry run black . +poetry run pylint -j4 wkcuber +poetry run pytest tests +``` + +Please, run the extended test suite: +``` +tests/scripts/all_tests.sh +``` + +PyPi releases are automatically pushed when creating a new Git tag/Github release. + +## Test Data Credits Excerpts for testing purposes have been sampled from: - Dow Jacobo Hossain Siletti Hudspeth (2018). **Connectomics of the zebrafish's lateral-line neuromast reveals wiring and miswiring in a simple microcircuit.** eLife. [DOI:10.7554/eLife.33988](https://elifesciences.org/articles/33988) - Zheng Lauritzen Perlman Robinson Nichols Milkie Torrens Price Fisher Sharifi Calle-Schuler Kmecova Ali Karsh Trautman Bogovic Hanslovsky Jefferis Kazhdan Khairy Saalfeld Fetter Bock (2018). **A Complete Electron Microscopy Volume of the Brain of Adult Drosophila melanogaster.** Cell. [DOI:10.1016/j.cell.2018.06.019](https://www.cell.com/cell/fulltext/S0092-8674(18)30787-6). License: [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/) ## License -AGPLv3 +AGPLv3 Copyright scalable minds diff --git a/poetry.lock b/poetry.lock index 68ca20ad9..c886122b0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,17 +8,17 @@ python-versions = "*" [[package]] name = "astroid" -version = "2.3.3" +version = "2.4.2" description = "An abstract syntax tree for Python with inference support." category = "dev" optional = false -python-versions = ">=3.5.*" +python-versions = ">=3.5" [package.dependencies] lazy-object-proxy = ">=1.4.0,<1.5.0" six = ">=1.12,<2.0" typed-ast = {version = ">=1.4.0,<1.5", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} -wrapt = ">=1.11.0,<1.12.0" +wrapt = ">=1.11,<2.0" [[package]] name = "atomicwrites" @@ -44,7 +44,7 @@ tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.i [[package]] name = "black" -version = "19.3b0" +version = "20.8b1" description = "The uncompromising code formatter." category = "dev" optional = false @@ -52,11 +52,17 @@ python-versions = ">=3.6" [package.dependencies] appdirs = "*" -attrs = ">=18.1.0" -click = ">=6.5" -toml = ">=0.9.4" +click = ">=7.1.2" +dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} +mypy-extensions = ">=0.4.3" +pathspec = ">=0.6,<1" +regex = ">=2020.1.8" +toml = ">=0.10.1" +typed-ast = ">=1.4.0" +typing-extensions = ">=3.7.4" [package.extras] +colorama = ["colorama (>=0.4.3)"] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] @@ -88,11 +94,11 @@ python-versions = "*" [[package]] name = "click" -version = "7.0" +version = "7.1.2" description = "Composable command line interface toolkit" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "cloudpickle" @@ -132,6 +138,14 @@ python-versions = "*" [package.dependencies] six = "*" +[[package]] +name = "dataclasses" +version = "0.8" +description = "A backport of the dataclasses module for Python 3.6" +category = "dev" +optional = false +python-versions = ">=3.6, <3.7" + [[package]] name = "decorator" version = "4.4.1" @@ -182,6 +196,14 @@ zipp = ">=0.5" docs = ["sphinx", "rst.linker"] testing = ["packaging", "importlib-resources"] +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "isort" version = "4.3.21" @@ -243,6 +265,14 @@ category = "dev" optional = false python-versions = ">=3.5" +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "natsort" version = "6.2.0" @@ -317,6 +347,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" pyparsing = ">=2.0.2" six = "*" +[[package]] +name = "pathspec" +version = "0.8.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "pillow" version = "6.2.1" @@ -352,7 +390,7 @@ enum = ["enum34"] [[package]] name = "py" -version = "1.8.0" +version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "dev" optional = false @@ -368,17 +406,18 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pylint" -version = "2.3.1" +version = "2.6.0" description = "python code static checker" category = "dev" optional = false -python-versions = ">=3.4.*" +python-versions = ">=3.5.*" [package.dependencies] -astroid = ">=2.2.0,<3" +astroid = ">=2.4.0,<=2.5" colorama = {version = "*", markers = "sys_platform == \"win32\""} -isort = ">=4.2.5,<5" +isort = ">=4.2.5,<6" mccabe = ">=0.6,<0.7" +toml = ">=0.7.1" [[package]] name = "pyparsing" @@ -390,22 +429,22 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "5.3.2" +version = "6.2.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} -more-itertools = ">=4.0.0" +iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" -py = ">=1.5.0" -wcwidth = "*" +pluggy = ">=0.12,<1.0.0a1" +py = ">=1.8.2" +toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] @@ -432,6 +471,14 @@ python-versions = ">=3.5" [package.dependencies] numpy = ">=1.13.3" +[[package]] +name = "regex" +version = "2020.11.13" +description = "Alternative regular expression module, to replace re." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "requests" version = "2.22.0" @@ -448,7 +495,7 @@ urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" [package.extras] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "scikit-image" @@ -467,7 +514,7 @@ PyWavelets = ">=0.4.0" scipy = ">=0.19.0" [package.extras] -docs = ["sphinx (>=1.3,<1.7.8 || >1.7.8)", "numpydoc (>=0.9)", "sphinx-gallery", "sphinx-copybutton", "pytest-runner", "scikit-learn", "matplotlib (>=3.0.1)", "dask[array] (>=0.15.0)", "cloudpickle (>=0.2.1)"] +docs = ["sphinx (>=1.3,!=1.7.8)", "numpydoc (>=0.9)", "sphinx-gallery", "sphinx-copybutton", "pytest-runner", "scikit-learn", "matplotlib (>=3.0.1)", "dask[array] (>=0.15.0)", "cloudpickle (>=0.2.1)"] optional = ["simpleitk", "astropy (>=1.2.0)", "tifffile", "qtpy", "pyamg", "dask[array] (>=0.15.0)", "cloudpickle (>=0.2.1)"] test = ["pytest (!=3.7.3)", "pytest-cov", "pytest-localserver", "flake8", "codecov"] @@ -500,11 +547,11 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*" [[package]] name = "toml" -version = "0.10.0" +version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" optional = false -python-versions = "*" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typed-ast" @@ -514,6 +561,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "typing-extensions" +version = "3.7.4.3" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "urllib3" version = "1.25.7" @@ -525,15 +580,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" [package.extras] brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] - -[[package]] -name = "wcwidth" -version = "0.1.7" -description = "Measures number of Terminal column cells of wide-character codes" -category = "dev" -optional = false -python-versions = "*" +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wkw" @@ -573,7 +620,7 @@ testing = ["pathlib2", "contextlib2", "unittest2"] [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "51aef09d78b5ea3f7d17045a3b638f4eea71693ecbf51418b9328fdf5051810d" +content-hash = "85bcb808e67144f1a8b9984980cfe6a009e40bc2b8d743f55288ba638e347c00" [metadata.files] appdirs = [ @@ -581,8 +628,8 @@ appdirs = [ {file = "appdirs-1.4.3.tar.gz", hash = "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92"}, ] astroid = [ - {file = "astroid-2.3.3-py3-none-any.whl", hash = "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42"}, - {file = "astroid-2.3.3.tar.gz", hash = "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a"}, + {file = "astroid-2.4.2-py3-none-any.whl", hash = "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386"}, + {file = "astroid-2.4.2.tar.gz", hash = "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703"}, ] atomicwrites = [ {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, @@ -593,8 +640,7 @@ attrs = [ {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, ] black = [ - {file = "black-19.3b0-py36-none-any.whl", hash = "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf"}, - {file = "black-19.3b0.tar.gz", hash = "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c"}, + {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] certifi = [ {file = "certifi-2019.11.28-py2.py3-none-any.whl", hash = "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"}, @@ -640,8 +686,8 @@ chardet = [ {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] click = [ - {file = "Click-7.0-py2.py3-none-any.whl", hash = "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"}, - {file = "Click-7.0.tar.gz", hash = "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"}, + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] cloudpickle = [ {file = "cloudpickle-1.2.2-py2.py3-none-any.whl", hash = "sha256:f3ef2c9d438f1553ce7795afb18c1f190d8146132496169ef6aa9b7b65caa4c3"}, @@ -658,6 +704,10 @@ cycler = [ {file = "cycler-0.10.0-py2.py3-none-any.whl", hash = "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d"}, {file = "cycler-0.10.0.tar.gz", hash = "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"}, ] +dataclasses = [ + {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, + {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, +] decorator = [ {file = "decorator-4.4.1-py2.py3-none-any.whl", hash = "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"}, {file = "decorator-4.4.1.tar.gz", hash = "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce"}, @@ -674,6 +724,10 @@ importlib-metadata = [ {file = "importlib_metadata-1.3.0-py2.py3-none-any.whl", hash = "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"}, {file = "importlib_metadata-1.3.0.tar.gz", hash = "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45"}, ] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] isort = [ {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"}, {file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"}, @@ -763,6 +817,10 @@ more-itertools = [ {file = "more-itertools-8.0.2.tar.gz", hash = "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d"}, {file = "more_itertools-8.0.2-py3-none-any.whl", hash = "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"}, ] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] natsort = [ {file = "natsort-6.2.0-py2.py3-none-any.whl", hash = "sha256:4f0de45639f1fa43dede43ae1919bcab66e0a6fc19b68ea24f0a250814b4f176"}, {file = "natsort-6.2.0.tar.gz", hash = "sha256:58c6fb2f355117e88a19808394ec1ed30a2ff881bdd2c81c436952caebd30668"}, @@ -803,6 +861,10 @@ packaging = [ {file = "packaging-19.2-py2.py3-none-any.whl", hash = "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"}, {file = "packaging-19.2.tar.gz", hash = "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47"}, ] +pathspec = [ + {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, + {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, +] pillow = [ {file = "Pillow-6.2.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9"}, {file = "Pillow-6.2.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab"}, @@ -853,23 +915,23 @@ psutil = [ {file = "psutil-5.6.7.tar.gz", hash = "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"}, ] py = [ - {file = "py-1.8.0-py2.py3-none-any.whl", hash = "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa"}, - {file = "py-1.8.0.tar.gz", hash = "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pycparser = [ {file = "pycparser-2.19.tar.gz", hash = "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"}, ] pylint = [ - {file = "pylint-2.3.1-py3-none-any.whl", hash = "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09"}, - {file = "pylint-2.3.1.tar.gz", hash = "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1"}, + {file = "pylint-2.6.0-py3-none-any.whl", hash = "sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f"}, + {file = "pylint-2.6.0.tar.gz", hash = "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210"}, ] pyparsing = [ {file = "pyparsing-2.4.5-py2.py3-none-any.whl", hash = "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f"}, {file = "pyparsing-2.4.5.tar.gz", hash = "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"}, ] pytest = [ - {file = "pytest-5.3.2-py3-none-any.whl", hash = "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4"}, - {file = "pytest-5.3.2.tar.gz", hash = "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa"}, + {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, + {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, ] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, @@ -898,6 +960,49 @@ pywavelets = [ {file = "PyWavelets-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:935ff247b8b78bdf77647fee962b1cc208c51a7b229db30b9ba5f6da3e675178"}, {file = "PyWavelets-1.1.1.tar.gz", hash = "sha256:1a64b40f6acb4ffbaccce0545d7fc641744f95351f62e4c6aaa40549326008c9"}, ] +regex = [ + {file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"}, + {file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"}, + {file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"}, + {file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"}, + {file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"}, + {file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"}, + {file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"}, + {file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"}, + {file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"}, + {file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"}, + {file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"}, + {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"}, + {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"}, +] requests = [ {file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"}, {file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"}, @@ -957,9 +1062,8 @@ six = [ {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, ] toml = [ - {file = "toml-0.10.0-py2.7.egg", hash = "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3"}, - {file = "toml-0.10.0-py2.py3-none-any.whl", hash = "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"}, - {file = "toml-0.10.0.tar.gz", hash = "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c"}, + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] typed-ast = [ {file = "typed_ast-1.4.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e"}, @@ -983,14 +1087,15 @@ typed-ast = [ {file = "typed_ast-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e"}, {file = "typed_ast-1.4.0.tar.gz", hash = "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34"}, ] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] urllib3 = [ {file = "urllib3-1.25.7-py2.py3-none-any.whl", hash = "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293"}, {file = "urllib3-1.25.7.tar.gz", hash = "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"}, ] -wcwidth = [ - {file = "wcwidth-0.1.7-py2.py3-none-any.whl", hash = "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"}, - {file = "wcwidth-0.1.7.tar.gz", hash = "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e"}, -] wkw = [ {file = "wkw-0.1.4-py3-none-manylinux1_x86_64.whl", hash = "sha256:464a32de456c7999a029b50b12e35e2a117f85ddc3d02eac4ea203e2cbc70a13"}, {file = "wkw-0.1.4-py3-none-manylinux2010_x86_64.whl", hash = "sha256:0f03145f4b088ca607c022315d4cb0ee2e17e69d40c4ee74bd8d29f1e0a9e0eb"}, diff --git a/pyproject.toml b/pyproject.toml index 3f2cdf847..44032a82d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,9 @@ [tool.poetry] name = "wkcuber" version = "0.0.0" # filled by setuptools-scm -description = "A cubing tool for webKnossos" +description = "Python package to create, cube, and work with webKnossos WKW datasets" authors = ["scalable minds "] +readme = "README.md" license = "AGPL-3.0" [tool.poetry.dependencies] @@ -19,9 +20,9 @@ nibabel = "^2.5.1" scikit-image = "^0.16.2" [tool.poetry.dev-dependencies] -pylint = "2.3.1" -black = "19.3b0" -pytest = "^5.3.2" +pylint = "^2.6.0" +black = "^20.8b1" +pytest = "^6.2.1" setuptools-scm = "^3.3.3" [build-system] diff --git a/tests/scripts/all_tests.sh b/tests/scripts/all_tests.sh new file mode 100755 index 000000000..96c77d0c7 --- /dev/null +++ b/tests/scripts/all_tests.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eEuo pipefail + +BASEDIR=./tests/scripts + +sh ${BASEDIR}/tiff_cubing.sh +sh ${BASEDIR}/simple_tiff_cubing.sh +sh ${BASEDIR}/simple_tiff_cubing_no_compression.sh +sh ${BASEDIR}/tile_cubing.sh +sh ${BASEDIR}/meta_generation.sh +sh ${BASEDIR}/knossos_conversion.sh + +mkdir -p testdata/tiff_mag_2_reference +tar -xzvf testdata/tiff_mag_2_reference.tar.gz -C testdata/tiff_mag_2_reference + +sh ${BASEDIR}/anisotropic_downsampling.sh +sh ${BASEDIR}/compression_and_verification.sh +sh ${BASEDIR}/downsampling.sh +sh ${BASEDIR}/in_memory_downsampled_cubing.sh +sh ${BASEDIR}/in_place_compression.sh +sh ${BASEDIR}/simple_anisotropic_tiff_cubing.sh + +rm -r testdata/tiff_mag_2_reference/ \ No newline at end of file diff --git a/wkcuber/api/bounding_box.py b/wkcuber/api/bounding_box.py index bfebb00ff..69bcc40e1 100644 --- a/wkcuber/api/bounding_box.py +++ b/wkcuber/api/bounding_box.py @@ -63,8 +63,7 @@ def from_named_tuple(bb_named_tuple: BoundingBoxNamedTuple): @staticmethod def from_checkpoint_name(checkpoint_name: str) -> "BoundingBox": - """ This function extracts a bounding box in the format x_y_z_sx_sy_xz which is contained in a string. - """ + """This function extracts a bounding box in the format x_y_z_sx_sy_xz which is contained in a string.""" regex = r"(([0-9]+_){5}([0-9]+))" match = re.search(regex, checkpoint_name) assert ( @@ -243,10 +242,10 @@ def chunk( ) -> Generator["BoundingBox", None, None]: """Decompose the bounding box into smaller chunks of size `chunk_size`. - Chunks at the border of the bounding box might be smaller than chunk_size. - If `chunk_border_alignment` is set, all border coordinates - *between two chunks* will be divisible by that value. - """ + Chunks at the border of the bounding box might be smaller than chunk_size. + If `chunk_border_alignment` is set, all border coordinates + *between two chunks* will be divisible by that value. + """ start = self.topleft.copy() chunk_size = np.array(chunk_size) diff --git a/wkcuber/convert_nifti.py b/wkcuber/convert_nifti.py index c07c15c82..a8d3af392 100644 --- a/wkcuber/convert_nifti.py +++ b/wkcuber/convert_nifti.py @@ -137,9 +137,7 @@ def convert_nifti( cube_data = np.array(source_nifti.get_fdata()) is_probably_binary = np.unique(cube_data).shape[0] == 2 - assume_segmentation_layer = ( - False - ) # Since webKnossos does not support multiple segmention layers, this is hardcoded to False right now. + assume_segmentation_layer = False # Since webKnossos does not support multiple segmention layers, this is hardcoded to False right now. max_cell_id_args = ( {"largest_segment_id": int(np.max(cube_data) + 1)} if assume_segmentation_layer diff --git a/wkcuber/downsampling.py b/wkcuber/downsampling.py index 2a799e869..769f5e63b 100644 --- a/wkcuber/downsampling.py +++ b/wkcuber/downsampling.py @@ -541,10 +541,11 @@ def downsample_mags( scale = read_datasource_properties(path)["scale"] except Exception as exc: logging.error( - f"Could not get the scale from the datasource-properties.json. Probably your path is wrong. " + "Could not get the scale from the datasource-properties.json. Probably your path is wrong. " "If you do not provide the layer_name or from_mag, they need to be included in the path." "(e.g. dataset/color/1). Otherwise the path should just point at the dataset directory." - "the path: {path}" + "the path: %s", + path, ) raise exc downsample_mags_anisotropic( diff --git a/wkcuber/export_wkw_as_tiff.py b/wkcuber/export_wkw_as_tiff.py index 2158e112c..3d00c547a 100644 --- a/wkcuber/export_wkw_as_tiff.py +++ b/wkcuber/export_wkw_as_tiff.py @@ -162,7 +162,7 @@ def export_tiff_slice( image = wkw_slice_to_image(tiff_data[:, :, :, slice_index], downsample) image.save(tiff_file_path) - logging.info(f"saved slice {slice_name_number}") + logging.info("Saved slice %s", slice_name_number) else: for y_tile_index in range(ceil(tiff_bbox["size"][1] / tiling_size[1])): @@ -216,7 +216,7 @@ def export_tiff_stack( num_slices = ceil(bbox["size"][2] / batch_size) slices = range(0, num_slices) - logging.info(f"starting jobs") + logging.info("starting jobs") futures = executor.map_to_futures( partial( export_tiff_slice, diff --git a/wkcuber/tile_cubing.py b/wkcuber/tile_cubing.py index 81519f8fa..54470cce0 100644 --- a/wkcuber/tile_cubing.py +++ b/wkcuber/tile_cubing.py @@ -42,9 +42,9 @@ def check_input_pattern(input_pattern: str) -> str: def replace_coordinates( pattern: str, coord_ids_with_replacement_info: Dict[str, Tuple[int, int]] ) -> str: - """ Replaces the coordinates with a specific length. - The coord_ids_with_replacement_info is a Dict that maps a dimension - to a tuple of the coordinate value and the desired length. """ + """Replaces the coordinates with a specific length. + The coord_ids_with_replacement_info is a Dict that maps a dimension + to a tuple of the coordinate value and the desired length.""" occurrences = re.findall("({x+}|{y+}|{z+})", pattern) for occurrence in occurrences: coord = occurrence[1] diff --git a/wkcuber/utils.py b/wkcuber/utils.py index 760515538..d8eedfd60 100644 --- a/wkcuber/utils.py +++ b/wkcuber/utils.py @@ -1,22 +1,24 @@ import re import time -import wkw -import numpy as np import logging import argparse +import wkw +import numpy as np import cluster_tools import json import os import psutil +import traceback +import concurrent + from typing import List, Tuple, Union from glob import iglob from collections import namedtuple from multiprocessing import cpu_count -import concurrent from os import path, getpid from math import floor, ceil from logging import getLogger -import traceback + from wkcuber.api.bounding_box import BoundingBox from .knossos import KnossosDataset @@ -68,15 +70,15 @@ def parse_scale(scale): try: scale = tuple(float(x) for x in scale.split(",")) return scale - except Exception: - raise argparse.ArgumentTypeError("The scale could not be parsed") + except Exception as e: + raise argparse.ArgumentTypeError("The scale could not be parsed") from e def parse_bounding_box(bbox_str): try: return BoundingBox.from_csv(bbox_str) - except Exception: - raise argparse.ArgumentTypeError("The bounding box could not be parsed.") + except Exception as e: + raise argparse.ArgumentTypeError("The bounding box could not be parsed.") from e def open_knossos(info): diff --git a/wkcuber/vendor/dm3.py b/wkcuber/vendor/dm3.py index 67c643bb7..3ed0d096f 100644 --- a/wkcuber/vendor/dm3.py +++ b/wkcuber/vendor/dm3.py @@ -887,9 +887,11 @@ def makePNGThumbnail(self, tn_file=""): tn_path = tn_file # - save tn file try: - self.thumbnail.save(tn_path, "PNG") - if self._debug > 0: - print("Thumbnail saved as '%s'." % tn_path) + if hasattr(self, "thumbnail"): + # pylint: disable=no-member + self.thumbnail.save(tn_path, "PNG") + if self._debug > 0: + print("Thumbnail saved as '%s'." % tn_path) except: print("Warning: could not save thumbnail.") diff --git a/wkcuber/vendor/dm4.py b/wkcuber/vendor/dm4.py index ccf61a01b..bf96837ae 100644 --- a/wkcuber/vendor/dm4.py +++ b/wkcuber/vendor/dm4.py @@ -104,7 +104,7 @@ def _get_struct_endian_str(endian): def read_root_tag_dir_header_dm4(dmfile, endian): """Read the root directory information from a dm4 file. - File seek position is left at end of root_tag_dir_header""" + File seek position is left at end of root_tag_dir_header""" if not isinstance(endian, str): endian = _get_struct_endian_str(endian)