tensorstore (#1174)

* migrate changes from branch tensorstore2 * correct decorator @lru_cache usage * correct parameter of TensorStoreArray::read * debug TensorStoreArray::open * update poetry.lock * format, pytest argument -s and skip failing test * install docker-compose in nightly.yml * fix import for python<3.11 * format * refreshed snapshots * progress * re-add displayname * progress * wip * progress * progress * progress * progress * newer wk * try ubicloud * http_proxy for tensorstore * refresh snapshots * refresh snapshots * rm ZarritaArray and ZarrArray + changelog * cassettes * --add-snapshots * tests/client/test_context.py::test_user_organization * Apply suggestions from code review Co-authored-by: Mark Bader <[email protected]> * bump wk * Apply suggestions from code review Co-authored-by: Mark Bader <[email protected]> * revert test_from_images * Remove test_examples.py * Fix typing issue. * wip: pr feedback * types and fixes * rm zarrita in test_layer * update cassettes * format * update cassettes * typing_extensions * fix test_dataset_download_upload_remote.py::test_remote_dataset * fix docs/uv.lock * format * use current wk version in nightly * LazyPath * fix test_annotation * Update .github/workflows/ci.yml --------- Co-authored-by: Norman Rzepka <[email protected]> Co-authored-by: Mark Bader <[email protected]>
scalableminds · Dec 17, 2024 · bbcf756 · bbcf756
1 parent 3744f1f
commit bbcf756
Show file tree

Hide file tree

Showing 75 changed files with 7,837 additions and 77,988 deletions.
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -34,6 +34,10 @@ jobs:
         git diff --no-ext-diff --quiet --exit-code
         [[ -z $(git status -s) ]]
 
+    - name: Use currently deployed Webknossos version
+      run: |
+        sed -i "s/export DOCKER_TAG=.*$/export DOCKER_TAG=master__$(curl https://webknossos.org/api/buildinfo | jq -r .webknossos.version)/" local_wk_setup.sh
+
     - name: Python tests, refreshing the network snapshots
       env:
         WK_TOKEN: ${{ secrets.WK_TOKEN }}

diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -234,7 +234,6 @@ nav:
         - webknossos-py/examples/download_image_data.md
         - webknossos-py/examples/download_tiff_stack.md
         - webknossos-py/examples/remote_datasets.md
-        - webknossos-py/examples/zarr_and_dask.md
         - webknossos-py/examples/convert_4d_tiff.md
         - webknossos-py/examples/announce_dataset_upload.md
         - webknossos-py/examples/accessing_metadata.md

diff --git a/docs/src/webknossos-py/examples/zarr_and_dask.md b/docs/src/webknossos-py/examples/zarr_and_dask.md
diff --git a/docs/uv.lock b/docs/uv.lock
diff --git a/webknossos/Changelog.md b/webknossos/Changelog.md
@@ -13,10 +13,12 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 [Commits](https://github.com/scalableminds/webknossos-libs/compare/v0.16.1...HEAD)
 
 ### Breaking Changes
+- `MagView.get_zarr_array` now returns a `tensorstore` array instead of a `zarr-python` array. [#1174](https://github.com/scalableminds/webknossos-libs/pull/1174) 
 
 ### Added
 
 ### Changed
+- Using tensorstore for reading and writing zarr 2 and 3 arrays. Removed `zarrita` and `zarr` dependency. [#1174](https://github.com/scalableminds/webknossos-libs/pull/1174) 
 
 ### Fixed
 

diff --git a/webknossos/examples/apply_merger_mode.py b/webknossos/examples/apply_merger_mode.py
@@ -1,4 +1,3 @@
-from os import environ
 from pathlib import Path
 from typing import Tuple, cast
 
@@ -84,19 +83,18 @@ def apply_mapping_for_chunk(args: Tuple[wk.View, wk.View, int]) -> None:
     # Optionally, downsample and re-upload #
     ########################################
 
-    if "PYTEST_CURRENT_TEST" not in environ:
-        out_layer.downsample()
-        dataset.delete_layer("segmentation")
-        dataset.upload(
-            "l4_sample_remapped",
-            layers_to_link=[
-                wk.LayerToLink(
-                    dataset_name="l4_sample",
-                    layer_name="color",
-                    organization_id="scalable_minds",
-                )
-            ],
-        )
+    out_layer.downsample()
+    dataset.delete_layer("segmentation")
+    dataset.upload(
+        "l4_sample_remapped",
+        layers_to_link=[
+            wk.LayerToLink(
+                dataset_name="l4_sample",
+                layer_name="color",
+                organization_id="scalable_minds",
+            )
+        ],
+    )
 
 
 if __name__ == "__main__":

diff --git a/webknossos/examples/zarr_and_dask.py b/webknossos/examples/zarr_and_dask.py
diff --git a/webknossos/local_wk_setup.sh b/webknossos/local_wk_setup.sh
@@ -1,7 +1,7 @@
 function export_vars {
     export WK_TOKEN=1b88db86331a38c21a0b235794b9e459856490d70408bcffb767f64ade0f83d2bdb4c4e181b9a9a30cdece7cb7c65208cc43b6c1bb5987f5ece00d348b1a905502a266f8fc64f0371cd6559393d72e031d0c2d0cabad58cccf957bb258bc86f05b5dc3d4fff3d5e3d9c0389a6027d861a21e78e3222fb6c5b7944520ef21761e
     export WK_URL=http://localhost:9000
-    export DOCKER_TAG=master__30776
+    export DOCKER_TAG=master__31116
 }
 
 function ensure_local_test_wk {

diff --git a/webknossos/pyproject.toml b/webknossos/pyproject.toml
@@ -38,25 +38,22 @@ dependencies = [
     "cattrs >=22.0.0",
     "certifi>=2023",
     "cluster-tools",
-    "fsspec ~=2024.6.0",
     "httpx ~=0.27.0",
     "loxun ~=2.0.0",
     "natsort ~=8.4.0",
     "networkx ~=3.2.1",
-    "numcodecs ~=0.12",
     "numpy ~=2.0.0",
     "pims ~=0.7.0",
     "psutil ~=6.0.0",
     "python-dateutil ~=2.8.0",
     "python-dotenv ~=1.0.1",
     "rich ~=13.8.0",
     "scipy ~=1.13",
+    "tensorstore~=0.1.69",
     "typer ~=0.12.5",
     "typing-extensions ~=4.0",
     "universal-pathlib ~=0.2",
     "wkw ==1.1.24",
-    "zarr ~=2.18.0",
-    "zarrita ==0.2.7",
     "zipp ~=3.5.0",
 ]
 
@@ -91,7 +88,7 @@ all = [
 [tool.uv]
 extra-index-url = ["https://pypi.scm.io/simple"]
 dev-dependencies = [
-    "dask[distributed] ~=2023.9.1; python_version >='3.9'",
+    "fsspec ~=2024.6.0",
     "hypothesis ~=6.35.0",
     "icecream ~=2.1.1",
     "inducoapi ~=2.0.2",
@@ -160,7 +157,7 @@ target-version = "py39"
 # PERF = Perflint https://docs.astral.sh/ruff/rules/#perflint-perf
 # FIX = Flake8 fixme https://docs.astral.sh/ruff/rules/#flake8-fixme-fix
 # ARG = Flake8 unused-arguments https://docs.astral.sh/ruff/rules/#flake8-unused-arguments-arg
-select = ["F", "E", "W", "I", "A", "PERF", "FIX", "ARG"]
+select = ["F", "E", "W", "I", "A", "PERF", "FIX", "ARG", "T201"]
 
 # Allow fix for all enabled rules (when `--fix`) is provided.
 fixable = ["ALL"]
@@ -169,6 +166,9 @@ fixable = ["ALL"]
 # E712 true-false-comparison https://docs.astral.sh/ruff/rules/true-false-comparison/
 ignore = ["E501", "E712"]
 
+[tool.ruff.lint.per-file-ignores]
+"{examples,tests,script_collection,webknossos/cli}/*" = ["T201"]
+
 [tool.ruff.format]
 # Like Black, use double quotes for strings.
 quote-style = "double"

diff --git a/webknossos/test.sh b/webknossos/test.sh
@@ -31,6 +31,21 @@ if [ $# -gt 0 ] && [ "$1" = "--refresh-snapshots" ]; then
 
     stop_local_test_wk
 
+    exit $PYTEST_EXIT_CODE
+elif [ $# -gt 0 ] && [ "$1" = "--add-snapshots" ]; then
+    ensure_local_test_wk
+
+    # Starts a proxy server in record mode on port 3000 and sets the HTTP_PROXY env var
+    proxay --mode record --host http://localhost:9000 --tapes-dir tests/cassettes &
+    PROXAY_PID=$!
+
+    shift
+    $PYTEST "-m" "use_proxay" "$@"
+    PYTEST_EXIT_CODE=$?
+    trap 'kill $PROXAY_PID' EXIT
+
+    stop_local_test_wk
+
     exit $PYTEST_EXIT_CODE
 elif [ $# -gt 0 ] && [ "$1" = "--debug-cassettes" ]; then
     # This will start a proxay server in replay mode so that the stored cassettes can be used for debugging tests.

diff --git a/webknossos/testdata/annotations/l4_sample__explorational__suser__94b271.zip b/webknossos/testdata/annotations/l4_sample__explorational__suser__94b271.zip
diff --git a/webknossos/testdata/simple_zarr3_dataset/color/1/zarr.json b/webknossos/testdata/simple_zarr3_dataset/color/1/zarr.json
@@ -16,17 +16,17 @@
       "configuration": {
         "chunk_shape": [3, 16, 16, 16],
         "codecs": [
-          { "name": "endian", "configuration": { "endian": "little" } }
+          { "name": "bytes", "configuration": { "endian": "little" } }
         ],
         "index_codecs": [
-          { "name": "endian", "configuration": { "endian": "little" } },
+          { "name": "bytes", "configuration": { "endian": "little" } },
           { "name": "crc32c" }
         ]
       },
       "name": "sharding_indexed"
     }
   ],
-  "dimension_names": null,
+  "dimension_names": ["c", "x", "y", "z"],
   "zarr_format": 3,
   "node_type": "array"
 }