diff --git a/_tooling/local_minio_setup.sh b/_tooling/local_minio_setup.sh deleted file mode 100644 index 5809e8e0f..000000000 --- a/_tooling/local_minio_setup.sh +++ /dev/null @@ -1,19 +0,0 @@ -export MINIO_ROOT_USER="TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur" -export MINIO_ROOT_PASSWORD="ANTN35UAENTS5UIAEATD" - -# Minio is an S3 clone and is used as local test server -docker run \ - -p 8000:9000 \ - -e MINIO_ROOT_USER=$MINIO_ROOT_USER \ - -e MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD \ - --name minio \ - --rm \ - -d \ - minio/minio server /data - -stop_minio () { - ARG=$? - docker stop minio - exit $ARG -} -trap stop_minio EXIT diff --git a/webknossos/local_wk_setup.sh b/webknossos/local_wk_setup.sh old mode 100644 new mode 100755 diff --git a/webknossos/test.sh b/webknossos/test.sh index 247889828..8d7d25085 100755 --- a/webknossos/test.sh +++ b/webknossos/test.sh @@ -2,7 +2,6 @@ set -eEuo pipefail source local_wk_setup.sh -source ../_tooling/local_minio_setup.sh export_vars diff --git a/webknossos/tests/test_dataset.py b/webknossos/tests/test_dataset.py index 238977dae..f350d6292 100644 --- a/webknossos/tests/test_dataset.py +++ b/webknossos/tests/test_dataset.py @@ -1,10 +1,11 @@ import itertools import json -import os import pickle +import shlex +import subprocess import warnings from pathlib import Path -from typing import Optional, Tuple, cast +from typing import Iterator, Optional, Tuple, cast import numpy as np import pytest @@ -37,11 +38,39 @@ from .constants import TESTDATA_DIR, TESTOUTPUT_DIR +MINIO_ROOT_USER = "TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur" +MINIO_ROOT_PASSWORD = "ANTN35UAENTS5UIAEATD" +MINIO_PORT = "8000" + + +@pytest.fixture(autouse=True, scope="module") +def docker_minio() -> Iterator[None]: + """Minio is an S3 clone and is used as local test server""" + container_name = "minio" + cmd = ( + "docker run" + f" -p {MINIO_PORT}:9000" + f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}" + f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}" + f" --name {container_name}" + " --rm" + " -d" + " minio/minio server /data" + ) + print("BEFORE", flush=True) + subprocess.check_output(shlex.split(cmd)) + REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True) + try: + yield + finally: + subprocess.check_output(["docker", "stop", container_name]) + + REMOTE_TESTOUTPUT_DIR = UPath( "s3://testoutput", - key=os.environ["MINIO_ROOT_USER"], - secret=os.environ["MINIO_ROOT_PASSWORD"], - client_kwargs={"endpoint_url": "http://localhost:8000"}, + key=MINIO_ROOT_USER, + secret=MINIO_ROOT_PASSWORD, + client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"}, ) DATA_FORMATS = [DataFormat.WKW, DataFormat.Zarr] @@ -192,11 +221,6 @@ def assure_exported_properties(ds: Dataset) -> None: ), "The properties did not match after reopening the dataset. This might indicate that the properties were not exported after they were changed in memory." -@pytest.fixture(scope="session", autouse=True) -def create_bucket() -> None: - REMOTE_TESTOUTPUT_DIR.fs.mkdirs("testoutput", exist_ok=True) - - @pytest.mark.parametrize("data_format,output_path", DATA_FORMATS_AND_OUTPUT_PATHS) def test_create_dataset_with_layer_and_mag( data_format: DataFormat, output_path: Path diff --git a/wkcuber/test.sh b/wkcuber/test.sh index eefd485d3..b0e657634 100755 --- a/wkcuber/test.sh +++ b/wkcuber/test.sh @@ -1,12 +1,6 @@ #!/usr/bin/env bash set -eEuo pipefail -set +u -if [[ -z $RUNNER_OS || "$RUNNER_OS" == "Linux" ]]; then - source ../_tooling/local_minio_setup.sh -fi -set -u - # Note that pytest should be executed via `python -m`, since # this will ensure that the current directory is added to sys.path # (which is standard python behavior). This is necessary so that the imports diff --git a/wkcuber/tests/test_cli.py b/wkcuber/tests/test_cli.py index dd765971d..d482fb465 100644 --- a/wkcuber/tests/test_cli.py +++ b/wkcuber/tests/test_cli.py @@ -3,7 +3,6 @@ import sys from os import environ from pathlib import Path -from shutil import unpack_archive from typing import Union import numpy as np @@ -14,6 +13,8 @@ from webknossos.utils import copytree, rmtree from .constants import TESTDATA_DIR +import subprocess +import shlex def check_call(*args: Union[str, int, Path]) -> None: @@ -28,16 +29,37 @@ def count_wkw_files(mag_path: Path) -> int: return len(list(mag_path.glob("**/x*.wkw"))) -@pytest.fixture(scope="session") +MINIO_ROOT_USER = "TtnuieannGt2rGuie2t8Tt7urarg5nauedRndrur" +MINIO_ROOT_PASSWORD = "ANTN35UAENTS5UIAEATD" +MINIO_PORT = "8000" + + +@pytest.fixture(scope="module") def remote_testoutput_path() -> UPath: + """Minio is an S3 clone and is used as local test server""" + container_name = "minio" + cmd = ( + "docker run" + f" -p {MINIO_PORT}:9000" + f" -e MINIO_ROOT_USER={MINIO_ROOT_USER}" + f" -e MINIO_ROOT_PASSWORD={MINIO_ROOT_PASSWORD}" + f" --name {container_name}" + " --rm" + " -d" + " minio/minio server /data" + ) + subprocess.check_output(shlex.split(cmd)) remote_path = UPath( "s3://testoutput", - key=environ["MINIO_ROOT_USER"], - secret=environ["MINIO_ROOT_PASSWORD"], - client_kwargs={"endpoint_url": "http://localhost:8000"}, + key=MINIO_ROOT_USER, + secret=MINIO_ROOT_PASSWORD, + client_kwargs={"endpoint_url": f"http://localhost:{MINIO_PORT}"}, ) remote_path.fs.mkdirs("testoutput", exist_ok=True) - return remote_path + try: + yield remote_path + finally: + subprocess.check_output(["docker", "stop", container_name]) def _tiff_cubing( @@ -91,9 +113,9 @@ def test_tiff_cubing(tmp_path: Path) -> None: ) def test_tiff_cubing_zarr_s3(remote_testoutput_path: UPath) -> None: out_path = remote_testoutput_path / "tiff_cubing" - environ["AWS_SECRET_ACCESS_KEY"] = environ["MINIO_ROOT_PASSWORD"] - environ["AWS_ACCESS_KEY_ID"] = environ["MINIO_ROOT_USER"] - environ["S3_ENDPOINT_URL"] = "http://localhost:8000" + environ["AWS_SECRET_ACCESS_KEY"] = MINIO_ROOT_PASSWORD + environ["AWS_ACCESS_KEY_ID"] = MINIO_ROOT_USER + environ["S3_ENDPOINT_URL"] = f"http://localhost:{MINIO_PORT}" _tiff_cubing(out_path, DataFormat.Zarr, 1)