diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 0000000..780bd54 --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,25 @@ +name: integration tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + name: integration tests + steps: + - uses: actions/checkout@v1 + - name: Install + run: pip install -e . + - name: Install packages needed for tests + # pyright 1.1.336 can produce annoying errors + run: pip install pytest pytest-cov pyright==1.1.335 + - name: Run pyright + run: cd lindi && pyright + - name: Run tests and collect coverage + run: pytest --cov lindi --cov-report=xml --cov-report=term tests/ + # - uses: codecov/codecov-action@v3 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # fail_ci_if_error: true + # file: ./coverage.xml + # flags: unittests \ No newline at end of file diff --git a/.github/workflows/linter_checks.yml b/.github/workflows/linter_checks.yml new file mode 100644 index 0000000..3e03a0f --- /dev/null +++ b/.github/workflows/linter_checks.yml @@ -0,0 +1,22 @@ +name: linter-checks + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + name: Linter checks + steps: + - uses: actions/checkout@v1 + - name: Install + run: pip install -e . + - name: Install packages needed for tests + # pyright 1.1.336 can produce annoying errors + run: pip install pyright==1.1.335 flake8 + - name: Run flake8 + run: cd lindi && flake8 --config ../.flake8 + - name: Run pyright + run: cd lindi && pyright diff --git a/.vscode/tasks/test.sh b/.vscode/tasks/test.sh index a6bf42a..0e09ca0 100755 --- a/.vscode/tasks/test.sh +++ b/.vscode/tasks/test.sh @@ -2,5 +2,6 @@ set -ex # black --check . +flake8 . pyright pytest --cov=lindi --cov-report=xml --cov-report=term tests/ diff --git a/devel/old_tests/test_lindi_client.py b/devel/old_tests/test_lindi_client.py index 8e8738f..2fa62de 100644 --- a/devel/old_tests/test_lindi_client.py +++ b/devel/old_tests/test_lindi_client.py @@ -15,7 +15,9 @@ def test_lindi_client(): for k in acquisition.keys(): print(k) - x = client["acquisition/ElectricalSeriesAp"]["data"] + aa = client["acquisition/ElectricalSeriesAp"] + assert isinstance(aa, LindiGroup) + x = aa["data"] assert isinstance(x, LindiDataset) print(x.shape) diff --git a/examples/example1.py b/examples/example1.py index ddb011b..b22de17 100644 --- a/examples/example1.py +++ b/examples/example1.py @@ -1,6 +1,7 @@ import numpy as np import h5py import tempfile +import lindi from lindi import LindiH5Store, LindiClient, LindiDataset from _check_equal import _check_equal @@ -76,8 +77,9 @@ def example1(): G1 = h5f["group"] G2 = client["group"] for k, v in G1.attrs.items(): - assert k in G2.attrs - assert _check_equal(v, G2.attrs[k]) + if not isinstance(G2, lindi.LindiReference): + assert k in G2.attrs + assert _check_equal(v, G2.attrs[k]) print("Comparing root group") for k, v in h5f.attrs.items(): diff --git a/examples/try_pynwb.py b/examples/try_pynwb.py index 5721e52..610b66c 100644 --- a/examples/try_pynwb.py +++ b/examples/try_pynwb.py @@ -28,4 +28,4 @@ def try_pynwb(): if __name__ == "__main__": - try_pynwb() \ No newline at end of file + try_pynwb() diff --git a/lindi/LindiClient/LindiAttributes.py b/lindi/LindiClient/LindiAttributes.py index afba68a..0b9682d 100644 --- a/lindi/LindiClient/LindiAttributes.py +++ b/lindi/LindiClient/LindiAttributes.py @@ -5,7 +5,7 @@ class LindiAttributes: def __init__(self, *, _object: Union[zarr.Group, zarr.Array]): self._object = _object - + def get(self, key, default=None): return self._object.attrs.get(key, default) diff --git a/lindi/LindiClient/LindiClient.py b/lindi/LindiClient/LindiClient.py index 62f6766..152105f 100644 --- a/lindi/LindiClient/LindiClient.py +++ b/lindi/LindiClient/LindiClient.py @@ -1,7 +1,7 @@ from typing import Union import json import tempfile -from altair import Literal +from typing import Literal from fsspec import FSMap import zarr import urllib.request diff --git a/lindi/LindiClient/__init__.py b/lindi/LindiClient/__init__.py index 77b822b..6d8a5e0 100644 --- a/lindi/LindiClient/__init__.py +++ b/lindi/LindiClient/__init__.py @@ -2,3 +2,4 @@ from .LindiGroup import LindiGroup # noqa: F401 from .LindiDataset import LindiDataset # noqa: F401 from .LindiAttributes import LindiAttributes # noqa: F401 +from .LindiGroup import LindiReference # noqa: F401 diff --git a/lindi/LindiH5Store/LindiH5Store.py b/lindi/LindiH5Store/LindiH5Store.py index b7e9d0b..7e022de 100644 --- a/lindi/LindiH5Store/LindiH5Store.py +++ b/lindi/LindiH5Store/LindiH5Store.py @@ -481,7 +481,7 @@ def _get_chunk_names_for_dataset(chunk_coords_shape: List[int]) -> List[str]: return names -def _reformat_json(x: bytes): +def _reformat_json(x: Union[bytes, None]) -> Union[bytes, None]: if x is None: return None return json.dumps(json.loads(x.decode("utf-8"))).encode("utf-8") diff --git a/lindi/__init__.py b/lindi/__init__.py index c28a4f7..7d82117 100644 --- a/lindi/__init__.py +++ b/lindi/__init__.py @@ -1,2 +1,2 @@ -from .LindiClient import LindiClient, LindiGroup, LindiDataset, LindiAttributes # noqa: F401 +from .LindiClient import LindiClient, LindiGroup, LindiDataset, LindiAttributes, LindiReference # noqa: F401 from .LindiH5Store import LindiH5Store, LindiH5StoreOpts # noqa: F401 diff --git a/pyproject.toml b/pyproject.toml index 90518e5..6ec534d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ numcodecs = "^0.12.1" zarr = "^2.16.1" h5py = "^3.10.0" remfile = "^0.1.9" +fsspec = "^2023.12.2" [tool.poetry.dev-dependencies] pytest = "^7.4.4"