Skip to content
This repository has been archived by the owner on Oct 1, 2024. It is now read-only.

Commit

Permalink
add threading and tests
Browse files Browse the repository at this point in the history
  • Loading branch information
clbarnes committed Jul 18, 2019
1 parent 4ce43b7 commit fdc3989
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 8 deletions.
36 changes: 34 additions & 2 deletions pyn5/dataset.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
from copy import copy

from typing import Union, Tuple, Optional, Any

import numpy as np

from h5py_like import DatasetBase, AttributeManagerBase, mutation, Name
from h5py_like.shape_utils import thread_read_fn, thread_write_fn
from pyn5.attributes import AttributeManager
from .pyn5 import (
DatasetUINT8,
Expand Down Expand Up @@ -33,6 +36,8 @@


class Dataset(DatasetBase):
threads = None

def __init__(self, name: str, parent: "Group"): # noqa would need circular imports
"""
Expand All @@ -44,6 +49,7 @@ def __init__(self, name: str, parent: "Group"): # noqa would need circular impo
self._parent = parent
self._path = self.parent._path / name
self._attrs = AttributeManager.from_parent(self)
self.threads = copy(self.threads)

attrs = self._attrs._read_attributes()

Expand Down Expand Up @@ -90,20 +96,46 @@ def resize(self, size: Union[int, Tuple[int, ...]], axis: Optional[int] = None):
raise NotImplementedError()

def __getitem__(self, args) -> np.ndarray:
def fn(translation, dimensions):
def inner_fn(translation, dimensions):
return self._impl.read_ndarray(
translation[::-1], dimensions[::-1]
).transpose()

if self.threads:
def fn(translation, dimensions):
return thread_read_fn(
translation,
dimensions,
self.chunks,
self.shape,
inner_fn,
self.threads
)
else:
fn = inner_fn

return self._getitem(args, fn, self._astype)

@mutation
def __setitem__(self, args, val):
def fn(offset, arr):
def inner_fn(offset, arr):
return self._impl.write_ndarray(
offset[::-1], arr.transpose(), self.fillvalue
)

if self.threads:
def fn(offset, arr):
return thread_write_fn(
offset,
arr,
self.chunks,
self.shape,
inner_fn,
self.threads
)
else:
fn = inner_fn

return self._setitem(args, val, fn)

@property
Expand Down
2 changes: 1 addition & 1 deletion requirements_dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ setuptools_rust==0.10.6
numpy==1.16.4
pytest==4.6.3
pytest-runner==5.1
h5py_like==0.4.0
h5py_like==0.5.2
h5py==2.9.0
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
with open("HISTORY.rst") as history_file:
history = history_file.read()

requirements = ["numpy", "h5py_like>=0.4.0"]
requirements = ["numpy", "h5py_like>=0.5.2"]

setup_requirements = []
test_requirements = []
Expand Down
10 changes: 6 additions & 4 deletions tests/test_h5_like.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
from pathlib import Path

from h5py_like import Mode, FileMixin
from h5py_like.test_utils import FileTestBase, DatasetTestBase, GroupTestBase, ModeTestBase
from h5py_like.test_utils import (
FileTestBase, ThreadedDatasetTestBase, GroupTestBase, ModeTestBase,
)
from pyn5 import File

from .common import blocks_hash
from .common import blocks_in, attrs_in

ds_kwargs = deepcopy(DatasetTestBase.dataset_kwargs)
ds_kwargs = deepcopy(ThreadedDatasetTestBase.dataset_kwargs)
ds_kwargs["chunks"] = (5, 5, 5)


Expand All @@ -28,8 +30,8 @@ class TestGroup(GroupTestBase):
pass


class TestDataset(DatasetTestBase):
dataset_kwargs = ds_kwargs
class TestDataset(ThreadedDatasetTestBase):
dataset_kwargs = deepcopy(ThreadedDatasetTestBase.dataset_kwargs)

def test_has_metadata(self, file_):
ds = self.dataset(file_)
Expand Down

0 comments on commit fdc3989

Please sign in to comment.