From 9ae9c5aa3c916e2fecb9ab735456d96c5f18beaa Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 14:45:49 -0700 Subject: [PATCH 01/10] add testing for python3.8, update minor python versions used on travis --- .travis.yml | 9 +++++---- tox.ini | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 865e8f72..15bec089 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,10 +29,11 @@ before_install: - export PYENV_ROOT="$HOME/.pyenv" - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - - pyenv install -s 3.5.5 - - pyenv install -s 3.6.5 - - pyenv install -s 3.7.1 - - pyenv local 3.5.5 3.6.5 3.7.1 + - pyenv install -s 3.5.7 + - pyenv install -s 3.6.9 + - pyenv install -s 3.7.5 + - pyenv install -s 3.8.0 + - pyenv local 3.5.7 3.6.9 3.7.5 3.8.0 - pip install tox tox-pyenv codecov twine # Command to run tests, e.g. python setup.py test diff --git a/tox.ini b/tox.ini index 56ee095f..9185b773 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] -envlist = py36-docs,begin,py35-dependencies,py35-versions,py{35,36,37},end +envlist = py36-docs,begin,py35-dependencies,py35-versions,py{35,36,37,38},end [travis] python = + 3.8: py38 3.7: py37 3.6: py36, py36-docs 3.5: py35, py35-dependencies, py35-versions @@ -84,6 +85,6 @@ commands = coverage report --omit='.tox/*' coverage html --omit='.tox/*' skip_install = true -depends = py{35,36,37} +depends = py{35,36,37,38} deps = coverage From 21a1f27f4e6d0a8683b6184d2072f4f70c49523b Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 14:46:13 -0700 Subject: [PATCH 02/10] always recreate tox virtualenvs this means we will notice when upstream changes cause breakages in our tests, right now we don't notice since tox never recreates virtualenvs --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9185b773..c74716a5 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ python = [testenv] setenv = PYTHONPATH = {toxinidir} +recreate = true depends = begin deps = pytest<5.1 From 67696801aabfba04ee7699045c280e98fc4464f8 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 14:48:05 -0700 Subject: [PATCH 03/10] remove pytest version constraint in tox config --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index c74716a5..1f385024 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ setenv = recreate = true depends = begin deps = - pytest<5.1 + pytest sympy numpy h5py @@ -32,7 +32,7 @@ commands = [testenv:py35-versions] deps = - pytest<5.1 + pytest sympy==1.2 numpy==1.13.3 h5py==2.6.0 @@ -48,7 +48,7 @@ commands = [testenv:py35-dependencies] deps = - pytest<5.1 + pytest sympy numpy coverage From 4aac2c3dab3e0e7ff3bcf284666f52fcbe1bc46d Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 14:48:26 -0700 Subject: [PATCH 04/10] pin coverage to earlier than coverage 5.0 to avoid SQL errors --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 1f385024..2cc65dc9 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ deps = h5py pint astropy - coverage + coverage<5.0 pytest-cov pytest-doctestplus flake8 @@ -38,7 +38,7 @@ deps = h5py==2.6.0 pint==0.6 astropy==1.3.3 - coverage + coverage<5.0 pytest-cov pytest-doctestplus commands = @@ -51,7 +51,7 @@ deps = pytest sympy numpy - coverage + coverage<5.0 pytest-cov pytest-doctestplus depends = begin @@ -79,7 +79,7 @@ commands = depends = skip_install = true deps = - coverage + coverage<5.0 [testenv:end] commands = @@ -88,4 +88,4 @@ commands = skip_install = true depends = py{35,36,37,38} deps = - coverage + coverage<5.0 From 71873b0b827d81732c7c898fa8ac53f8804c2236 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 14:53:49 -0700 Subject: [PATCH 05/10] fix warnings from newer versions of h5py --- unyt/array.py | 4 ++-- unyt/tests/test_unyt_array.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unyt/array.py b/unyt/array.py index b1c30f7a..ef366728 100644 --- a/unyt/array.py +++ b/unyt/array.py @@ -1322,7 +1322,7 @@ def write_hdf5(self, filename, dataset_name=None, info=None, group_name=None): if dataset_name is None: dataset_name = "array_data" - f = h5py.File(filename) + f = h5py.File(filename, "a") if group_name is not None: if group_name in f: g = f[group_name] @@ -1372,7 +1372,7 @@ def from_hdf5(cls, filename, dataset_name=None, group_name=None): if dataset_name is None: dataset_name = "array_data" - f = h5py.File(filename) + f = h5py.File(filename, "r") if group_name is not None: g = f[group_name] else: diff --git a/unyt/tests/test_unyt_array.py b/unyt/tests/test_unyt_array.py index 5c031cc4..67226860 100644 --- a/unyt/tests/test_unyt_array.py +++ b/unyt/tests/test_unyt_array.py @@ -1477,7 +1477,7 @@ def test_h5_io(): # write to a group that does exist - with _h5py.File("test.h5") as f: + with _h5py.File("test.h5", "a") as f: f.create_group("/arrays/test_group") warr.write_hdf5( From 873e38e14a6ce06db3ad8adc493f3202630daa51 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 15:58:50 -0700 Subject: [PATCH 06/10] fix usage of rationalize transfomer by ensuring auto_number runs first --- unyt/_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unyt/_parsing.py b/unyt/_parsing.py index 54a048f1..2ce43930 100644 --- a/unyt/_parsing.py +++ b/unyt/_parsing.py @@ -76,7 +76,7 @@ def _auto_positive_symbol(tokens, local_dict, global_dict): "sqrt": sqrt, } -unit_text_transform = (_auto_positive_symbol, rationalize, auto_number) +unit_text_transform = (_auto_positive_symbol, auto_number, rationalize) def parse_unyt_expr(unit_expr): From 9f9d3ccf8a1a7f784cced0f3d4b00f75842ae4fc Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 15:59:34 -0700 Subject: [PATCH 07/10] fix latex output for non-float scalar unit multipliers --- unyt/unit_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unyt/unit_object.py b/unyt/unit_object.py index cb93a43c..dace50f6 100644 --- a/unyt/unit_object.py +++ b/unyt/unit_object.py @@ -87,7 +87,7 @@ def _get_latex_representation(expr, registry): l_expr = expr if isinstance(expr, Mul): coeffs = expr.as_coeff_Mul() - if coeffs[0] == 1 or not isinstance(coeffs[0], Float): + if coeffs[0] == 1 or not isinstance(coeffs[0], Number): l_expr = coeffs[1] else: l_expr = coeffs[1] From 87b3e80f2e5975da52b1eb0860889a804ba3c515 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 16:00:58 -0700 Subject: [PATCH 08/10] fix operators with multiple outputs in numpy 1.8 --- unyt/array.py | 39 +++++++++++++++++++++++++++-------- unyt/tests/test_unyt_array.py | 10 +++++++-- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/unyt/array.py b/unyt/array.py index ef366728..b92b2e06 100644 --- a/unyt/array.py +++ b/unyt/array.py @@ -321,6 +321,8 @@ def _sanitize_units_convert(possible_units, registry): trigonometric_operators = (sin, cos, tan) +multiple_output_operators = {modf: 2, frexp: 2, divmod_: 2} + LARGE_INPUT = {4: 16777217, 8: 9007199254740993} @@ -1584,18 +1586,29 @@ def __getitem__(self, item): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): func = getattr(ufunc, method) if "out" not in kwargs: - out = None - out_func = None + if ufunc in multiple_output_operators: + out = (None,) * multiple_output_operators[ufunc] + out_func = out + else: + out = None + out_func = None else: # we need to get both the actual "out" object and a view onto it # in case we need to do in-place operations - out = kwargs.pop("out")[0] - if out.dtype.kind in ("u", "i"): - new_dtype = "f" + str(out.dtype.itemsize) - float_values = out.astype(new_dtype) - out.dtype = new_dtype - np.copyto(out, float_values) - out_func = out.view(np.ndarray) + out = kwargs.pop("out") + if ufunc in multiple_output_operators: + out_func = [] + for arr in out: + out_func.append(arr.view(np.ndarray)) + out_func = tuple(out_func) + else: + out = out[0] + if out.dtype.kind in ("u", "i"): + new_dtype = "f" + str(out.dtype.itemsize) + float_values = out.astype(new_dtype) + out.dtype = new_dtype + np.copyto(out, float_values) + out_func = out.view(np.ndarray) if len(inputs) == 1: # Unary ufuncs inp = inputs[0] @@ -1775,6 +1788,14 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): except AttributeError: # out_arr is an ndarray out.units = Unit("", registry=self.units.registry) + elif isinstance(out, tuple): + for o, oa in zip(out, out_arr): + if o is None: + continue + try: + o.units = oa.units + except AttributeError: + o.units = Unit("", registry=self.units.registry) if mul == 1: return out_arr return mul * out_arr diff --git a/unyt/tests/test_unyt_array.py b/unyt/tests/test_unyt_array.py index 67226860..02fd4339 100644 --- a/unyt/tests/test_unyt_array.py +++ b/unyt/tests/test_unyt_array.py @@ -1010,7 +1010,10 @@ def unary_ufunc_comparison(ufunc, a): def binary_ufunc_comparison(ufunc, a, b): - out = b.copy() + if ufunc in [np.divmod]: + out = (b.copy(), b.copy()) + else: + out = b.copy() if ufunc in yield_np_ufuncs( [ "add", @@ -1073,7 +1076,10 @@ def binary_ufunc_comparison(ufunc, a, b): ): assert not isinstance(ret, unyt_array) and isinstance(ret, np.ndarray) if isinstance(ret, tuple): - assert_array_equal(ret[0], out) + assert isinstance(out, tuple) + assert len(out) == len(ret) + for o, r in zip(out, ret): + assert_array_equal(r, o) else: assert_array_equal(ret, out) if ufunc in (np.divide, np.true_divide, np.arctan2) and ( From 1fe0439811fcbb2ef02475f94c5b7119ae112cba Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 16:13:03 -0700 Subject: [PATCH 09/10] fix warnings from Pint --- unyt/tests/test_unyt_array.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/unyt/tests/test_unyt_array.py b/unyt/tests/test_unyt_array.py index 02fd4339..18c17737 100644 --- a/unyt/tests/test_unyt_array.py +++ b/unyt/tests/test_unyt_array.py @@ -1359,6 +1359,10 @@ def test_astropy(): def test_pint(): + def assert_pint_array_equal(arr1, arr2): + assert_array_equal(arr1.magnitude, arr2.magnitude) + assert str(arr1.units) == str(arr2.units) + if isinstance(_pint.UnitRegistry, NotAModule): return ureg = _pint.UnitRegistry() @@ -1371,13 +1375,11 @@ def test_pint(): yt_quan = unyt_quantity(10.0, "sqrt(g)/mm**3") yt_quan2 = unyt_quantity.from_pint(p_quan) - assert_array_equal(p_arr, yt_arr.to_pint()) - assert_equal(p_quan, yt_quan.to_pint()) + assert_pint_array_equal(p_arr, yt_arr.to_pint()) assert_array_equal(yt_arr, unyt_array.from_pint(p_arr)) assert_array_equal(yt_arr, yt_arr2) - assert_equal(p_quan.magnitude, yt_quan.to_pint().magnitude) - assert_equal(p_quan, yt_quan.to_pint()) + assert_pint_array_equal(p_quan, yt_quan.to_pint()) assert_equal(yt_quan, unyt_quantity.from_pint(p_quan)) assert_equal(yt_quan, yt_quan2) From 4d1e9cf7307fa43c15604d81c5b6bcaff02f082f Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 30 Dec 2019 16:55:04 -0700 Subject: [PATCH 10/10] make the unit registry saved to hdf5 files smaller to avoid triggering new size limits --- docs/usage.rst | 2 +- unyt/array.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 3414a7f8..acaf020b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1057,7 +1057,7 @@ HDF5 Files The :mod:`unyt` library provides a hook for writing data both to a new HDF5 file and an existing file and then subsequently reading that data back in to restore the array. This works via the :meth:`unyt_array.write_hdf5 ` and :meth:`unyt_array.from_hdf5 ` methods. The simplest way to use these functions is to write data to a file that does not exist yet: - >>> from unyt import cm, unyt_array + >>> from unyt import cm >>> import os >>> data = [1, 2, 3]*cm >>> data.write_hdf5('my_data.h5') diff --git a/unyt/array.py b/unyt/array.py index b92b2e06..a58f90fd 100644 --- a/unyt/array.py +++ b/unyt/array.py @@ -127,8 +127,13 @@ from unyt.equivalencies import equivalence_registry from unyt._on_demand_imports import _astropy, _pint from unyt._pint_conversions import convert_pint_units +from unyt._unit_lookup_table import default_unit_symbol_lut from unyt.unit_object import _check_em_conversion, _em_conversion, Unit -from unyt.unit_registry import _sanitize_unit_system, UnitRegistry +from unyt.unit_registry import ( + _sanitize_unit_system, + UnitRegistry, + default_unit_registry, +) NULL_UNIT = Unit() POWER_SIGN_MAPPING = {multiply: 1, divide: -1} @@ -1319,7 +1324,11 @@ def write_hdf5(self, filename, dataset_name=None, info=None, group_name=None): info = {} info["units"] = str(self.units) - info["unit_registry"] = np.void(pickle.dumps(self.units.registry.lut)) + lut = {} + for k, v in self.units.registry.lut.items(): + if k not in default_unit_registry.lut: + lut[k] = v + info["unit_registry"] = np.void(pickle.dumps(lut)) if dataset_name is None: dataset_name = "array_data" @@ -1382,7 +1391,9 @@ def from_hdf5(cls, filename, dataset_name=None, group_name=None): dataset = g[dataset_name] data = dataset[:] units = dataset.attrs.get("units", "") - unit_lut = pickle.loads(dataset.attrs["unit_registry"].tostring()) + unit_lut = default_unit_symbol_lut.copy() + unit_lut_load = pickle.loads(dataset.attrs["unit_registry"].tostring()) + unit_lut.update(unit_lut_load) f.close() registry = UnitRegistry(lut=unit_lut, add_default_symbols=False) return cls(data, units, registry=registry)