From 838dbda5ed63f7fa75281a15d231dfb0c4768d90 Mon Sep 17 00:00:00 2001 From: vtavana <120411540+vtavana@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:35:00 -0500 Subject: [PATCH] Implementation of `dpnp.append` and `dpnp.asarray_chkfinite` (#2015) * implement dpnp.append * implement dpnp.asarray_chkfinite * remove limitation for array creation functions * fix pre-commit * fix failed tests * add no copy test --- doc/reference/manipulation.rst | 3 + dpnp/dpnp_iface.py | 6 +- dpnp/dpnp_iface_arraycreation.py | 20 +- dpnp/dpnp_iface_manipulation.py | 185 ++++++++++++++++++ dpnp/dpnp_iface_nanfunctions.py | 2 +- tests/test_arraycreation.py | 22 ++- tests/test_manipulation.py | 106 ++++++++++ tests/test_sycl_queue.py | 20 ++ tests/test_usm_type.py | 13 ++ .../manipulation_tests/test_add_remove.py | 11 +- .../cupy/manipulation_tests/test_kind.py | 2 - 11 files changed, 359 insertions(+), 31 deletions(-) diff --git a/doc/reference/manipulation.rst b/doc/reference/manipulation.rst index c6aeb65de92..67bd6b6fb2c 100644 --- a/doc/reference/manipulation.rst +++ b/doc/reference/manipulation.rst @@ -11,7 +11,9 @@ Basic operations :nosignatures: dpnp.copyto + dpnp.ndim dpnp.shape + dpnp.size Changing array shape @@ -73,6 +75,7 @@ Changing kind of array dpnp.asfarray dpnp.asfortranarray dpnp.ascontiguousarray + dpnp.asarray_chkfinite dpnp.require diff --git a/dpnp/dpnp_iface.py b/dpnp/dpnp_iface.py index 15b68aa658b..62cd0683fe5 100644 --- a/dpnp/dpnp_iface.py +++ b/dpnp/dpnp_iface.py @@ -326,7 +326,7 @@ def check_limitations( """ Checking limitation kwargs for their supported values. - Parameter `order` is only supported with values ``"C"``, ``"F"`` + Parameter `order` is only supported with values ``"C"``, ``"F"``, and ``None``. Parameter `subok` is only supported with default value ``False``. Parameter `like` is only supported with default value ``None``. @@ -343,12 +343,12 @@ def check_limitations( if order in ("A", "a", "K", "k"): raise NotImplementedError( "Keyword argument `order` is supported only with " - f"values ``'C'`` and ``'F'``, but got {order}" + f"values 'C' and 'F', but got '{order}'" ) if order not in ("C", "c", "F", "f", None): raise ValueError( "Unrecognized `order` keyword value, expecting " - f"``'C'`` or ``'F'``, but got {order}" + f"'C' or 'F', but got '{order}'" ) if like is not None: raise NotImplementedError( diff --git a/dpnp/dpnp_iface_arraycreation.py b/dpnp/dpnp_iface_arraycreation.py index 2c76b72d0da..d15f62bda27 100644 --- a/dpnp/dpnp_iface_arraycreation.py +++ b/dpnp/dpnp_iface_arraycreation.py @@ -1151,8 +1151,6 @@ def empty( Limitations ----------- - Parameter `order` is supported only with values ``"C"``, ``"F"`` and - ``None``. Parameter `like` is supported only with default value ``None``. Otherwise, the function raises `NotImplementedError` exception. @@ -1186,7 +1184,7 @@ def empty( """ - dpnp.check_limitations(order=order, like=like) + dpnp.check_limitations(like=like) if usm_type is None: usm_type = "device" @@ -1373,8 +1371,6 @@ def eye( Limitations ----------- - Parameter `order` is supported only with values ``"C"``, ``"F"`` and - ``None``. Parameter `like` is supported only with default value ``None``. Otherwise, the function raises `NotImplementedError` exception. @@ -1414,7 +1410,7 @@ def eye( """ - dpnp.check_limitations(order=order, like=like) + dpnp.check_limitations(like=like) if usm_type is None: usm_type = "device" @@ -2013,8 +2009,6 @@ def full( Limitations ----------- - Parameter `order` is supported only with values ``"C"``, ``"F"`` and - ``None``. Parameter `like` is supported only with default value ``None``. Otherwise, the function raises `NotImplementedError` exception. @@ -2048,7 +2042,7 @@ def full( """ - dpnp.check_limitations(order=order, like=like) + dpnp.check_limitations(like=like) return dpnp_container.full( shape, @@ -3057,8 +3051,6 @@ def ones( Limitations ----------- - Parameter `order` is supported only with values ``"C"``, ``"F"`` and - ``None``. Parameter `like` is supported only with default value ``None``. Otherwise, the function raises `NotImplementedError` exception. @@ -3098,7 +3090,7 @@ def ones( """ - dpnp.check_limitations(order=order, like=like) + dpnp.check_limitations(like=like) if usm_type is None: usm_type = "device" @@ -3706,8 +3698,6 @@ def zeros( Limitations ----------- - Parameter `order` is supported only with values ``"C"``, ``"F"`` and - ``None``. Parameter `like` is supported only with default value ``None``. Otherwise, the function raises `NotImplementedError` exception. @@ -3747,7 +3737,7 @@ def zeros( """ - dpnp.check_limitations(order=order, like=like) + dpnp.check_limitations(like=like) if usm_type is None: usm_type = "device" diff --git a/dpnp/dpnp_iface_manipulation.py b/dpnp/dpnp_iface_manipulation.py index 1e157de8021..73127307be4 100644 --- a/dpnp/dpnp_iface_manipulation.py +++ b/dpnp/dpnp_iface_manipulation.py @@ -49,6 +49,8 @@ from .dpnp_array import dpnp_array __all__ = [ + "append", + "asarray_chkfinite", "asfarray", "atleast_1d", "atleast_2d", @@ -260,6 +262,189 @@ def _unpack_tuple(a): return a +def append(arr, values, axis=None): + """ + Append values to the end of an array. + + For full documentation refer to :obj:`numpy.append`. + + Parameters + ---------- + arr : {dpnp.ndarray, usm_ndarray} + Values are appended to a copy of this array. + values : {scalar, array_like} + These values are appended to a copy of `arr`. It must be of the + correct shape (the same shape as `arr`, excluding `axis`). If + `axis` is not specified, `values` can be any shape and will be + flattened before use. + These values can be in any form that can be converted to an array. + This includes scalars, lists, lists of tuples, tuples, + tuples of tuples, tuples of lists, and ndarrays. + axis : {None, int}, optional + The axis along which `values` are appended. If `axis` is not + given, both `arr` and `values` are flattened before use. + Default: ``None``. + + Returns + ------- + out : dpnp.ndarray + A copy of `arr` with `values` appended to `axis`. Note that + `append` does not occur in-place: a new array is allocated and + filled. If `axis` is None, `out` is a flattened array. + + See Also + -------- + :obj:`dpnp.insert` : Insert elements into an array. + :obj:`dpnp.delete` : Delete elements from an array. + + Examples + -------- + >>> import dpnp as np + >>> a = np.array([1, 2, 3]) + >>> np.append(a, [[4, 5, 6], [7, 8, 9]]) + array([1, 2, 3, 4, 5, 6, 7, 8, 9]) + + When `axis` is specified, `values` must have the correct shape. + + >>> b = np.array([[1, 2, 3], [4, 5, 6]]) + >>> np.append(b, [[7, 8, 9]], axis=0) + array([[1, 2, 3], + [4, 5, 6], + [7, 8, 9]]) + >>> np.append(b, [7, 8, 9], axis=0) + Traceback (most recent call last): + ... + ValueError: all the input arrays must have same number of dimensions, but + the array at index 0 has 2 dimension(s) and the array at index 1 has 1 + dimension(s) + + """ + + dpnp.check_supported_arrays_type(arr) + if not dpnp.is_supported_array_type(values): + values = dpnp.array( + values, usm_type=arr.usm_type, sycl_queue=arr.sycl_queue + ) + + if axis is None: + if arr.ndim != 1: + arr = dpnp.ravel(arr) + if values.ndim != 1: + values = dpnp.ravel(values) + axis = 0 + return dpnp.concatenate((arr, values), axis=axis) + + +def asarray_chkfinite( + a, dtype=None, order=None, *, device=None, usm_type=None, sycl_queue=None +): + """ + Convert the input to an array, checking for NaNs or Infs. + + For full documentation refer to :obj:`numpy.asarray_chkfinite`. + + Parameters + ---------- + arr : array_like + Input data, in any form that can be converted to an array. This + includes lists, lists of tuples, tuples, tuples of tuples, tuples + of lists and ndarrays. Success requires no NaNs or Infs. + dtype : str or dtype object, optional + By default, the data-type is inferred from the input data. + default: ``None``. + order : {"C", "F", "A", "K"}, optional + Memory layout of the newly output array. + Default: "K". + device : {None, string, SyclDevice, SyclQueue}, optional + An array API concept of device where the output array is created. + The `device` can be ``None`` (the default), an OneAPI filter selector + string, an instance of :class:`dpctl.SyclDevice` corresponding to + a non-partitioned SYCL device, an instance of :class:`dpctl.SyclQueue`, + or a `Device` object returned by + :obj:`dpnp.dpnp_array.dpnp_array.device` property. + Default: ``None``. + usm_type : {None, "device", "shared", "host"}, optional + The type of SYCL USM allocation for the output array. + Default: ``None``. + sycl_queue : {None, SyclQueue}, optional + A SYCL queue to use for output array allocation and copying. The + `sycl_queue` can be passed as ``None`` (the default), which means + to get the SYCL queue from `device` keyword if present or to use + a default queue. + Default: ``None``. + + Returns + ------- + out : dpnp.ndarray + Array interpretation of `a`. No copy is performed if the input is + already an ndarray. + + Raises + ------- + ValueError + Raises ``ValueError`` if `a` contains NaN (Not a Number) or + Inf (Infinity). + + See Also + -------- + :obj:`dpnp.asarray` : Create an array. + :obj:`dpnp.asanyarray` : Converts an input object into array. + :obj:`dpnp.ascontiguousarray` : Convert input to a c-contiguous array. + :obj:`dpnp.asfortranarray` : Convert input to an array with column-major + memory order. + :obj:`dpnp.fromiter` : Create an array from an iterator. + :obj:`dpnp.fromfunction` : Construct an array by executing a function + on grid positions. + + Examples + -------- + >>> import dpnp as np + + Convert a list into an array. If all elements are finite, + ``asarray_chkfinite`` is identical to ``asarray``. + + >>> a = [1, 2] + >>> np.asarray_chkfinite(a, dtype=np.float32) + array([1., 2.]) + + Raises ``ValueError`` if array_like contains Nans or Infs. + + >>> a = [1, 2, np.inf] + >>> try: + ... np.asarray_chkfinite(a) + ... except ValueError: + ... print('ValueError') + ValueError + + Creating an array on a different device or with a specified usm_type + + >>> x = np.asarray_chkfinite([1, 2, 3]) # default case + >>> x, x.device, x.usm_type + (array([1, 2, 3]), Device(level_zero:gpu:0), 'device') + + >>> y = np.asarray_chkfinite([1, 2, 3], device="cpu") + >>> y, y.device, y.usm_type + (array([1, 2, 3]), Device(opencl:cpu:0), 'device') + + >>> z = np.asarray_chkfinite([1, 2, 3], usm_type="host") + >>> z, z.device, z.usm_type + (array([1, 2, 3]), Device(level_zero:gpu:0), 'host') + + """ + + a = dpnp.asarray( + a, + dtype=dtype, + order=order, + device=device, + usm_type=usm_type, + sycl_queue=sycl_queue, + ) + if dpnp.issubdtype(a.dtype, dpnp.inexact) and not dpnp.isfinite(a).all(): + raise ValueError("array must not contain infs or NaNs") + return a + + def asfarray(a, dtype=None, *, device=None, usm_type=None, sycl_queue=None): """ Return an array converted to a float type. diff --git a/dpnp/dpnp_iface_nanfunctions.py b/dpnp/dpnp_iface_nanfunctions.py index d2f52b0b439..4e0cde48e44 100644 --- a/dpnp/dpnp_iface_nanfunctions.py +++ b/dpnp/dpnp_iface_nanfunctions.py @@ -84,7 +84,7 @@ def _replace_nan(a, val): """ dpnp.check_supported_arrays_type(a) - if issubclass(a.dtype.type, dpnp.inexact): + if dpnp.issubdtype(a.dtype, dpnp.inexact): mask = dpnp.isnan(a) if not dpnp.any(mask): mask = None diff --git a/tests/test_arraycreation.py b/tests/test_arraycreation.py index ca91ec6f699..a0978107e4e 100644 --- a/tests/test_arraycreation.py +++ b/tests/test_arraycreation.py @@ -70,24 +70,34 @@ def test_out(self): @pytest.mark.parametrize( "func, args", [ - pytest.param("empty", [3]), pytest.param("empty_like", [dpnp.ones(10)]), - pytest.param("eye", [3]), - pytest.param("full", [3, 7]), pytest.param("full_like", [dpnp.ones(10), 7]), - pytest.param("ones", [3]), pytest.param("ones_like", [dpnp.ones(10)]), - pytest.param("zeros", [3]), pytest.param("zeros_like", [dpnp.ones(10)]), ], ) -def test_exception_order(func, args): +def test_exception_order1(func, args): with pytest.raises(NotImplementedError): getattr(dpnp, func)(*args, order="K") with pytest.raises(ValueError): getattr(dpnp, func)(*args, order="S") +@pytest.mark.parametrize( + "func, args", + [ + pytest.param("empty", [3]), + pytest.param("eye", [3]), + pytest.param("full", [3, 7]), + pytest.param("ones", [3]), + pytest.param("zeros", [3]), + ], +) +def test_exception_order2(func, args): + with pytest.raises(ValueError): + getattr(dpnp, func)(*args, order="S") + + @pytest.mark.parametrize( "func, args", [ diff --git a/tests/test_manipulation.py b/tests/test_manipulation.py index 76b1e786539..3d6d1ec9524 100644 --- a/tests/test_manipulation.py +++ b/tests/test_manipulation.py @@ -7,6 +7,7 @@ import dpnp from .helper import ( + assert_dtype_allclose, get_all_dtypes, get_complex_dtypes, get_float_complex_dtypes, @@ -113,6 +114,111 @@ def test_size(): assert dpnp.size(ia, 0) == exp +class TestAppend: + @pytest.mark.parametrize( + "arr", + [[], [1, 2, 3], [[1, 2, 3], [4, 5, 6]]], + ids=["empty", "1D", "2D"], + ) + @pytest.mark.parametrize( + "value", + [[], [1, 2, 3], [[1, 2, 3], [4, 5, 6]]], + ids=["empty", "1D", "2D"], + ) + def test_basic(self, arr, value): + a = numpy.array(arr) + b = numpy.array(value) + ia = dpnp.array(a) + ib = dpnp.array(b) + + expected = numpy.append(a, b) + result = dpnp.append(ia, ib) + assert_array_equal(result, expected) + + @pytest.mark.parametrize( + "arr", + [[], [1, 2, 3], [[1, 2, 3], [4, 5, 6]]], + ids=["empty", "1D", "2D"], + ) + @pytest.mark.parametrize( + "value", + [5, [1, 2, 3], [[1, 2, 3], [4, 5, 6]]], + ids=["scalar", "1D", "2D"], + ) + def test_array_like_value(self, arr, value): + a = numpy.array(arr) + ia = dpnp.array(a) + + expected = numpy.append(a, value) + result = dpnp.append(ia, value) + assert_array_equal(result, expected) + + @pytest.mark.parametrize( + "arr", + [[1, 2, 3], [[1, 2, 3], [4, 5, 6]]], + ids=["1D", "2D"], + ) + @pytest.mark.parametrize( + "value", + [[1, 2, 3], [[1, 2, 3], [4, 5, 6]]], + ids=["1D", "2D"], + ) + def test_usm_ndarray(self, arr, value): + a = numpy.array(arr) + b = numpy.array(value) + ia = dpt.asarray(a) + ib = dpt.asarray(b) + + expected = numpy.append(a, b) + result = dpnp.append(ia, ib) + assert_array_equal(result, expected) + + @pytest.mark.parametrize("dtype1", get_all_dtypes(no_none=True)) + @pytest.mark.parametrize("dtype2", get_all_dtypes(no_none=True)) + def test_axis(self, dtype1, dtype2): + a = numpy.ones((2, 3), dtype=dtype1) + b = numpy.zeros((2, 4), dtype=dtype1) + ia = dpnp.asarray(a) + ib = dpnp.asarray(b) + + expected = numpy.append(a, b, axis=1) + result = dpnp.append(ia, ib, axis=1) + assert_array_equal(result, expected) + + +class TestAsarrayCheckFinite: + @pytest.mark.parametrize("dtype", get_all_dtypes()) + def test_basic(self, dtype): + a = [1, 2, 3] + expected = numpy.asarray_chkfinite(a, dtype=dtype) + result = dpnp.asarray_chkfinite(a, dtype=dtype) + assert_dtype_allclose(result, expected) + + @pytest.mark.parametrize("xp", [numpy, dpnp]) + def test_error(self, xp): + b = [1, 2, numpy.inf] + c = [1, 2, numpy.nan] + assert_raises(ValueError, xp.asarray_chkfinite, b) + assert_raises(ValueError, xp.asarray_chkfinite, c) + + @pytest.mark.parametrize("order", ["C", "F", "A", "K"]) + def test_dtype_order(self, order): + a = [1, 2, 3] + expected = numpy.asarray_chkfinite(a, order=order) + result = dpnp.asarray_chkfinite(a, order=order) + assert_array_equal(result, expected) + + def test_no_copy(self): + a = dpnp.ones(10) + + # No copy is performed if the input is already an ndarray + b = dpnp.asarray_chkfinite(a) + + # b is a view of a, changing b, modifies a + b[0::2] = 0 + assert_array_equal(b, a) + + class TestRepeat: @pytest.mark.parametrize( "data", diff --git a/tests/test_sycl_queue.py b/tests/test_sycl_queue.py index 09b7113c811..7e9c35507c4 100644 --- a/tests/test_sycl_queue.py +++ b/tests/test_sycl_queue.py @@ -1829,6 +1829,7 @@ def test_to_device(device_from, device_to): [ "array", "asarray", + "asarray_chkfinite", "asanyarray", "ascontiguousarray", "asfarray", @@ -1955,6 +1956,25 @@ def test_concat_stack(func, data1, data2, device): assert_sycl_queue_equal(result.sycl_queue, x2.sycl_queue) +@pytest.mark.parametrize( + "device", + valid_devices, + ids=[device.filter_string for device in valid_devices], +) +def test_append(device): + x1_orig = numpy.array([1, 2, 3]) + x2_orig = numpy.array([4, 5, 6]) + expected = numpy.append(x1_orig, x2_orig) + + x1 = dpnp.array(x1_orig, device=device) + x2 = dpnp.array(x2_orig, device=device) + result = dpnp.append(x1, x2) + + assert_allclose(result, expected) + assert_sycl_queue_equal(result.sycl_queue, x1.sycl_queue) + assert_sycl_queue_equal(result.sycl_queue, x2.sycl_queue) + + @pytest.mark.parametrize( "device_x", valid_devices, diff --git a/tests/test_usm_type.py b/tests/test_usm_type.py index 0f3f685d48c..db9438d6a8c 100644 --- a/tests/test_usm_type.py +++ b/tests/test_usm_type.py @@ -322,6 +322,7 @@ def test_logspace_base(usm_type_x, usm_type_y): [ "array", "asarray", + "asarray_chkfinite", "asanyarray", "ascontiguousarray", "asfarray", @@ -767,6 +768,18 @@ def test_concat_stack(func, data1, data2, usm_type_x, usm_type_y): assert z.usm_type == du.get_coerced_usm_type([usm_type_x, usm_type_y]) +@pytest.mark.parametrize("usm_type_x", list_of_usm_types, ids=list_of_usm_types) +@pytest.mark.parametrize("usm_type_y", list_of_usm_types, ids=list_of_usm_types) +def test_append(usm_type_x, usm_type_y): + x = dp.array([1, 2, 3], usm_type=usm_type_x) + y = dp.array([4, 5, 6], usm_type=usm_type_y) + z = dp.append(x, y) + + assert x.usm_type == usm_type_x + assert y.usm_type == usm_type_y + assert z.usm_type == du.get_coerced_usm_type([usm_type_x, usm_type_y]) + + @pytest.mark.parametrize("usm_type", list_of_usm_types, ids=list_of_usm_types) @pytest.mark.parametrize( "p", diff --git a/tests/third_party/cupy/manipulation_tests/test_add_remove.py b/tests/third_party/cupy/manipulation_tests/test_add_remove.py index e5083848b57..1e24175b6c4 100644 --- a/tests/third_party/cupy/manipulation_tests/test_add_remove.py +++ b/tests/third_party/cupy/manipulation_tests/test_add_remove.py @@ -4,6 +4,7 @@ import pytest import dpnp as cupy +from tests.helper import has_support_aspect64 from tests.third_party.cupy import testing from tests.third_party.cupy.testing._loops import ( _complex_dtypes, @@ -58,7 +59,6 @@ def test_delete_with_indices_as_int(self, xp): return xp.delete(arr, indices) -@pytest.mark.skip("append() is not implemented yet") class TestAppend(unittest.TestCase): @testing.for_all_dtypes_combination( names=["dtype1", "dtype2"], no_bool=True @@ -69,6 +69,7 @@ def test(self, xp, dtype1, dtype2): b = testing.shaped_random((6, 7), xp, dtype2) return xp.append(a, b) + @pytest.mark.skip("Scalar input is not supported") @testing.for_all_dtypes_combination( names=["dtype1", "dtype2"], no_bool=True ) @@ -80,11 +81,12 @@ def test_scalar_lhs(self, xp, dtype1, dtype2): @testing.for_all_dtypes_combination( names=["dtype1", "dtype2"], no_bool=True ) - @testing.numpy_cupy_array_equal() + @testing.numpy_cupy_array_equal(type_check=has_support_aspect64()) def test_scalar_rhs(self, xp, dtype1, dtype2): scalar = xp.dtype(dtype2).type(10).item() return xp.append(xp.arange(20, dtype=dtype1), scalar) + @pytest.mark.skip("Scalar input is not supported") @testing.for_all_dtypes_combination( names=["dtype1", "dtype2"], no_bool=True ) @@ -96,12 +98,13 @@ def test_numpy_scalar_lhs(self, xp, dtype1, dtype2): @testing.for_all_dtypes_combination( names=["dtype1", "dtype2"], no_bool=True ) - @testing.numpy_cupy_array_equal() + @testing.numpy_cupy_array_equal(type_check=has_support_aspect64()) def test_numpy_scalar_rhs(self, xp, dtype1, dtype2): scalar = xp.dtype(dtype2).type(10) return xp.append(xp.arange(20, dtype=dtype1), scalar) @testing.numpy_cupy_array_equal() + @pytest.mark.skip("Scalar input is not supported") def test_scalar_both(self, xp): return xp.append(10, 10) @@ -115,7 +118,7 @@ def test_axis(self, xp): def test_zerodim(self, xp): return xp.append(xp.array(0), xp.arange(10)) - @testing.numpy_cupy_array_equal() + @testing.numpy_cupy_array_equal(type_check=has_support_aspect64()) def test_empty(self, xp): return xp.append(xp.array([]), xp.arange(10)) diff --git a/tests/third_party/cupy/manipulation_tests/test_kind.py b/tests/third_party/cupy/manipulation_tests/test_kind.py index e322bab7a05..8bc98cdc248 100644 --- a/tests/third_party/cupy/manipulation_tests/test_kind.py +++ b/tests/third_party/cupy/manipulation_tests/test_kind.py @@ -9,7 +9,6 @@ class TestKind(unittest.TestCase): - @pytest.mark.skip("dpnp.asarray_chkfinite() is not implemented yet") @testing.for_orders("CFAK") @testing.for_all_dtypes() @testing.numpy_cupy_array_equal() @@ -17,7 +16,6 @@ def test_asarray_chkfinite(self, xp, dtype, order): a = [0, 4, 0, 5] return xp.asarray_chkfinite(a, dtype=dtype, order=order) - @pytest.mark.skip("dpnp.asarray_chkfinite() is not implemented yet") @testing.for_orders("CFAK") @testing.for_all_dtypes(no_bool=True) def test_asarray_chkfinite_non_finite_vals(self, dtype, order):