From 1dbd7929639a76104f29e479b17b4f30a4112f83 Mon Sep 17 00:00:00 2001 From: "Lumberbot (aka Jack)" <39504233+meeseeksmachine@users.noreply.github.com> Date: Tue, 22 Aug 2023 14:18:15 -0700 Subject: [PATCH] Backport PR #54566 on branch 2.1.x (ENH: support Index.any/all with float, timedelta64 dtypes) (#54693) Backport PR #54566: ENH: support Index.any/all with float, timedelta64 dtypes Co-authored-by: jbrockmendel --- doc/source/whatsnew/v2.1.0.rst | 1 + pandas/core/indexes/base.py | 28 +++++++++++--------- pandas/tests/indexes/numeric/test_numeric.py | 8 ++++++ pandas/tests/indexes/test_base.py | 7 ++++- pandas/tests/indexes/test_old_base.py | 26 +++++++++++------- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index d2b9705a72f75..7dd2fb570c695 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -265,6 +265,7 @@ Other enhancements - Many read/to_* functions, such as :meth:`DataFrame.to_pickle` and :func:`read_csv`, support forwarding compression arguments to ``lzma.LZMAFile`` (:issue:`52979`) - Reductions :meth:`Series.argmax`, :meth:`Series.argmin`, :meth:`Series.idxmax`, :meth:`Series.idxmin`, :meth:`Index.argmax`, :meth:`Index.argmin`, :meth:`DataFrame.idxmax`, :meth:`DataFrame.idxmin` are now supported for object-dtype (:issue:`4279`, :issue:`18021`, :issue:`40685`, :issue:`43697`) - :meth:`DataFrame.to_parquet` and :func:`read_parquet` will now write and read ``attrs`` respectively (:issue:`54346`) +- :meth:`Index.all` and :meth:`Index.any` with floating dtypes and timedelta64 dtypes no longer raise ``TypeError``, matching the :meth:`Series.all` and :meth:`Series.any` behavior (:issue:`54566`) - :meth:`Series.cummax`, :meth:`Series.cummin` and :meth:`Series.cumprod` are now supported for pyarrow dtypes with pyarrow version 13.0 and above (:issue:`52085`) - Added support for the DataFrame Consortium Standard (:issue:`54383`) - Performance improvement in :meth:`.DataFrameGroupBy.quantile` and :meth:`.SeriesGroupBy.quantile` (:issue:`51722`) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 241b2de513a04..1f15a4ad84755 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -7215,11 +7215,12 @@ def any(self, *args, **kwargs): """ nv.validate_any(args, kwargs) self._maybe_disable_logical_methods("any") - # error: Argument 1 to "any" has incompatible type "ArrayLike"; expected - # "Union[Union[int, float, complex, str, bytes, generic], Sequence[Union[int, - # float, complex, str, bytes, generic]], Sequence[Sequence[Any]], - # _SupportsArray]" - return np.any(self.values) # type: ignore[arg-type] + vals = self._values + if not isinstance(vals, np.ndarray): + # i.e. EA, call _reduce instead of "any" to get TypeError instead + # of AttributeError + return vals._reduce("any") + return np.any(vals) def all(self, *args, **kwargs): """ @@ -7262,11 +7263,12 @@ def all(self, *args, **kwargs): """ nv.validate_all(args, kwargs) self._maybe_disable_logical_methods("all") - # error: Argument 1 to "all" has incompatible type "ArrayLike"; expected - # "Union[Union[int, float, complex, str, bytes, generic], Sequence[Union[int, - # float, complex, str, bytes, generic]], Sequence[Sequence[Any]], - # _SupportsArray]" - return np.all(self.values) # type: ignore[arg-type] + vals = self._values + if not isinstance(vals, np.ndarray): + # i.e. EA, call _reduce instead of "all" to get TypeError instead + # of AttributeError + return vals._reduce("all") + return np.all(vals) @final def _maybe_disable_logical_methods(self, opname: str_t) -> None: @@ -7275,9 +7277,9 @@ def _maybe_disable_logical_methods(self, opname: str_t) -> None: """ if ( isinstance(self, ABCMultiIndex) - or needs_i8_conversion(self.dtype) - or isinstance(self.dtype, (IntervalDtype, CategoricalDtype)) - or is_float_dtype(self.dtype) + # TODO(3.0): PeriodArray and DatetimeArray any/all will raise, + # so checking needs_i8_conversion will be unnecessary + or (needs_i8_conversion(self.dtype) and self.dtype.kind != "m") ): # This call will raise make_invalid_op(opname)(self) diff --git a/pandas/tests/indexes/numeric/test_numeric.py b/pandas/tests/indexes/numeric/test_numeric.py index 977c7da7d866f..8cd295802a5d1 100644 --- a/pandas/tests/indexes/numeric/test_numeric.py +++ b/pandas/tests/indexes/numeric/test_numeric.py @@ -227,6 +227,14 @@ def test_fillna_float64(self): exp = Index([1.0, "obj", 3.0], name="x") tm.assert_index_equal(idx.fillna("obj"), exp, exact=True) + def test_logical_compat(self, simple_index): + idx = simple_index + assert idx.all() == idx.values.all() + assert idx.any() == idx.values.any() + + assert idx.all() == idx.to_series().all() + assert idx.any() == idx.to_series().any() + class TestNumericInt: @pytest.fixture(params=[np.int64, np.int32, np.int16, np.int8, np.uint64]) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index ffa0b115e34fb..bc04c1c6612f4 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -692,7 +692,12 @@ def test_format_missing(self, vals, nulls_fixture): @pytest.mark.parametrize("op", ["any", "all"]) def test_logical_compat(self, op, simple_index): index = simple_index - assert getattr(index, op)() == getattr(index.values, op)() + left = getattr(index, op)() + assert left == getattr(index.values, op)() + right = getattr(index.to_series(), op)() + # left might not match right exactly in e.g. string cases where the + # because we use np.any/all instead of .any/all + assert bool(left) == bool(right) @pytest.mark.parametrize( "index", ["string", "int64", "int32", "float64", "float32"], indirect=True diff --git a/pandas/tests/indexes/test_old_base.py b/pandas/tests/indexes/test_old_base.py index f8f5a543a9c19..79dc423f12a85 100644 --- a/pandas/tests/indexes/test_old_base.py +++ b/pandas/tests/indexes/test_old_base.py @@ -209,17 +209,25 @@ def test_numeric_compat(self, simple_index): 1 // idx def test_logical_compat(self, simple_index): - if ( - isinstance(simple_index, RangeIndex) - or is_numeric_dtype(simple_index.dtype) - or simple_index.dtype == object - ): + if simple_index.dtype == object: pytest.skip("Tested elsewhere.") idx = simple_index - with pytest.raises(TypeError, match="cannot perform all"): - idx.all() - with pytest.raises(TypeError, match="cannot perform any"): - idx.any() + if idx.dtype.kind in "iufcbm": + assert idx.all() == idx._values.all() + assert idx.all() == idx.to_series().all() + assert idx.any() == idx._values.any() + assert idx.any() == idx.to_series().any() + else: + msg = "cannot perform (any|all)" + if isinstance(idx, IntervalIndex): + msg = ( + r"'IntervalArray' with dtype interval\[.*\] does " + "not support reduction '(any|all)'" + ) + with pytest.raises(TypeError, match=msg): + idx.all() + with pytest.raises(TypeError, match=msg): + idx.any() def test_repr_roundtrip(self, simple_index): if isinstance(simple_index, IntervalIndex):