Uncaught pandas.errors.InvalidIndexError

[retifrav]

There is an uncaught exception in multiindex_into_df_with_nans() function on the line return df.loc[(x[i1], x[i2]), i3]:

pandas.errors.InvalidIndexError: (MaskedNDArray(0.00629842), 0.004086749358975794)

Full error output, just in case

Detrending fake LC:

Found 8 candidate(s) in the (0,5731) gap.
100%|################################################################################################################################################################|
The total number of injected flares is 1200.
Traceback (most recent call last):
  File "/path/to/some.py", line 47, in <module>
    flcc = flcd.characterize_flares(ampl_bins=10, dur_bins=10)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/path/to/AltaiPony/altaipony/flarelc.py", line 947, in characterize_flares
    flares = wrap_characterization_of_flares(flc.fake_flares, flc.flares,
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/path/to/AltaiPony/altaipony/injrecanalysis.py", line 44, in wrap_characterization_of_flares
    flcc, dscc = characterize_flares(flares, injrec, otherfunc="count",
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/path/to/AltaiPony/altaipony/injrecanalysis.py", line 111, in characterize_flares
    flares[typ] = flares.apply(helper, axis=1)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/frame.py", line 9423, in apply
    return op.apply().__finalize__(self, method="apply")
           ^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/apply.py", line 678, in apply
    return self.apply_standard()
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/apply.py", line 798, in apply_standard
    results, res_index = self.apply_series_generator()
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/apply.py", line 814, in apply_series_generator
    results[i] = self.f(v)
                 ^^^^^^^^^
  File "/path/to/AltaiPony/altaipony/injrecanalysis.py", line 108, in <lambda>
    helper = lambda x: multiindex_into_df_with_nans(x, d,
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/path/to/AltaiPony/altaipony/injrecanalysis.py", line 231, in multiindex_into_df_with_nans
    return df.loc[(x[i1], x[i2]), i3]
           ~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py", line 1097, in __getitem__
    return self._getitem_tuple(key)
           ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py", line 1280, in _getitem_tuple
    return self._getitem_lowerdim(tup)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py", line 976, in _getitem_lowerdim
    return self._getitem_nested_tuple(tup)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py", line 1077, in _getitem_nested_tuple
    obj = getattr(obj, self.name)._getitem_axis(key, axis=axis)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py", line 1343, in _getitem_axis
    return self._get_label(key, axis=axis)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexing.py", line 1293, in _get_label
    return self.obj.xs(label, axis=axis)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/generic.py", line 4088, in xs
    loc, new_index = index._get_loc_level(key, level=0)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexes/multi.py", line 2998, in _get_loc_level
    indexer = self.get_loc(key)
              ^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexes/multi.py", line 2795, in get_loc
    self._check_indexing_error(key)
  File "/usr/local/lib/python3.11/site-packages/pandas/core/indexes/multi.py", line 2516, in _check_indexing_error
    raise InvalidIndexError(key)
pandas.errors.InvalidIndexError: (MaskedNDArray(0.00629842), 0.004086749358975794)

I get it when I call characterize_flares() function.

A script to reproduce the problem

from altaipony.lcio import from_mast
import pandas

flc = from_mast(
    "Kepler-114",
    mode="LC",
    cadence="short",
    mission="Kepler"
)

for j in range(len(flc)):
    print(f"\n--- {j} ---\n")
    # detrend curve
    flcd = flc[j].detrend("savgol")
    # find flares
    flcd = flcd.find_flares(N1=3, N2=1, N3=3, minsep=3)
    flcdpanda = flcd.flares
    # print(flcdpanda)
    if not flcdpanda.empty:
        # injection of simulated flares
        flcd, fakeflc = flcd.sample_flare_recovery(
            inject_before_detrending=True,
            mode="savgol",
            iterations=20,
            fakefreq=2,
            ampl=[1e-4, 0.5],
            dur=[.001/6., 0.1/6.]
        )
        print(f"Total number of injected flares is {flcd.fake_flares.shape[0]}")

        # here it fails with uncaught exception
        flcc = flcd.characterize_flares(ampl_bins=10, dur_bins=10)
    else:
        print("DataFrame is empty, no flares")

I believe, the problem is that except block catches only KeyError, while the actual exception in my case was pandas.errors.InvalidIndexError. So the patch would be:

diff --git a/altaipony/injrecanalysis.py b/altaipony/injrecanalysis.py
index cb9234f..36ac505 100644
--- a/altaipony/injrecanalysis.py
+++ b/altaipony/injrecanalysis.py
@@ -229,8 +229,14 @@ def multiindex_into_df_with_nans(x, df, i1="ampl_rec", i2="dur", i3="edrat"):
     """
     try:
         return df.loc[(x[i1], x[i2]), i3]
-    except KeyError:
+    except (
+        KeyError,
+        pd.errors.InvalidIndexError
+    ):
         return np.nan
+    except Exception as ex:
+        print(f"Unexpected exception ({type(ex)}): {ex}")
+        raise
 
 
 def percentile(x, q):

Tested with aa6ba8d revision.