From 4ce3970f885f762f031dfb718b807ae06d76f924 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Wed, 20 Nov 2019 17:15:07 -0600 Subject: [PATCH] Start using RegularArray everywhere it needs to be used. (#24) This check-in includes a commented-out attempt at `lower_getitem_nothing` in `awkward1._numba.array.numpyarray`. In the next PR, it will be gone. I concluded that there is no non-dependent type inference (where array dimensionality is part of the type but the length is not) for `__getitem__` of arrays for `NumpyArray` because an empty array would reduce the dimensionality by one yet return nothing, but reducing the dimensionality of a one-dimensional `NumpyArray` would return a scalar, which is not nothing. Whereas the C++ implementation could be fully transferred from a `ListArray` wrapper to a `RegularArray` wrapper (because it is dynamically typed, ironically), the Numba implementation could not (because it is not dynamically typed). The Numba implementation only wraps with `RegularArray` if the types are sound. * Start using RegularArray everywhere it needs to be used. * [skip ci] ListArray64 wrapper -> RegularArray wrapper partly works. * [skip ci] ListArray64 wrapper -> RegularArray wrapper still only works partially. * ListArray64 wrapper -> RegularArray wrapper now works * RegularArray::tojson_part and fix EmptyArray::tojson_part. * Centralize lower_getitem_tuple and lower_getitem_other. * Avoid selecting types based on a value (length of array) in Content::getitem(Slice). * getitem_tuple for Numba mirrors C++ as well as possible: if the 'outtpe' is 'NumpyArrayType', then you're going to have to 'lower_getitem_int'; 'lower_getitem_nothing' is impossible. * ListArray and ListOffsetArray::getitem_next(array, not advanced) returns a RegularArray now (C++ and Numba). * ListArray and ListOffsetArray::getitem_next(array) returns the same shape of RegularArrays as the index array. * Now newaxis uses RegularArray, too. * Ambiguous identities in ListArray are now refused with an error. * [skip ci] writing awkward_identity*_from_listoffsetarray*. * ListOffsetArray now uses awkward_identity*_from_listoffsetarray*. * Addressed the FIXMEs in getitem.cpp; ready to merge PR. --- README.md | 2 +- VERSION_INFO | 2 +- awkward1/_numba/array/emptyarray.py | 7 + awkward1/_numba/array/listarray.py | 34 ++-- awkward1/_numba/array/listoffsetarray.py | 31 ++-- awkward1/_numba/array/numpyarray.py | 38 +++++ awkward1/_numba/array/regulararray.py | 21 +-- awkward1/_numba/content.py | 61 +++++++ awkward1/_numba/util.py | 29 +--- awkward1/signatures/Content_8cpp.xml | 18 +- awkward1/signatures/ListArray_8cpp.xml | 8 +- awkward1/signatures/ListOffsetArray_8cpp.xml | 12 +- awkward1/signatures/NumpyArray_8cpp.xml | 10 +- awkward1/signatures/getitem_8cpp.xml | 84 ++++------ awkward1/signatures/libawkward_2util_8cpp.xml | 24 +-- include/awkward/Content.h | 3 + include/awkward/array/EmptyArray.h | 1 + include/awkward/array/ListArray.h | 1 + include/awkward/array/ListOffsetArray.h | 1 + include/awkward/array/NumpyArray.h | 1 + include/awkward/array/RawArray.h | 4 + include/awkward/array/RegularArray.h | 1 + include/awkward/cpu-kernels/getitem.h | 6 +- include/awkward/cpu-kernels/identity.h | 5 + include/awkward/util.h | 4 +- src/cpu-kernels/getitem.cpp | 45 +++-- src/cpu-kernels/identity.cpp | 41 +++++ src/libawkward/Content.cpp | 29 +++- src/libawkward/Slice.cpp | 5 +- src/libawkward/array/EmptyArray.cpp | 7 +- src/libawkward/array/ListArray.cpp | 14 +- src/libawkward/array/ListOffsetArray.cpp | 43 ++--- src/libawkward/array/NumpyArray.cpp | 19 ++- src/libawkward/array/RegularArray.cpp | 16 +- src/libawkward/util.cpp | 21 ++- src/pyawkward.cpp | 21 +-- tests/test_PR013_error_handling_struct.py | 18 +- ...test_PR016_finish_getitem_for_rawarray.cpp | 3 - tests/test_PR023_regular_array.py | 157 +++++++++--------- tests/test_PR024_use_regular_array.py | 85 ++++++++++ 40 files changed, 584 insertions(+), 348 deletions(-) create mode 100644 tests/test_PR024_use_regular_array.py diff --git a/README.md b/README.md index f5b216a9ac..2864c6b874 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit- * [ ] Start the `awkward → awkward0`, `awkward1 → awkward` transition. * [ ] Translation to and from Apache Arrow and Parquet in C++. * [ ] Persistence to any medium that stores named binary blobs, as before, but accessible via C++ (especially for writing). The persistence format might differ slightly from the existing one (break backward compatibility, if needed). - * [ ] Universal `array.get[...]` as a softer form of `array[...]` that skips non-existent indexes, rather than raising errors. + * [ ] Universal `array.get[...]` as a softer form of `array[...]` that inserts `None` for non-existent indexes, rather than raising errors. * [ ] Explicit interface with [NumExpr](https://numexpr.readthedocs.io/en/latest/index.html). ### At some point in the future diff --git a/VERSION_INFO b/VERSION_INFO index 001d752878..5a48b6be2a 100644 --- a/VERSION_INFO +++ b/VERSION_INFO @@ -1 +1 @@ -0.1.23 +0.1.24 diff --git a/awkward1/_numba/array/emptyarray.py b/awkward1/_numba/array/emptyarray.py index fe1b4060b6..6c528e8fdc 100644 --- a/awkward1/_numba/array/emptyarray.py +++ b/awkward1/_numba/array/emptyarray.py @@ -48,6 +48,10 @@ def carry(self): def lower_len(self): return lower_len + @property + def lower_getitem_nothing(self): + return lower_getitem_nothing + @property def lower_getitem_range(self): return lower_getitem_range @@ -95,6 +99,9 @@ def box(tpe, val, c): def lower_len(context, builder, sig, args): return context.get_constant(numba.intp, 0) +def lower_getitem_nothing(context, builder, tpe, val): + return val + @numba.extending.lower_builtin(operator.getitem, EmptyArrayType, numba.types.slice2_type) def lower_getitem_range(context, builder, sig, args): rettpe, (tpe, wheretpe) = sig.return_type, sig.args diff --git a/awkward1/_numba/array/listarray.py b/awkward1/_numba/array/listarray.py index 7524765c0a..dcd8686ac2 100644 --- a/awkward1/_numba/array/listarray.py +++ b/awkward1/_numba/array/listarray.py @@ -52,7 +52,7 @@ def getitem_tuple(self, wheretpe): return outtpe.getitem_int() def getitem_next(self, wheretpe, isadvanced): - import awkward1._numba.array.listoffsetarray + import awkward1._numba.array.regulararray if len(wheretpe.types) == 0: return self headtpe = wheretpe.types[0] @@ -76,7 +76,7 @@ def getitem_next(self, wheretpe, isadvanced): raise NotImplementedError("array.ndim != 1") contenttpe = self.contenttpe.carry().getitem_next(tailtpe, True) if not isadvanced: - return awkward1._numba.array.listoffsetarray.ListOffsetArrayType(util.indextpe(self.indexname), contenttpe, self.idtpe) + return awkward1._numba.array.regulararray.RegularArrayType(contenttpe, self.idtpe) else: return contenttpe @@ -90,6 +90,10 @@ def carry(self): def lower_len(self): return lower_len + @property + def lower_getitem_nothing(self): + return content.lower_getitem_nothing + @property def lower_getitem_int(self): return lower_getitem_int @@ -222,16 +226,7 @@ def lower_getitem_range(context, builder, sig, args): @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.BaseTuple) def lower_getitem_tuple(context, builder, sig, args): - rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args - arrayval, whereval = args - - wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval) - nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval) - - outtpe = nexttpe.getitem_next(wheretpe, False) - outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None) - - return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0))) + return content.lower_getitem_tuple(context, builder, sig, args) @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.Array) @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.List) @@ -239,14 +234,11 @@ def lower_getitem_tuple(context, builder, sig, args): @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.EllipsisType) @numba.extending.lower_builtin(operator.getitem, ListArrayType, type(numba.typeof(numpy.newaxis))) def lower_getitem_other(context, builder, sig, args): - rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args - arrayval, whereval = args - wrappedtpe = numba.types.Tuple((wheretpe,)) - wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,)) - return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval)) + return content.lower_getitem_other(context, builder, sig, args) def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, advanced): import awkward1._numba.array.listoffsetarray + import awkward1._numba.array.regulararray if len(wheretpe.types) == 0: return arrayval @@ -400,10 +392,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, nextcarry = util.newindex64(context, builder, numba.int64, lencarry) nextadvanced = util.newindex64(context, builder, numba.int64, lencarry) - nextoffsets = util.newindex(arraytpe.indexname, context, builder, numba.int64, lenoffsets) util.call(context, builder, kernel, - (util.arrayptr(context, builder, util.indextpe(arraytpe.indexname), nextoffsets), - util.arrayptr(context, builder, util.index64tpe, nextcarry), + (util.arrayptr(context, builder, util.index64tpe, nextcarry), util.arrayptr(context, builder, util.index64tpe, nextadvanced), util.arrayptr(context, builder, arraytpe.startstpe, proxyin.starts), util.arrayptr(context, builder, arraytpe.stopstpe, proxyin.stops), @@ -421,10 +411,10 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, contenttpe = nexttpe.getitem_next(tailtpe, True) contentval = nexttpe.lower_getitem_next(context, builder, nexttpe, tailtpe, nextval, tailval, nextadvanced) - outtpe = awkward1._numba.array.listoffsetarray.ListOffsetArrayType(util.indextpe(arraytpe.indexname), contenttpe, arraytpe.idtpe) + outtpe = awkward1._numba.array.regulararray.RegularArrayType(contenttpe, arraytpe.idtpe) proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder) - proxyout.offsets = nextoffsets proxyout.content = contentval + proxyout.size = lenflathead if outtpe.idtpe != numba.none: proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, outtpe.idtpe, util.index64tpe, proxyin.id, flathead) return proxyout._getvalue() diff --git a/awkward1/_numba/array/listoffsetarray.py b/awkward1/_numba/array/listoffsetarray.py index 745bbfe50b..48388e32f4 100644 --- a/awkward1/_numba/array/listoffsetarray.py +++ b/awkward1/_numba/array/listoffsetarray.py @@ -47,6 +47,7 @@ def getitem_tuple(self, wheretpe): return out.getitem_int() def getitem_next(self, wheretpe, isadvanced): + import awkward1._numba.array.regulararray if len(wheretpe.types) == 0: return self headtpe = wheretpe.types[0] @@ -70,7 +71,7 @@ def getitem_next(self, wheretpe, isadvanced): raise NotImplementedError("array.ndim != 1") contenttpe = self.contenttpe.carry().getitem_next(tailtpe, True) if not isadvanced: - return ListOffsetArrayType(util.indextpe(self.indexname), contenttpe, self.idtpe) + return awkward1._numba.array.regulararray.RegularArrayType(contenttpe, self.idtpe) else: return contenttpe @@ -85,6 +86,10 @@ def carry(self): def lower_len(self): return lower_len + @property + def lower_getitem_nothing(self): + return content.lower_getitem_nothing + @property def lower_getitem_int(self): return lower_getitem_int @@ -219,15 +224,7 @@ def lower_getitem_range(context, builder, sig, args): @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.BaseTuple) def lower_getitem_tuple(context, builder, sig, args): - rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args - arrayval, whereval = args - - wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval) - nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval) - - outtpe = nexttpe.getitem_next(wheretpe, False) - outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None) - return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0))) + return content.lower_getitem_tuple(context, builder, sig, args) @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.Array) @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.List) @@ -235,11 +232,7 @@ def lower_getitem_tuple(context, builder, sig, args): @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.EllipsisType) @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, type(numba.typeof(numpy.newaxis))) def lower_getitem_other(context, builder, sig, args): - rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args - arrayval, whereval = args - wrappedtpe = numba.types.Tuple((wheretpe,)) - wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,)) - return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval)) + return content.lower_getitem_other(context, builder, sig, args) def starts_stops(context, builder, offsetstpe, offsetsval, lenstarts, lenoffsets): proxyslicestarts = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder) @@ -411,10 +404,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, nextcarry = util.newindex64(context, builder, numba.int64, lencarry) nextadvanced = util.newindex64(context, builder, numba.int64, lencarry) - nextoffsets = util.newindex(arraytpe.indexname, context, builder, numba.int64, lenoffsets) util.call(context, builder, kernel, - (util.arrayptr(context, builder, util.indextpe(arraytpe.indexname), nextoffsets), - util.arrayptr(context, builder, util.index64tpe, nextcarry), + (util.arrayptr(context, builder, util.index64tpe, nextcarry), util.arrayptr(context, builder, util.index64tpe, nextadvanced), util.arrayptr(context, builder, arraytpe.offsetstpe, starts), util.arrayptr(context, builder, arraytpe.offsetstpe, stops), @@ -432,10 +423,10 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, contenttpe = nexttpe.getitem_next(tailtpe, True) contentval = nexttpe.lower_getitem_next(context, builder, nexttpe, tailtpe, nextval, tailval, nextadvanced) - outtpe = ListOffsetArrayType(util.indextpe(arraytpe.indexname), contenttpe, arraytpe.idtpe) + outtpe = awkward1._numba.array.regulararray.RegularArrayType(contenttpe, arraytpe.idtpe) proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder) - proxyout.offsets = nextoffsets proxyout.content = contentval + proxyout.size = lenflathead if outtpe.idtpe != numba.none: proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, outtpe.idtpe, util.index64tpe, proxyin.id, flathead) return proxyout._getvalue() diff --git a/awkward1/_numba/array/numpyarray.py b/awkward1/_numba/array/numpyarray.py index 5e2776ae8e..82f88f8ae0 100644 --- a/awkward1/_numba/array/numpyarray.py +++ b/awkward1/_numba/array/numpyarray.py @@ -58,6 +58,10 @@ def carry(self): def lower_len(self): return lower_len + @property + def lower_getitem_nothing(self): + return None + @property def lower_getitem_int(self): return lower_getitem @@ -119,6 +123,40 @@ def lower_len(context, builder, sig, args): proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) return numba.targets.arrayobj.array_len(context, builder, numba.intp(tpe.arraytpe), (proxyin.array,)) +# def lower_getitem_nothing(context, builder, tpe, val): +# import awkward1._numba.identity +# +# proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) +# +# proxyslice = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder) +# proxyslice.start = context.get_constant(numba.intp, 0) +# proxyslice.stop = context.get_constant(numba.intp, 0) +# proxyslice.step = context.get_constant(numba.intp, 1) +# emptyslice = proxyslice._getvalue() +# emptyarray = numba.targets.arrayobj.getitem_arraynd_intp(context, builder, tpe.arraytpe(tpe.arraytpe, numba.types.slice2_type), (proxyin.array, emptyslice)) +# +# if tpe.arraytpe.ndim > 1: +# shapetpe = numba.types.Tuple((numba.intp,) * tpe.arraytpe.ndim) +# shapeval = numba.targets.arrayobj.make_array(tpe.arraytpe)(context, builder, proxyin.array).shape +# +# newshapetpe = numba.types.Tuple((numba.intp,) * (tpe.arraytpe.ndim - 1)) +# newshapeval = context.make_tuple(builder, newshapetpe, tuple(builder.extract_value(shapeval, i) for i in range(tpe.arraytpe.ndim - 1))) +# +# arraytpe = numba.types.Array(tpe.arraytpe.dtype, tpe.arraytpe.ndim - 1, tpe.arraytpe.layout) +# arrayval = numba.targets.arrayobj.array_reshape(context, builder, arraytpe(tpe.arraytpe, newshapetpe), (proxyin.array, newshapeval)) +# +# else: +# arraytpe = tpe.arraytpe +# arrayval = emptyarray +# +# outtpe = NumpyArrayType(arraytpe, tpe.idtpe) +# proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder) +# proxyout.array = arrayval +# if tpe.idtpe != numba.none: +# proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, tpe.idtpe, numba.types.slice2_type, proxyin.id, emptyslice) +# +# return proxyout._getvalue() + @numba.extending.lower_builtin(operator.getitem, NumpyArrayType, numba.types.Integer) @numba.extending.lower_builtin(operator.getitem, NumpyArrayType, numba.types.SliceType) @numba.extending.lower_builtin(operator.getitem, NumpyArrayType, numba.types.Array) diff --git a/awkward1/_numba/array/regulararray.py b/awkward1/_numba/array/regulararray.py index 6369e7e829..03ab92422b 100644 --- a/awkward1/_numba/array/regulararray.py +++ b/awkward1/_numba/array/regulararray.py @@ -72,6 +72,10 @@ def carry(self): def lower_len(self): return lower_len + @property + def lower_getitem_nothing(self): + return content.lower_getitem_nothing + @property def lower_getitem_int(self): return lower_getitem_int @@ -188,16 +192,7 @@ def lower_getitem_range(context, builder, sig, args): @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.BaseTuple) def lower_getitem_tuple(context, builder, sig, args): - rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args - arrayval, whereval = args - - wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval) - nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval) - - outtpe = nexttpe.getitem_next(wheretpe, False) - outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None) - - return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0))) + return content.lower_getitem_tuple(context, builder, sig, args) @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.Array) @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.List) @@ -205,11 +200,7 @@ def lower_getitem_tuple(context, builder, sig, args): @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.EllipsisType) @numba.extending.lower_builtin(operator.getitem, RegularArrayType, type(numba.typeof(numpy.newaxis))) def lower_getitem_other(context, builder, sig, args): - rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args - arrayval, whereval = args - wrappedtpe = numba.types.Tuple((wheretpe,)) - wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,)) - return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval)) + return content.lower_getitem_other(context, builder, sig, args) def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, advanced): if len(wheretpe.types) == 0: diff --git a/awkward1/_numba/content.py b/awkward1/_numba/content.py index 492ba2c031..bc93005fb3 100644 --- a/awkward1/_numba/content.py +++ b/awkward1/_numba/content.py @@ -42,3 +42,64 @@ def generic(self, args, kwargs): raise TypeError("only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`), and integer or boolean arrays (possibly jagged) are valid indices") return numba.typing.templates.signature(arraytpe.getitem_tuple(wheretpe), arraytpe, original_wheretpe) + +def lower_getitem_nothing(context, builder, tpe, val): + proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) + proxyslice = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder) + proxyslice.start = context.get_constant(numba.intp, 0) + proxyslice.stop = context.get_constant(numba.intp, 0) + proxyslice.step = context.get_constant(numba.intp, 1) + outtpe = tpe.contenttpe.getitem_range() + return tpe.contenttpe.lower_getitem_range(context, builder, outtpe(tpe.contenttpe, numba.types.slice2_type), (proxyin.content, proxyslice._getvalue())) + +def lower_getitem_tuple(context, builder, sig, args): + import awkward1._numba.array.listarray + import awkward1._numba.array.regulararray + + rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args + arrayval, whereval = args + + wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval) + + nexttpe = awkward1._numba.array.regulararray.RegularArrayType(arraytpe, numba.types.none) + outtpe = nexttpe.getitem_next(wheretpe, False) + if outtpe.lower_getitem_nothing is None: + nexttpe = awkward1._numba.array.listarray.ListArrayType(util.index64tpe, util.index64tpe, arraytpe, numba.types.none) + outtpe = nexttpe.getitem_next(wheretpe, False) + + length = util.arraylen(context, builder, arraytpe, arrayval, totpe=numba.int64) + proxynext = numba.cgutils.create_struct_proxy(nexttpe)(context, builder) + proxynext.starts = util.newindex64(context, builder, numba.int64, context.get_constant(numba.int64, 1)) + proxynext.stops = util.newindex64(context, builder, numba.int64, context.get_constant(numba.int64, 1)) + numba.targets.arrayobj.store_item(context, builder, util.index64tpe, context.get_constant(numba.int64, 0), util.arrayptr(context, builder, util.index64tpe, proxynext.starts)) + numba.targets.arrayobj.store_item(context, builder, util.index64tpe, length, util.arrayptr(context, builder, util.index64tpe, proxynext.stops)) + proxynext.content = arrayval + nextval = proxynext._getvalue() + + outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None) + + return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0))) + + else: + proxynext = numba.cgutils.create_struct_proxy(nexttpe)(context, builder) + proxynext.content = arrayval + proxynext.size = util.arraylen(context, builder, arraytpe, arrayval, totpe=numba.int64) + nextval = proxynext._getvalue() + + outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None) + + lenout = util.arraylen(context, builder, outtpe, outval) + outputptr = numba.cgutils.alloca_once(builder, context.get_value_type(outtpe.getitem_int())) + with builder.if_else(builder.icmp_signed("==", lenout, context.get_constant(numba.intp, 0)), likely=False) as (nothing, something): + with nothing: + builder.store(outtpe.lower_getitem_nothing(context, builder, outtpe, outval), outputptr) + with something: + builder.store(outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0))), outputptr) + return builder.load(outputptr) + +def lower_getitem_other(context, builder, sig, args): + rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args + arrayval, whereval = args + wrappedtpe = numba.types.Tuple((wheretpe,)) + wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,)) + return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval)) diff --git a/awkward1/_numba/util.py b/awkward1/_numba/util.py index 12e73cc5ed..0a02a92853 100644 --- a/awkward1/_numba/util.py +++ b/awkward1/_numba/util.py @@ -84,18 +84,6 @@ def call(context, builder, fcn, args, errormessage=None): with builder.if_then(builder.icmp_signed("!=", proxyerr.str, context.get_constant(numba.intp, 0)), likely=False): context.call_conv.return_user_exc(builder, ValueError, (errormessage,)) - # pyapi = context.get_python_api(builder) - # exc = pyapi.serialize_object(ValueError(errormessage)) - # excptr = context.call_conv._get_excinfo_argument(builder.function) - # if excptr.name == "excinfo" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.struct([llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)), llvmlite.llvmpy.core.Type.int(32)]))): - # builder.store(exc, excptr) - # builder.ret(numba.targets.callconv.RETCODE_USEREXC) - # elif excptr.name == "py_args" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)): - # pyapi.raise_object(exc) - # builder.ret(llvmlite.llvmpy.core.Constant.null(context.get_value_type(numba.types.pyobject))) - # else: - # raise AssertionError("unrecognized exception calling convention: {}".format(excptr)) - def newindex8(context, builder, lentpe, lenval): return numba.targets.arrayobj.numpy_empty_nd(context, builder, index8tpe(lentpe), (lenval,)) def newindexU8(context, builder, lentpe, lenval): @@ -125,6 +113,8 @@ def shapeat(shapeat, array, at, ndim): redat = at - (ndim - array.ndim) if redat < 0: return 1 + elif shapeat == 0: + return 0 elif shapeat == 1: return array.shape[redat] elif shapeat == array.shape[redat] or array.shape[redat] == 1: @@ -234,18 +224,3 @@ def preprocess_slicetuple(context, builder, wheretpe, whereval): whereval3 = context.call_internal(builder, cres2.fndesc, wheretpe3(wheretpe2), (whereval2,)) return wheretpe3, whereval3 - -def wrap_for_slicetuple(context, builder, arraytpe, arrayval): - import awkward1._numba.array.listarray - - length = arraylen(context, builder, arraytpe, arrayval, totpe=numba.int64) - nexttpe = awkward1._numba.array.listarray.ListArrayType(index64tpe, index64tpe, arraytpe, numba.types.none) - proxynext = numba.cgutils.create_struct_proxy(nexttpe)(context, builder) - proxynext.starts = newindex64(context, builder, numba.int64, context.get_constant(numba.int64, 1)) - proxynext.stops = newindex64(context, builder, numba.int64, context.get_constant(numba.int64, 1)) - numba.targets.arrayobj.store_item(context, builder, index64tpe, context.get_constant(numba.int64, 0), arrayptr(context, builder, index64tpe, proxynext.starts)) - numba.targets.arrayobj.store_item(context, builder, index64tpe, length, arrayptr(context, builder, index64tpe, proxynext.stops)) - proxynext.content = arrayval - nextval = proxynext._getvalue() - - return nexttpe, nextval diff --git a/awkward1/signatures/Content_8cpp.xml b/awkward1/signatures/Content_8cpp.xml index e77ead2724..be1b996d47 100644 --- a/awkward1/signatures/Content_8cpp.xml +++ b/awkward1/signatures/Content_8cpp.xml @@ -2,7 +2,9 @@ Content.cpp + awkward/array/RegularArray.h awkward/array/ListArray.h + awkward/array/EmptyArray.h awkward/type/ArrayType.h awkward/Content.h @@ -15,16 +17,26 @@ + + + + - + - + + + + - + + + + awkward diff --git a/awkward1/signatures/ListArray_8cpp.xml b/awkward1/signatures/ListArray_8cpp.xml index c6fe1cf6e0..86cba0382f 100644 --- a/awkward1/signatures/ListArray_8cpp.xml +++ b/awkward1/signatures/ListArray_8cpp.xml @@ -9,6 +9,7 @@ awkward/type/ListType.h awkward/Slice.h awkward/array/ListOffsetArray.h + awkward/array/RegularArray.h awkward/array/ListArray.h @@ -36,17 +37,22 @@ + + + + + - + diff --git a/awkward1/signatures/ListOffsetArray_8cpp.xml b/awkward1/signatures/ListOffsetArray_8cpp.xml index e006922065..edcdd673a6 100644 --- a/awkward1/signatures/ListOffsetArray_8cpp.xml +++ b/awkward1/signatures/ListOffsetArray_8cpp.xml @@ -9,6 +9,7 @@ awkward/type/ListType.h awkward/Slice.h awkward/array/ListArray.h + awkward/array/RegularArray.h awkward/array/ListOffsetArray.h @@ -20,6 +21,9 @@ + + + @@ -39,8 +43,10 @@ + + - + @@ -78,7 +84,7 @@ - + @@ -100,7 +106,7 @@ - + diff --git a/awkward1/signatures/NumpyArray_8cpp.xml b/awkward1/signatures/NumpyArray_8cpp.xml index 70eec7eb4a..b3aa0e8a62 100644 --- a/awkward1/signatures/NumpyArray_8cpp.xml +++ b/awkward1/signatures/NumpyArray_8cpp.xml @@ -111,7 +111,7 @@ - + @@ -141,7 +141,7 @@ - + @@ -171,7 +171,7 @@ - + const std::vector< ssize_t > @@ -188,7 +188,7 @@ - + const std::vector< ssize_t > @@ -205,7 +205,7 @@ - + diff --git a/awkward1/signatures/getitem_8cpp.xml b/awkward1/signatures/getitem_8cpp.xml index 2e5cdc8a84..6c286382ad 100644 --- a/awkward1/signatures/getitem_8cpp.xml +++ b/awkward1/signatures/getitem_8cpp.xml @@ -1898,7 +1898,7 @@ - + typename C @@ -1909,12 +1909,8 @@ ERROR ERROR awkward_listarray_getitem_next_array - (C *tooffsets, T *tocarry, T *toadvanced, const C *fromstarts, const C *fromstops, const T *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (T *tocarry, T *toadvanced, const C *fromstarts, const C *fromstops, const T *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarray_getitem_next_array - - C * - tooffsets - T * tocarry @@ -1961,17 +1957,13 @@ - + - + ERROR ERROR awkward_listarray32_getitem_next_array_64 - (int32_t *tooffsets, int64_t *tocarry, int64_t *toadvanced, const int32_t *fromstarts, const int32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (int64_t *tocarry, int64_t *toadvanced, const int32_t *fromstarts, const int32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarray32_getitem_next_array_64 - - int32_t * - tooffsets - int64_t * tocarry @@ -2018,17 +2010,13 @@ - + - + ERROR ERROR awkward_listarrayU32_getitem_next_array_64 - (uint32_t *tooffsets, int64_t *tocarry, int64_t *toadvanced, const uint32_t *fromstarts, const uint32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (int64_t *tocarry, int64_t *toadvanced, const uint32_t *fromstarts, const uint32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarrayU32_getitem_next_array_64 - - uint32_t * - tooffsets - int64_t * tocarry @@ -2075,17 +2063,13 @@ - + - + ERROR ERROR awkward_listarray64_getitem_next_array_64 - (int64_t *tooffsets, int64_t *tocarry, int64_t *toadvanced, const int64_t *fromstarts, const int64_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (int64_t *tocarry, int64_t *toadvanced, const int64_t *fromstarts, const int64_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarray64_getitem_next_array_64 - - int64_t * - tooffsets - int64_t * tocarry @@ -2132,7 +2116,7 @@ - + @@ -2197,7 +2181,7 @@ - + ERROR @@ -2254,7 +2238,7 @@ - + ERROR @@ -2311,7 +2295,7 @@ - + ERROR @@ -2368,7 +2352,7 @@ - + @@ -2425,7 +2409,7 @@ - + ERROR @@ -2474,7 +2458,7 @@ - + ERROR @@ -2523,7 +2507,7 @@ - + ERROR @@ -2572,7 +2556,7 @@ - + @@ -2606,7 +2590,7 @@ - + ERROR @@ -2635,7 +2619,7 @@ - + @@ -2677,7 +2661,7 @@ - + ERROR @@ -2714,7 +2698,7 @@ - + @@ -2748,7 +2732,7 @@ - + ERROR @@ -2777,7 +2761,7 @@ - + @@ -2811,7 +2795,7 @@ - + ERROR @@ -2840,7 +2824,7 @@ - + @@ -2882,7 +2866,7 @@ - + ERROR @@ -2919,7 +2903,7 @@ - + @@ -2965,7 +2949,7 @@ - + ERROR @@ -3006,7 +2990,7 @@ - + @@ -3040,7 +3024,7 @@ - + ERROR @@ -3069,7 +3053,7 @@ - + diff --git a/awkward1/signatures/libawkward_2util_8cpp.xml b/awkward1/signatures/libawkward_2util_8cpp.xml index f917759b77..2bc02bd37b 100644 --- a/awkward1/signatures/libawkward_2util_8cpp.xml +++ b/awkward1/signatures/libawkward_2util_8cpp.xml @@ -803,17 +803,13 @@ - + Error Error awkward::util::awkward_listarray_getitem_next_array_64< int32_t > - (int32_t *tooffsets, int64_t *tocarry, int64_t *toadvanced, const int32_t *fromstarts, const int32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (int64_t *tocarry, int64_t *toadvanced, const int32_t *fromstarts, const int32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarray_getitem_next_array_64< int32_t > - - int32_t * - tooffsets - int64_t * tocarry @@ -862,17 +858,13 @@ - + Error Error awkward::util::awkward_listarray_getitem_next_array_64< uint32_t > - (uint32_t *tooffsets, int64_t *tocarry, int64_t *toadvanced, const uint32_t *fromstarts, const uint32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (int64_t *tocarry, int64_t *toadvanced, const uint32_t *fromstarts, const uint32_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarray_getitem_next_array_64< uint32_t > - - uint32_t * - tooffsets - int64_t * tocarry @@ -921,17 +913,13 @@ - + Error Error awkward::util::awkward_listarray_getitem_next_array_64< int64_t > - (int64_t *tooffsets, int64_t *tocarry, int64_t *toadvanced, const int64_t *fromstarts, const int64_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) + (int64_t *tocarry, int64_t *toadvanced, const int64_t *fromstarts, const int64_t *fromstops, const int64_t *fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) awkward_listarray_getitem_next_array_64< int64_t > - - int64_t * - tooffsets - int64_t * tocarry diff --git a/include/awkward/Content.h b/include/awkward/Content.h index 1c4e8edc7d..6f32e71588 100644 --- a/include/awkward/Content.h +++ b/include/awkward/Content.h @@ -26,6 +26,7 @@ namespace awkward { virtual int64_t length() const = 0; virtual const std::shared_ptr shallow_copy() const = 0; virtual void check_for_iteration() const = 0; + virtual const std::shared_ptr getitem_nothing() const = 0; virtual const std::shared_ptr getitem_at(int64_t at) const = 0; virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const = 0; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const = 0; @@ -47,6 +48,8 @@ namespace awkward { virtual const std::shared_ptr getitem_next(const SliceNewAxis& newaxis, const Slice& tail, const Index64& advanced) const; virtual const std::shared_ptr getitem_next(const SliceArray64& array, const Slice& tail, const Index64& advanced) const = 0; + const std::shared_ptr getitem_next_array_wrap(const std::shared_ptr outcontent, const std::vector& shape) const; + }; } diff --git a/include/awkward/array/EmptyArray.h b/include/awkward/array/EmptyArray.h index 99f8ba0584..5a159c8745 100644 --- a/include/awkward/array/EmptyArray.h +++ b/include/awkward/array/EmptyArray.h @@ -27,6 +27,7 @@ namespace awkward { virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; virtual void check_for_iteration() const; + virtual const std::shared_ptr getitem_nothing() const; virtual const std::shared_ptr getitem_at(int64_t at) const; virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; diff --git a/include/awkward/array/ListArray.h b/include/awkward/array/ListArray.h index 86330d2391..9f96863277 100644 --- a/include/awkward/array/ListArray.h +++ b/include/awkward/array/ListArray.h @@ -34,6 +34,7 @@ namespace awkward { virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; virtual void check_for_iteration() const; + virtual const std::shared_ptr getitem_nothing() const; virtual const std::shared_ptr getitem_at(int64_t at) const; virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; diff --git a/include/awkward/array/ListOffsetArray.h b/include/awkward/array/ListOffsetArray.h index 08f9f5a4d0..980c0ad7b1 100644 --- a/include/awkward/array/ListOffsetArray.h +++ b/include/awkward/array/ListOffsetArray.h @@ -32,6 +32,7 @@ namespace awkward { virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; virtual void check_for_iteration() const; + virtual const std::shared_ptr getitem_nothing() const; virtual const std::shared_ptr getitem_at(int64_t at) const; virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; diff --git a/include/awkward/array/NumpyArray.h b/include/awkward/array/NumpyArray.h index 5dd2586c60..87ffd861e9 100644 --- a/include/awkward/array/NumpyArray.h +++ b/include/awkward/array/NumpyArray.h @@ -51,6 +51,7 @@ namespace awkward { virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; virtual void check_for_iteration() const; + virtual const std::shared_ptr getitem_nothing() const; virtual const std::shared_ptr getitem_at(int64_t at) const; virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; diff --git a/include/awkward/array/RawArray.h b/include/awkward/array/RawArray.h index 76deb72230..95df4044c4 100644 --- a/include/awkward/array/RawArray.h +++ b/include/awkward/array/RawArray.h @@ -226,6 +226,10 @@ namespace awkward { } } + virtual const std::shared_ptr getitem_nothing() const { + return getitem_range_nowrap(0, 0); + } + virtual const std::shared_ptr getitem_at(int64_t at) const { int64_t regular_at = at; if (regular_at < 0) { diff --git a/include/awkward/array/RegularArray.h b/include/awkward/array/RegularArray.h index 8b54004bfd..afa0bf9540 100644 --- a/include/awkward/array/RegularArray.h +++ b/include/awkward/array/RegularArray.h @@ -33,6 +33,7 @@ namespace awkward { virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; virtual void check_for_iteration() const; + virtual const std::shared_ptr getitem_nothing() const; virtual const std::shared_ptr getitem_at(int64_t at) const; virtual const std::shared_ptr getitem_at_nowrap(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; diff --git a/include/awkward/cpu-kernels/getitem.h b/include/awkward/cpu-kernels/getitem.h index 092440a42b..56de06e3ae 100644 --- a/include/awkward/cpu-kernels/getitem.h +++ b/include/awkward/cpu-kernels/getitem.h @@ -47,9 +47,9 @@ extern "C" { struct Error awkward_listarrayU32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const uint32_t* fromoffsets, int64_t lenstarts); struct Error awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts); - struct Error awkward_listarray32_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); - struct Error awkward_listarrayU32_getitem_next_array_64(uint32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); - struct Error awkward_listarray64_getitem_next_array_64(int64_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); + struct Error awkward_listarray32_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); + struct Error awkward_listarrayU32_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); + struct Error awkward_listarray64_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); struct Error awkward_listarray32_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); struct Error awkward_listarrayU32_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); diff --git a/include/awkward/cpu-kernels/identity.h b/include/awkward/cpu-kernels/identity.h index 4066dff4c1..14b17a4dac 100644 --- a/include/awkward/cpu-kernels/identity.h +++ b/include/awkward/cpu-kernels/identity.h @@ -11,6 +11,11 @@ extern "C" { struct Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length, int64_t width); + struct Error awkward_identity32_from_listoffsetarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); + struct Error awkward_identity64_from_listoffsetarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); + struct Error awkward_identity64_from_listoffsetarrayU32(int64_t* toptr, const int64_t* fromptr, const uint32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); + struct Error awkward_identity64_from_listoffsetarray64(int64_t* toptr, const int64_t* fromptr, const int64_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); + struct Error awkward_identity32_from_listarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); struct Error awkward_identity64_from_listarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); struct Error awkward_identity64_from_listarrayU32(int64_t* toptr, const int64_t* fromptr, const uint32_t* fromstarts, const uint32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); diff --git a/include/awkward/util.h b/include/awkward/util.h index 77030bc032..4ba327fa2b 100644 --- a/include/awkward/util.h +++ b/include/awkward/util.h @@ -27,6 +27,8 @@ namespace awkward { void operator()(T const *p) { } }; + template + Error awkward_identity64_from_listoffsetarray(int64_t* toptr, const int64_t* fromptr, const T* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); template Error awkward_identity64_from_listarray(int64_t* toptr, const int64_t* fromptr, const T* fromstarts, const T* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); template @@ -40,7 +42,7 @@ namespace awkward { template Error awkward_listarray_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const T* fromoffsets, int64_t lenstarts); template - Error awkward_listarray_getitem_next_array_64(T* tooffsets, int64_t* tocarry, int64_t* toadvanced, const T* fromstarts, const T* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); + Error awkward_listarray_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const T* fromstarts, const T* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); template Error awkward_listarray_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const T* fromstarts, const T* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); template diff --git a/src/cpu-kernels/getitem.cpp b/src/cpu-kernels/getitem.cpp index 4282c253e0..bc0d3ce7f4 100644 --- a/src/cpu-kernels/getitem.cpp +++ b/src/cpu-kernels/getitem.cpp @@ -269,24 +269,31 @@ template ERROR awkward_listarray_getitem_next_range(C* tooffsets, T* tocarry, const C* fromstarts, const C* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { int64_t k = 0; tooffsets[0] = 0; - for (int64_t i = 0; i < lenstarts; i++) { - int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; - int64_t regular_start = start; - int64_t regular_stop = stop; - awkward_regularize_rangeslice(®ular_start, ®ular_stop, step > 0, start != kSliceNone, stop != kSliceNone, length); - if (step > 0) { // FIXME: put this test outside the for loop + if (step > 0) { + for (int64_t i = 0; i < lenstarts; i++) { + int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; + int64_t regular_start = start; + int64_t regular_stop = stop; + awkward_regularize_rangeslice(®ular_start, ®ular_stop, step > 0, start != kSliceNone, stop != kSliceNone, length); for (int64_t j = regular_start; j < regular_stop; j += step) { tocarry[k] = fromstarts[startsoffset + i] + j; k++; } + tooffsets[i + 1] = (C)k; } - else { + } + else { + for (int64_t i = 0; i < lenstarts; i++) { + int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; + int64_t regular_start = start; + int64_t regular_stop = stop; + awkward_regularize_rangeslice(®ular_start, ®ular_stop, step > 0, start != kSliceNone, stop != kSliceNone, length); for (int64_t j = regular_start; j > regular_stop; j += step) { tocarry[k] = fromstarts[startsoffset + i] + j; k++; } + tooffsets[i + 1] = (C)k; } - tooffsets[i + 1] = (C)k; } return success(); } @@ -339,8 +346,7 @@ ERROR awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanc } template -ERROR awkward_listarray_getitem_next_array(C* tooffsets, T* tocarry, T* toadvanced, const C* fromstarts, const C* fromstops, const T* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - tooffsets[0] = 0; +ERROR awkward_listarray_getitem_next_array(T* tocarry, T* toadvanced, const C* fromstarts, const C* fromstops, const T* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { for (int64_t i = 0; i < lenstarts; i++) { if (fromstops[stopsoffset + i] < fromstarts[startsoffset + i]) { return failure("stops[i] < starts[i]", i, kSliceNone); @@ -360,18 +366,17 @@ ERROR awkward_listarray_getitem_next_array(C* tooffsets, T* tocarry, T* toadvanc tocarry[i*lenarray + j] = fromstarts[startsoffset + i] + regular_at; toadvanced[i*lenarray + j] = j; } - tooffsets[i + 1] = (C)((i + 1)*lenarray); } return success(); } -ERROR awkward_listarray32_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - return awkward_listarray_getitem_next_array(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); +ERROR awkward_listarray32_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { + return awkward_listarray_getitem_next_array(tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); } -ERROR awkward_listarrayU32_getitem_next_array_64(uint32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - return awkward_listarray_getitem_next_array(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); +ERROR awkward_listarrayU32_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { + return awkward_listarray_getitem_next_array(tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); } -ERROR awkward_listarray64_getitem_next_array_64(int64_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - return awkward_listarray_getitem_next_array(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); +ERROR awkward_listarray64_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { + return awkward_listarray_getitem_next_array(tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); } template @@ -384,12 +389,6 @@ ERROR awkward_listarray_getitem_next_array_advanced(T* tocarry, T* toadvanced, c return failure("stops[i] > len(content)", i, kSliceNone); } int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; - if (fromadvanced[i] >= lenarray) { - // FIXME: this might be weaker than it should be: the length of each advanced array should - // be exactly the same, and I think it was already checked when creating the Slice. - // If so, this check would be redundant. If not, it's not strong enough. - return failure("lengths of advanced indexes must match", i, kSliceNone); - } int64_t regular_at = fromarray[fromadvanced[i]]; if (regular_at < 0) { regular_at += length; diff --git a/src/cpu-kernels/identity.cpp b/src/cpu-kernels/identity.cpp index 4a3cd7e6f1..7dc70b9c10 100644 --- a/src/cpu-kernels/identity.cpp +++ b/src/cpu-kernels/identity.cpp @@ -23,6 +23,44 @@ ERROR awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, i return success(); } +template +ERROR awkward_identity_from_listoffsetarray(ID* toptr, const ID* fromptr, const T* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + int64_t globalstart = fromoffsets[offsetsoffset]; + int64_t globalstop = fromoffsets[offsetsoffset + fromlength]; + for (int64_t k = 0; k < globalstart*(fromwidth + 1); k++) { + toptr[k] = -1; + } + for (int64_t k = globalstop*(fromwidth + 1); k < tolength*(fromwidth + 1); k++) { + toptr[k] = -1; + } + for (int64_t i = 0; i < fromlength; i++) { + int64_t start = fromoffsets[offsetsoffset + i]; + int64_t stop = fromoffsets[offsetsoffset + i + 1]; + if (start != stop && stop > tolength) { + return failure("max(stop) > len(content)", i, kSliceNone); + } + for (int64_t j = start; j < stop; j++) { + for (int64_t k = 0; k < fromwidth; k++) { + toptr[j*(fromwidth + 1) + k] = fromptr[fromptroffset + i*(fromwidth) + k]; + } + toptr[j*(fromwidth + 1) + fromwidth] = ID(j - start); + } + } + return success(); +} +ERROR awkward_identity32_from_listoffsetarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity_from_listoffsetarray(toptr, fromptr, fromoffsets, fromptroffset, offsetsoffset, tolength, fromlength, fromwidth); +} +ERROR awkward_identity64_from_listoffsetarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity_from_listoffsetarray(toptr, fromptr, fromoffsets, fromptroffset, offsetsoffset, tolength, fromlength, fromwidth); +} +ERROR awkward_identity64_from_listoffsetarrayU32(int64_t* toptr, const int64_t* fromptr, const uint32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity_from_listoffsetarray(toptr, fromptr, fromoffsets, fromptroffset, offsetsoffset, tolength, fromlength, fromwidth); +} +ERROR awkward_identity64_from_listoffsetarray64(int64_t* toptr, const int64_t* fromptr, const int64_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity_from_listoffsetarray(toptr, fromptr, fromoffsets, fromptroffset, offsetsoffset, tolength, fromlength, fromwidth); +} + template ERROR awkward_identity_from_listarray(ID* toptr, const ID* fromptr, const T* fromstarts, const T* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { for (int64_t k = 0; k < tolength*(fromwidth + 1); k++) { @@ -35,6 +73,9 @@ ERROR awkward_identity_from_listarray(ID* toptr, const ID* fromptr, const T* fro return failure("max(stop) > len(content)", i, kSliceNone); } for (int64_t j = start; j < stop; j++) { + if (toptr[j*(fromwidth + 1) + fromwidth] != -1) { + return failure("item has ambiguous identity", i, kSliceNone); + } for (int64_t k = 0; k < fromwidth; k++) { toptr[j*(fromwidth + 1) + k] = fromptr[fromptroffset + i*(fromwidth) + k]; } diff --git a/src/libawkward/Content.cpp b/src/libawkward/Content.cpp index a6149ce0de..12c562583b 100644 --- a/src/libawkward/Content.cpp +++ b/src/libawkward/Content.cpp @@ -1,6 +1,8 @@ // BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE +#include "awkward/array/RegularArray.h" #include "awkward/array/ListArray.h" +#include "awkward/array/EmptyArray.h" #include "awkward/type/ArrayType.h" #include "awkward/Content.h" @@ -47,17 +49,19 @@ namespace awkward { } const std::shared_ptr Content::getitem(const Slice& where) const { - Index64 nextstarts(1); - Index64 nextstops(1); - *nextstarts.ptr().get() = 0; - *nextstops.ptr().get() = length(); - std::shared_ptr next(new ListArrayOf(std::shared_ptr(nullptr), nextstarts, nextstops, shallow_copy())); + std::shared_ptr next(new RegularArray(Identity::none(), shallow_copy(), length())); std::shared_ptr nexthead = where.head(); Slice nexttail = where.tail(); Index64 nextadvanced(0); std::shared_ptr out = next.get()->getitem_next(nexthead, nexttail, nextadvanced); - return out.get()->getitem_at(0); + + if (out.get()->length() == 0) { + return out.get()->getitem_nothing(); + } + else { + return out.get()->getitem_at_nowrap(0); + } } const std::shared_ptr Content::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { @@ -108,6 +112,17 @@ namespace awkward { } const std::shared_ptr Content::getitem_next(const SliceNewAxis& newaxis, const Slice& tail, const Index64& advanced) const { - throw std::runtime_error("FIXME: insert a RegularArray of 1 here"); + std::shared_ptr nexthead = tail.head(); + Slice nexttail = tail.tail(); + return std::shared_ptr(new RegularArray(Identity::none(), getitem_next(nexthead, nexttail, advanced), 1)); } + + const std::shared_ptr Content::getitem_next_array_wrap(const std::shared_ptr outcontent, const std::vector& shape) const { + std::shared_ptr out(new RegularArray(Identity::none(), outcontent, (int64_t)shape[shape.size() - 1])); + for (int64_t i = (int64_t)shape.size() - 2; i >= 0; i--) { + out = std::shared_ptr(new RegularArray(Identity::none(), out, (int64_t)shape[i])); + } + return out; + } + } diff --git a/src/libawkward/Slice.cpp b/src/libawkward/Slice.cpp index 4062225662..ce4975c372 100644 --- a/src/libawkward/Slice.cpp +++ b/src/libawkward/Slice.cpp @@ -207,7 +207,10 @@ namespace awkward { else { std::vector arrayshape = array->shape(); for (size_t j = 0; j < shape.size(); j++) { - if (arrayshape[j] > shape[j]) { + if (arrayshape[j] == 0) { + shape[j] = 0; + } + else if (arrayshape[j] > shape[j]) { shape[j] = arrayshape[j]; } } diff --git a/src/libawkward/array/EmptyArray.cpp b/src/libawkward/array/EmptyArray.cpp index 754c2a1575..5961ec8611 100644 --- a/src/libawkward/array/EmptyArray.cpp +++ b/src/libawkward/array/EmptyArray.cpp @@ -35,8 +35,7 @@ namespace awkward { } void EmptyArray::tojson_part(ToJson& builder) const { - builder.beginlist(); - builder.endlist(); + // Do nothing (builder.beginlist() and builder.endlist() are called outside of tojson_part). } const std::shared_ptr EmptyArray::type_part() const { @@ -53,6 +52,10 @@ namespace awkward { void EmptyArray::check_for_iteration() const { } + const std::shared_ptr EmptyArray::getitem_nothing() const { + return shallow_copy(); + } + const std::shared_ptr EmptyArray::getitem_at(int64_t at) const { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); return std::shared_ptr(nullptr); // make Windows compiler happy diff --git a/src/libawkward/array/ListArray.cpp b/src/libawkward/array/ListArray.cpp index 5c03cb7806..72d635fbbe 100644 --- a/src/libawkward/array/ListArray.cpp +++ b/src/libawkward/array/ListArray.cpp @@ -8,6 +8,7 @@ #include "awkward/type/ListType.h" #include "awkward/Slice.h" #include "awkward/array/ListOffsetArray.h" +#include "awkward/array/RegularArray.h" #include "awkward/array/ListArray.h" @@ -150,7 +151,8 @@ namespace awkward { template void ListArrayOf::tojson_part(ToJson& builder) const { - for (int64_t i = 0; i < length(); i++) { + int64_t len = length(); + for (int64_t i = 0; i < len; i++) { builder.beginlist(); getitem_at_nowrap(i).get()->tojson_part(builder); builder.endlist(); @@ -182,6 +184,11 @@ namespace awkward { } } + template + const std::shared_ptr ListArrayOf::getitem_nothing() const { + return content_.get()->getitem_range_nowrap(0, 0); + } + template const std::shared_ptr ListArrayOf::getitem_at(int64_t at) const { int64_t regular_at = at; @@ -375,9 +382,7 @@ namespace awkward { if (advanced.length() == 0) { Index64 nextcarry(lenstarts*flathead.length()); Index64 nextadvanced(lenstarts*flathead.length()); - IndexOf nextoffsets(lenstarts + 1); // FIXME: offsets are regular; don't generate them and replace ListOffsetArray output with a RegularArray struct Error err = util::awkward_listarray_getitem_next_array_64( - nextoffsets.ptr().get(), nextcarry.ptr().get(), nextadvanced.ptr().get(), starts_.ptr().get(), @@ -390,8 +395,7 @@ namespace awkward { content_.get()->length()); util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); + return getitem_next_array_wrap(nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced), array.shape()); } else { Index64 nextcarry(lenstarts); diff --git a/src/libawkward/array/ListOffsetArray.cpp b/src/libawkward/array/ListOffsetArray.cpp index 67fb263e37..8c0711f60c 100644 --- a/src/libawkward/array/ListOffsetArray.cpp +++ b/src/libawkward/array/ListOffsetArray.cpp @@ -8,6 +8,7 @@ #include "awkward/type/ListType.h" #include "awkward/Slice.h" #include "awkward/array/ListArray.h" +#include "awkward/array/RegularArray.h" #include "awkward/array/ListOffsetArray.h" @@ -47,8 +48,6 @@ namespace awkward { if (length() != id.get()->length()) { util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } - Index32 starts = make_starts(offsets_); - Index32 stops = make_stops(offsets_); std::shared_ptr bigid = id; if (content_.get()->length() > kMaxInt32) { bigid = id.get()->to64(); @@ -56,14 +55,12 @@ namespace awkward { if (Identity32* rawid = dynamic_cast(bigid.get())) { Identity32* rawsubid = new Identity32(Identity::newref(), rawid->fieldloc(), rawid->width() + 1, content_.get()->length()); std::shared_ptr subid(rawsubid); - struct Error err = awkward_identity32_from_listarray32( + struct Error err = awkward_identity32_from_listoffsetarray32( rawsubid->ptr().get(), rawid->ptr().get(), - starts.ptr().get(), - stops.ptr().get(), + offsets_.ptr().get(), rawid->offset(), - starts.offset(), - stops.offset(), + offsets_.offset(), content_.get()->length(), length(), rawid->width()); @@ -73,14 +70,12 @@ namespace awkward { else if (Identity64* rawid = dynamic_cast(bigid.get())) { Identity64* rawsubid = new Identity64(Identity::newref(), rawid->fieldloc(), rawid->width() + 1, content_.get()->length()); std::shared_ptr subid(rawsubid); - struct Error err = awkward_identity64_from_listarray32( + struct Error err = awkward_identity64_from_listoffsetarray32( rawsubid->ptr().get(), rawid->ptr().get(), - starts.ptr().get(), - stops.ptr().get(), + offsets_.ptr().get(), rawid->offset(), - starts.offset(), - stops.offset(), + offsets_.offset(), content_.get()->length(), length(), rawid->width()); @@ -103,20 +98,16 @@ namespace awkward { if (length() != id.get()->length()) { util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } - IndexOf starts = make_starts(offsets_); - IndexOf stops = make_stops(offsets_); std::shared_ptr bigid = id.get()->to64(); if (Identity64* rawid = dynamic_cast(bigid.get())) { Identity64* rawsubid = new Identity64(Identity::newref(), rawid->fieldloc(), rawid->width() + 1, content_.get()->length()); std::shared_ptr subid(rawsubid); - struct Error err = util::awkward_identity64_from_listarray( + struct Error err = util::awkward_identity64_from_listoffsetarray( rawsubid->ptr().get(), rawid->ptr().get(), - starts.ptr().get(), - stops.ptr().get(), + offsets_.ptr().get(), rawid->offset(), - starts.offset(), - stops.offset(), + offsets_.offset(), content_.get()->length(), length(), rawid->width()); @@ -163,7 +154,8 @@ namespace awkward { template void ListOffsetArrayOf::tojson_part(ToJson& builder) const { - for (int64_t i = 0; i < length(); i++) { + int64_t len = length(); + for (int64_t i = 0; i < len; i++) { builder.beginlist(); getitem_at_nowrap(i).get()->tojson_part(builder); builder.endlist(); @@ -192,6 +184,11 @@ namespace awkward { } } + template + const std::shared_ptr ListOffsetArrayOf::getitem_nothing() const { + return content_.get()->getitem_range_nowrap(0, 0); + } + template const std::shared_ptr ListOffsetArrayOf::getitem_at(int64_t at) const { int64_t regular_at = at; @@ -371,10 +368,7 @@ namespace awkward { if (advanced.length() == 0) { Index64 nextcarry(lenstarts*flathead.length()); Index64 nextadvanced(lenstarts*flathead.length()); - IndexOf nextoffsets(lenstarts + 1); // FIXME: offsets are regular; don't generate them and replace ListOffsetArray output with a RegularArray - IndexOf nextstops(lenstarts); struct Error err = util::awkward_listarray_getitem_next_array_64( - nextoffsets.ptr().get(), nextcarry.ptr().get(), nextadvanced.ptr().get(), starts.ptr().get(), @@ -387,8 +381,7 @@ namespace awkward { content_.get()->length()); util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray - return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); + return getitem_next_array_wrap(nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced), array.shape()); } else { Index64 nextcarry(lenstarts); diff --git a/src/libawkward/array/NumpyArray.cpp b/src/libawkward/array/NumpyArray.cpp index 275c13b2e7..3330704682 100644 --- a/src/libawkward/array/NumpyArray.cpp +++ b/src/libawkward/array/NumpyArray.cpp @@ -187,6 +187,10 @@ namespace awkward { return out.str(); } + // FIXME: turn each of these three functions into (builder, array, offset, length) + // so that the can be called once per multidimensional array (no getitem_at_nowrap; + // do it internally in tojson_boolean/integer/real). + void tojson_boolean(ToJson& builder, bool* array, int64_t length) { for (int i = 0; i < length; i++) { builder.boolean(array[i]); @@ -261,7 +265,8 @@ namespace awkward { } } else { - for (int64_t i = 0; i < length(); i++) { + int64_t len = length(); + for (int64_t i = 0; i < len; i++) { builder.beginlist(); getitem_at_nowrap(i).get()->tojson_part(builder); builder.endlist(); @@ -350,6 +355,16 @@ namespace awkward { } } + const std::shared_ptr NumpyArray::getitem_nothing() const { + const std::vector shape({ 0 }); + const std::vector strides({ itemsize_ }); + std::shared_ptr id; + if (id_.get() != nullptr) { + id = id_.get()->getitem_range_nowrap(0, 0); + } + return std::shared_ptr(new NumpyArray(id, ptr_, shape, strides, byteoffset_, itemsize_, format_)); + } + const std::shared_ptr NumpyArray::getitem_at(int64_t at) const { assert(!isscalar()); int64_t regular_at = at; @@ -905,7 +920,7 @@ namespace awkward { for (auto x = arrayshape.rbegin(); x != arrayshape.rend(); ++x) { outstrides.insert(outstrides.begin(), ((ssize_t)(*x))*outstrides[0]); } - return NumpyArray(out.id_, out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); + return NumpyArray(arrayshape.size() == 1 ? out.id_ : Identity::none(), out.ptr_, outshape, outstrides, out.byteoffset_, itemsize_, format_); } else { diff --git a/src/libawkward/array/RegularArray.cpp b/src/libawkward/array/RegularArray.cpp index 0db653683d..f7f3c8e1b4 100644 --- a/src/libawkward/array/RegularArray.cpp +++ b/src/libawkward/array/RegularArray.cpp @@ -91,7 +91,12 @@ namespace awkward { } void RegularArray::tojson_part(ToJson& builder) const { - throw std::runtime_error("tojson_part"); + int64_t len = length(); + for (int64_t i = 0; i < len; i++) { + builder.beginlist(); + getitem_at_nowrap(i).get()->tojson_part(builder); + builder.endlist(); + } } const std::shared_ptr RegularArray::type_part() const { @@ -99,7 +104,7 @@ namespace awkward { } int64_t RegularArray::length() const { - return content_.get()->length() / size_; // floor of length / size + return size_ == 0 ? 0 : content_.get()->length() / size_; // floor of length / size } const std::shared_ptr RegularArray::shallow_copy() const { @@ -108,6 +113,10 @@ namespace awkward { void RegularArray::check_for_iteration() const { } + const std::shared_ptr RegularArray::getitem_nothing() const { + return content_.get()->getitem_range_nowrap(0, 0); + } + const std::shared_ptr RegularArray::getitem_at(int64_t at) const { int64_t regular_at = at; int64_t len = length(); @@ -267,7 +276,8 @@ namespace awkward { util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); - return std::shared_ptr(new RegularArray(id_, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced), flathead.length())); + + return getitem_next_array_wrap(nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced), array.shape()); } else { Index64 nextcarry(len); diff --git a/src/libawkward/util.cpp b/src/libawkward/util.cpp index cf4438f557..ae48336797 100644 --- a/src/libawkward/util.cpp +++ b/src/libawkward/util.cpp @@ -32,6 +32,15 @@ namespace awkward { } } + template <> + Error awkward_identity64_from_listoffsetarray(int64_t* toptr, const int64_t* fromptr, const uint32_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity64_from_listoffsetarrayU32(toptr, fromptr, fromoffsets, fromptroffset, offsetsoffset, tolength, fromlength, fromwidth); + } + template <> + Error awkward_identity64_from_listoffsetarray(int64_t* toptr, const int64_t* fromptr, const int64_t* fromoffsets, int64_t fromptroffset, int64_t offsetsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { + return awkward_identity64_from_listoffsetarray64(toptr, fromptr, fromoffsets, fromptroffset, offsetsoffset, tolength, fromlength, fromwidth); + } + template <> Error awkward_identity64_from_listarray(int64_t* toptr, const int64_t* fromptr, const uint32_t* fromstarts, const uint32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { return awkward_identity64_from_listarrayU32(toptr, fromptr, fromstarts, fromstops, fromptroffset, startsoffset, stopsoffset, tolength, fromlength, fromwidth); @@ -107,16 +116,16 @@ namespace awkward { } template <> - Error awkward_listarray_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - return awkward_listarray32_getitem_next_array_64(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); + Error awkward_listarray_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { + return awkward_listarray32_getitem_next_array_64(tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); } template <> - Error awkward_listarray_getitem_next_array_64(uint32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - return awkward_listarrayU32_getitem_next_array_64(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); + Error awkward_listarray_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const uint32_t* fromstarts, const uint32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { + return awkward_listarrayU32_getitem_next_array_64(tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); } template <> - Error awkward_listarray_getitem_next_array_64(int64_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { - return awkward_listarray64_getitem_next_array_64(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); + Error awkward_listarray_getitem_next_array_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { + return awkward_listarray64_getitem_next_array_64(tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); } template <> diff --git a/src/pyawkward.cpp b/src/pyawkward.cpp index 3fd62ae1b2..53d1066f54 100644 --- a/src/pyawkward.cpp +++ b/src/pyawkward.cpp @@ -398,16 +398,17 @@ void toslice_part(ak::Slice& slice, py::object obj) { else { std::string format(info.format); format.erase(0, format.find_first_not_of("@=<>!")); - if (format.compare("c") != 0 && - format.compare("b") != 0 && - format.compare("B") != 0 && - format.compare("h") != 0 && - format.compare("H") != 0 && - format.compare("i") != 0 && - format.compare("I") != 0 && - format.compare("l") != 0 && - format.compare("L") != 0 && - format.compare("q") != 0 && + if (py::isinstance(obj) && + format.compare("c") != 0 && + format.compare("b") != 0 && + format.compare("B") != 0 && + format.compare("h") != 0 && + format.compare("H") != 0 && + format.compare("i") != 0 && + format.compare("I") != 0 && + format.compare("l") != 0 && + format.compare("L") != 0 && + format.compare("q") != 0 && format.compare("Q") != 0) { throw std::invalid_argument("arrays used as an index must be integer or boolean"); } diff --git a/tests/test_PR013_error_handling_struct.py b/tests/test_PR013_error_handling_struct.py index 01f8882dc3..8606edeafc 100644 --- a/tests/test_PR013_error_handling_struct.py +++ b/tests/test_PR013_error_handling_struct.py @@ -107,11 +107,11 @@ def test_listarray_numpyarray(): with pytest.raises(ValueError) as excinfo: array[20,] - assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get 20, index out of range" with pytest.raises(ValueError) as excinfo: array[-20,] - assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get -20, index out of range" array[-20:20,] @@ -121,11 +121,11 @@ def test_listarray_numpyarray(): with pytest.raises(ValueError) as excinfo: array[[2, 0, 0, 20, 3]] - assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get 20, index out of range" with pytest.raises(ValueError) as excinfo: array[[2, 0, 0, -20, 3]] - assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get -20, index out of range" starts = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6])) stops = awkward1.layout.Index64(numpy.array([3, 3, 5, 6, 10])) @@ -184,7 +184,7 @@ def test_listarray_listarray_numpyarray(): with pytest.raises(ValueError) as excinfo: array2[20,] - assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get 20, index out of range" with pytest.raises(ValueError) as excinfo: array2[2, 20] @@ -196,7 +196,7 @@ def test_listarray_listarray_numpyarray(): with pytest.raises(ValueError) as excinfo: array2[-20,] - assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get -20, index out of range" with pytest.raises(ValueError) as excinfo: array2[2, -20] @@ -214,7 +214,7 @@ def test_listarray_listarray_numpyarray(): with pytest.raises(ValueError) as excinfo: array2[20,] - assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get 20, index out of range" with pytest.raises(ValueError) as excinfo: array2[2, 20] @@ -230,7 +230,7 @@ def test_listarray_listarray_numpyarray(): with pytest.raises(ValueError) as excinfo: array2[-20,] - assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + assert str(excinfo.value) == "in RegularArray attempting to get -20, index out of range" with pytest.raises(ValueError) as excinfo: array2[2, -20] @@ -266,4 +266,4 @@ def f1(q): with pytest.raises(ValueError) as excinfo: f1(array) - assert str(excinfo.value) == "in ListArray64, indexing error" + assert str(excinfo.value) == "in RegularArray, indexing error" diff --git a/tests/test_PR016_finish_getitem_for_rawarray.cpp b/tests/test_PR016_finish_getitem_for_rawarray.cpp index ef44b6c4a3..dba63dd0e0 100644 --- a/tests/test_PR016_finish_getitem_for_rawarray.cpp +++ b/tests/test_PR016_finish_getitem_for_rawarray.cpp @@ -147,9 +147,6 @@ int main(int, char**) { return -1; } - // std::cout << tostring(listA.get()->getitem(slice(new SliceArray64(array1, std::vector({4}), std::vector({1})), new SliceArray64(array2, std::vector({4}), std::vector({1}))))) << std::endl; - // return -1; - std::shared_ptr listB(new ListOffsetArray32(Identity::none(), offsetsB, listA)); if (tostring(listB) != "[[[0, 1.1, 2.2], [], [3.3, 4.4]], [[5.5]], [], [[6.6, 7.7, 8.8, 9.9]]]") { return -1; diff --git a/tests/test_PR023_regular_array.py b/tests/test_PR023_regular_array.py index b40a146698..b698027f83 100644 --- a/tests/test_PR023_regular_array.py +++ b/tests/test_PR023_regular_array.py @@ -22,6 +22,9 @@ def test_type(): def test_iteration(): assert awkward1.tolist(regulararray) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] +def test_tojson(): + assert awkward1.tojson(regulararray) == "[[[0.0,1.1,2.2],[]],[[3.3,4.4],[5.5]],[[6.6,7.7,8.8,9.9],[]]]" + def test_getitem_at(): assert awkward1.tolist(regulararray[0]) == [[0.0, 1.1, 2.2], []] assert awkward1.tolist(regulararray[1]) == [[3.3, 4.4], [5.5]] @@ -178,114 +181,104 @@ def test_numba(): stops = awkward1.layout.Index64(numpy.array([2, 3])) listarray = awkward1.layout.ListArray64(starts, stops, regulararray) - for z in range(2): - @numba.njit - def f1(q): - return 3.14 - - assert sys.getrefcount(regulararray) == 2 - f1(regulararray) - assert sys.getrefcount(regulararray) == 2 - - @numba.njit - def f2(q): - return q + @numba.njit + def f1(q): + return 3.14 - assert sys.getrefcount(regulararray) == 2 - assert awkward1.tolist(f2(regulararray)) == awkward1.tolist(regulararray) - assert sys.getrefcount(regulararray) == 2 + assert sys.getrefcount(regulararray) == 2 + f1(regulararray) + assert sys.getrefcount(regulararray) == 2 - @numba.njit - def f3(q): - return len(q) + @numba.njit + def f2(q): + return q - assert f3(regulararray) == 3 - assert f3(regulararray_m1) == 2 - assert f3(regulararray_m2) == 2 - assert len(regulararray) == 3 - assert len(regulararray_m1) == 2 - assert len(regulararray_m2) == 2 + assert sys.getrefcount(regulararray) == 2 + assert awkward1.tolist(f2(regulararray)) == awkward1.tolist(regulararray) + assert sys.getrefcount(regulararray) == 2 - @numba.njit - def f4(q): - return q[1] + @numba.njit + def f3(q): + return len(q) - assert awkward1.tolist(f4(regulararray)) == [[3.3, 4.4], [5.5]] + assert f3(regulararray) == 3 + assert f3(regulararray_m1) == 2 + assert f3(regulararray_m2) == 2 + assert len(regulararray) == 3 + assert len(regulararray_m1) == 2 + assert len(regulararray_m2) == 2 - @numba.njit - def f5(q, i): - return q[i] + @numba.njit + def f4(q): + return q[1] - assert awkward1.tolist(f5(regulararray, 1)) == [[3.3, 4.4], [5.5]] + assert awkward1.tolist(f4(regulararray)) == [[3.3, 4.4], [5.5]] - @numba.njit - def f6(q, i): - return q[i] + @numba.njit + def f5(q, i): + return q[i] - assert awkward1.tolist(f6(regulararray, slice(1, None))) == awkward1.tolist(regulararray[slice(1, None)]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] - assert awkward1.tolist(f6(regulararray, slice(None, -1))) == awkward1.tolist(regulararray[slice(None, -1)]) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]] + assert awkward1.tolist(f5(regulararray, 1)) == [[3.3, 4.4], [5.5]] - @numba.njit - def f7(q, i): - return q[(i,)] + @numba.njit + def f6(q, i): + return q[i] - assert awkward1.tolist(f7(regulararray, 1)) == [[3.3, 4.4], [5.5]] + assert awkward1.tolist(f6(regulararray, slice(1, None))) == awkward1.tolist(regulararray[slice(1, None)]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(f6(regulararray, slice(None, -1))) == awkward1.tolist(regulararray[slice(None, -1)]) == [[[0.0, 1.1, 2.2], []], [[3.3, 4.4], [5.5]]] - @numba.njit - def f8(q, i): - return q[:, i] + @numba.njit + def f7(q, i): + return q[(i,)] - assert awkward1.tolist(f8(listarray, 1)) == awkward1.tolist(listarray[:, 1]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(f7(regulararray, 1)) == [[3.3, 4.4], [5.5]] - @numba.njit - def f9(q): - return q[:, 1, 0] + @numba.njit + def f8(q, i): + return q[:, i] - assert awkward1.tolist(listarray[:, 1, 0]) == awkward1.tolist(f9(listarray)) == [[3.3, 4.4], [6.6, 7.7, 8.8, 9.9]] + assert awkward1.tolist(f8(listarray, 1)) == awkward1.tolist(listarray[:, 1]) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] - @numba.njit - def f10(q, i): - return q[:, 1, i] + @numba.njit + def f9(q): + return q[:, 1, 0] - assert awkward1.tolist(f10(listarray, slice(None, 1))) == awkward1.tolist(listarray[:, 1, slice(None, 1)]) == [[[3.3, 4.4]], [[6.6, 7.7, 8.8, 9.9]]] - assert awkward1.tolist(f10(listarray, slice(None, None))) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] - assert awkward1.tolist(f10(listarray, slice(None, None, -1))) == [[[5.5], [3.3, 4.4]], [[], [6.6, 7.7, 8.8, 9.9]]] + assert awkward1.tolist(listarray[:, 1, 0]) == awkward1.tolist(f9(listarray)) == [[3.3, 4.4], [6.6, 7.7, 8.8, 9.9]] - @numba.njit - def f11(q, i): - return q[[0, 1], 1, i] + @numba.njit + def f10(q, i): + return q[:, 1, i] - assert awkward1.tolist(f11(listarray, slice(None, 1))) == awkward1.tolist(listarray[[0, 1], 1, slice(None, 1)]) == [[[3.3, 4.4]], [[6.6, 7.7, 8.8, 9.9]]] + assert awkward1.tolist(f10(listarray, slice(None, 1))) == awkward1.tolist(listarray[:, 1, slice(None, 1)]) == [[[3.3, 4.4]], [[6.6, 7.7, 8.8, 9.9]]] + assert awkward1.tolist(f10(listarray, slice(None, None))) == [[[3.3, 4.4], [5.5]], [[6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(f10(listarray, slice(None, None, -1))) == [[[5.5], [3.3, 4.4]], [[], [6.6, 7.7, 8.8, 9.9]]] - @numba.njit - def f12(q): - return q[:, 1:, [1, 0, 1]] + @numba.njit + def f11(q, i): + return q[[0, 1], 1, i] - assert awkward1.tolist(f12(listarray)) == awkward1.tolist(listarray[:, 1:, [1, 0, 1]]) == [[[[5.5], [3.3, 4.4], [5.5]]], [[[], [6.6, 7.7, 8.8, 9.9], []]]] + assert awkward1.tolist(f11(listarray, slice(None, 1))) == awkward1.tolist(listarray[[0, 1], 1, slice(None, 1)]) == [[[3.3, 4.4]], [[6.6, 7.7, 8.8, 9.9]]] - @numba.njit - def f13(q): - return q[:, [1], [1, 0, 1]] + @numba.njit + def f12(q): + return q[:, 1:, [1, 0, 1]] - assert awkward1.tolist(f13(listarray)) == awkward1.tolist(listarray[:, [1], [1, 0, 1]]) == [[[5.5], [3.3, 4.4], [5.5]], [[], [6.6, 7.7, 8.8, 9.9], []]] + assert awkward1.tolist(f12(listarray)) == awkward1.tolist(listarray[:, 1:, [1, 0, 1]]) == [[[[5.5], [3.3, 4.4], [5.5]]], [[[], [6.6, 7.7, 8.8, 9.9], []]]] - @numba.njit - def f14(q): - return q.content + @numba.njit + def f13(q): + return q[:, [1], [1, 0, 1]] - assert awkward1.tolist(f14(regulararray)) == awkward1.tolist(regulararray.content) + assert awkward1.tolist(f13(listarray)) == awkward1.tolist(listarray[:, [1], [1, 0, 1]]) == [[[5.5], [3.3, 4.4], [5.5]], [[], [6.6, 7.7, 8.8, 9.9], []]] - @numba.njit - def f15(q): - return q.size + @numba.njit + def f14(q): + return q.content - assert awkward1.tolist(f15(regulararray)) == awkward1.tolist(regulararray.size) == 2 + assert awkward1.tolist(f14(regulararray)) == awkward1.tolist(regulararray.content) - listarray.setid() - assert numpy.asarray(listarray.content.id).tolist() == [[0, 0], [1, 0], [1, 1]] + @numba.njit + def f15(q): + return q.size -# TODO: replace Content::getitem's promotion to ListArray with a promotion to RegularArray. -# TODO: ListArray's and ListOffsetArray's non-advanced getitem array should now output a RegularArray. -# TODO: all getitem arrays should handle non-flat SliceArray by wrapping in RegularArrays. -# TODO: check the FIXME in awkward_listarray_getitem_next_array_advanced. -# TODO: setid should not be allowed on data that can be reached by multiple paths (which will break the ListArray ids above, unfortunately). + assert awkward1.tolist(f15(regulararray)) == awkward1.tolist(regulararray.size) == 2 diff --git a/tests/test_PR024_use_regular_array.py b/tests/test_PR024_use_regular_array.py new file mode 100644 index 0000000000..1cb5e6b4c4 --- /dev/null +++ b/tests/test_PR024_use_regular_array.py @@ -0,0 +1,85 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import sys +import itertools + +import pytest +import numpy + +import awkward1 + +def test_empty_array_slice(): + # inspired by PR021::test_getitem + a = awkward1.fromjson("[[], [[], []], [[], [], []]]") + assert awkward1.tolist(a[2, 1, numpy.array([], dtype=int)]) == [] + assert awkward1.tolist(a[2, numpy.array([1], dtype=int), numpy.array([], dtype=int)]) == [] + + # inspired by PR015::test_deep_numpy + content = awkward1.layout.NumpyArray(numpy.array([[0.0, 1.1], [2.2, 3.3], [4.4, 5.5], [6.6, 7.7], [8.8, 9.9]])) + listarray = awkward1.layout.ListArray64(awkward1.layout.Index64(numpy.array([0, 3, 3])), awkward1.layout.Index64(numpy.array([3, 3, 5])), content) + assert awkward1.tolist(listarray[[2, 0, 0, -1], [1, -1, 0, 0], [0, 1, 0, 1]]) == [8.8, 5.5, 0.0, 7.7] + assert awkward1.tolist(listarray[2, 1, numpy.array([], dtype=int)]) == [] + assert awkward1.tolist(listarray[2, 1, []]) == [] + assert awkward1.tolist(listarray[2, [1], []]) == [] + assert awkward1.tolist(listarray[2, [], []]) == [] + +def test_nonflat_slice(): + array = numpy.arange(2*3*5).reshape(2, 3, 5) + numpyarray = awkward1.layout.NumpyArray(array) + + content = awkward1.layout.NumpyArray(array.reshape(-1)) + inneroffsets = awkward1.layout.Index64(numpy.array([0, 5, 10, 15, 20, 25, 30])) + outeroffsets = awkward1.layout.Index64(numpy.array([0, 3, 6])) + listoffsetarray = awkward1.layout.ListOffsetArray64(outeroffsets, awkward1.layout.ListOffsetArray64(inneroffsets, content)) + listoffsetarray.setid() + + assert awkward1.tolist(array[[1, 0, 1, 1, 1, 0], [2, 0, 1, 1, 2, 0], [2, 4, 2, 4, 0, 1]]) == [27, 4, 22, 24, 25, 1] + assert awkward1.tolist(array[[[1, 0], [1, 1], [1, 0]], [[2, 0], [1, 1], [2, 0]], [[2, 4], [2, 4], [0, 1]]]) == [[27, 4], [22, 24], [25, 1]] + + one = listoffsetarray[[1, 0, 1, 1, 1, 0], [2, 0, 1, 1, 2, 0], [2, 4, 2, 4, 0, 1]] + assert awkward1.tolist(one) == [27, 4, 22, 24, 25, 1] + assert numpy.asarray(one.id).tolist() == [ + [1, 2, 2], + [0, 0, 4], + [1, 1, 2], + [1, 1, 4], + [1, 2, 0], + [0, 0, 1]] + + two = listoffsetarray[[[1, 0], [1, 1], [1, 0]], [[2, 0], [1, 1], [2, 0]], [[2, 4], [2, 4], [0, 1]]] + assert awkward1.tolist(two) == [[27, 4], [22, 24], [25, 1]] + assert numpy.asarray(two.content.id).tolist() == [ + [1, 2, 2], + [0, 0, 4], + [1, 1, 2], + [1, 1, 4], + [1, 2, 0], + [0, 0, 1]] + assert two.id is None + +def test_newaxis(): + array = numpy.arange(2*3*5).reshape(2, 3, 5) + numpyarray = awkward1.layout.NumpyArray(array) + + content = awkward1.layout.NumpyArray(array.reshape(-1)) + inneroffsets = awkward1.layout.Index64(numpy.array([0, 5, 10, 15, 20, 25, 30])) + outeroffsets = awkward1.layout.Index64(numpy.array([0, 3, 6])) + listoffsetarray = awkward1.layout.ListOffsetArray64(outeroffsets, awkward1.layout.ListOffsetArray64(inneroffsets, content)) + + assert awkward1.tolist(array[:, numpy.newaxis]) == [[[[0, 1, 2, 3, 4], [5, 6, 7, 8, 9], [10, 11, 12, 13, 14]]], [[[15, 16, 17, 18, 19], [20, 21, 22, 23, 24], [25, 26, 27, 28, 29]]]] + + assert awkward1.tolist(listoffsetarray[:, numpy.newaxis]) == [[[[0, 1, 2, 3, 4], [5, 6, 7, 8, 9], [10, 11, 12, 13, 14]]], [[[15, 16, 17, 18, 19], [20, 21, 22, 23, 24], [25, 26, 27, 28, 29]]]] + +numba = pytest.importorskip("numba") + +def test_empty_array_slice_numba(): + # inspired by PR015::test_deep_numpy + content = awkward1.layout.NumpyArray(numpy.array([[0.0, 1.1], [2.2, 3.3], [4.4, 5.5], [6.6, 7.7], [8.8, 9.9]])) + listarray = awkward1.layout.ListArray64(awkward1.layout.Index64(numpy.array([0, 3, 3])), awkward1.layout.Index64(numpy.array([3, 3, 5])), content) + + @numba.njit + def f1(q, i, j): + return q[2, i, j] + + assert awkward1.tolist(f1(listarray, numpy.array([1], dtype=int), numpy.array([], dtype=int))) == [] + assert awkward1.tolist(f1(listarray, numpy.array([], dtype=int), numpy.array([], dtype=int))) == []