Start using RegularArray everywhere it needs to be used. (#24)

This check-in includes a commented-out attempt at `lower_getitem_nothing` in `awkward1._numba.array.numpyarray`. In the next PR, it will be gone. I concluded that there is no non-dependent type inference (where array dimensionality is part of the type but the length is not) for `__getitem__` of arrays for `NumpyArray` because an empty array would reduce the dimensionality by one yet return nothing, but reducing the dimensionality of a one-dimensional `NumpyArray` would return a scalar, which is not nothing. Whereas the C++ implementation could be fully transferred from a `ListArray` wrapper to a `RegularArray` wrapper (because it is dynamically typed, ironically), the Numba implementation could not (because it is not dynamically typed). The Numba implementation only wraps with `RegularArray` if the types are sound.

* Start using RegularArray everywhere it needs to be used.

* [skip ci] ListArray64 wrapper -> RegularArray wrapper partly works.

* [skip ci] ListArray64 wrapper -> RegularArray wrapper still only works partially.

* ListArray64 wrapper -> RegularArray wrapper now works

* RegularArray::tojson_part and fix EmptyArray::tojson_part.

* Centralize lower_getitem_tuple and lower_getitem_other.

* Avoid selecting types based on a value (length of array) in Content::getitem(Slice).

* getitem_tuple for Numba mirrors C++ as well as possible: if the 'outtpe' is 'NumpyArrayType', then you're going to have to 'lower_getitem_int'; 'lower_getitem_nothing' is impossible.

* ListArray and ListOffsetArray::getitem_next(array, not advanced) returns a RegularArray now (C++ and Numba).

* ListArray and ListOffsetArray::getitem_next(array) returns the same shape of RegularArrays as the index array.

* Now newaxis uses RegularArray, too.

* Ambiguous identities in ListArray are now refused with an error.

* [skip ci] writing awkward_identity*_from_listoffsetarray*.

* ListOffsetArray now uses awkward_identity*_from_listoffsetarray*.

* Addressed the FIXMEs in getitem.cpp; ready to merge PR.

README.md

@@ -126,7 +126,7 @@ Completed items are ☑check-marked. See [closed PRs](https://github.com/scikit-
  * [ ] Start the `awkward → awkward0`, `awkward1 → awkward` transition.
  * [ ] Translation to and from Apache Arrow and Parquet in C++.
  * [ ] Persistence to any medium that stores named binary blobs, as before, but accessible via C++ (especially for writing). The persistence format might differ slightly from the existing one (break backward compatibility, if needed).
- * [ ] Universal `array.get[...]` as a softer form of `array[...]` that skips non-existent indexes, rather than raising errors.
+ * [ ] Universal `array.get[...]` as a softer form of `array[...]` that inserts `None` for non-existent indexes, rather than raising errors.
  * [ ] Explicit interface with [NumExpr](https://numexpr.readthedocs.io/en/latest/index.html).
 
 ### At some point in the future

VERSION_INFO

@@ -1 +1 @@
-0.1.23
+0.1.24

awkward1/_numba/array/emptyarray.py

@@ -48,6 +48,10 @@ def carry(self):
  def lower_len(self):
  return lower_len
 
+ @property
+ def lower_getitem_nothing(self):
+ return lower_getitem_nothing
+
  @property
  def lower_getitem_range(self):
  return lower_getitem_range
@@ -95,6 +99,9 @@ def box(tpe, val, c):
 def lower_len(context, builder, sig, args):
  return context.get_constant(numba.intp, 0)
 
+def lower_getitem_nothing(context, builder, tpe, val):
+ return val
+
 @numba.extending.lower_builtin(operator.getitem, EmptyArrayType, numba.types.slice2_type)
 def lower_getitem_range(context, builder, sig, args):
  rettpe, (tpe, wheretpe) = sig.return_type, sig.args

awkward1/_numba/array/listarray.py

@@ -52,7 +52,7 @@ def getitem_tuple(self, wheretpe):
  return outtpe.getitem_int()
 
  def getitem_next(self, wheretpe, isadvanced):
- import awkward1._numba.array.listoffsetarray
+ import awkward1._numba.array.regulararray
  if len(wheretpe.types) == 0:
  return self
  headtpe = wheretpe.types[0]
@@ -76,7 +76,7 @@ def getitem_next(self, wheretpe, isadvanced):
  raise NotImplementedError("array.ndim != 1")
  contenttpe = self.contenttpe.carry().getitem_next(tailtpe, True)
  if not isadvanced:
- return awkward1._numba.array.listoffsetarray.ListOffsetArrayType(util.indextpe(self.indexname), contenttpe, self.idtpe)
+ return awkward1._numba.array.regulararray.RegularArrayType(contenttpe, self.idtpe)
  else:
  return contenttpe
 
@@ -90,6 +90,10 @@ def carry(self):
  def lower_len(self):
  return lower_len
 
+ @property
+ def lower_getitem_nothing(self):
+ return content.lower_getitem_nothing
+
  @property
  def lower_getitem_int(self):
  return lower_getitem_int
@@ -222,31 +226,19 @@ def lower_getitem_range(context, builder, sig, args):
 
 @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.BaseTuple)
 def lower_getitem_tuple(context, builder, sig, args):
- rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args
- arrayval, whereval = args
-
- wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval)
- nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval)
-
- outtpe = nexttpe.getitem_next(wheretpe, False)
- outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None)
-
- return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0)))
+ return content.lower_getitem_tuple(context, builder, sig, args)
 
 @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.Array)
 @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.List)
 @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.ArrayCompatible)
 @numba.extending.lower_builtin(operator.getitem, ListArrayType, numba.types.EllipsisType)
 @numba.extending.lower_builtin(operator.getitem, ListArrayType, type(numba.typeof(numpy.newaxis)))
 def lower_getitem_other(context, builder, sig, args):
- rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args
- arrayval, whereval = args
- wrappedtpe = numba.types.Tuple((wheretpe,))
- wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,))
- return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval))
+ return content.lower_getitem_other(context, builder, sig, args)
 
 def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, advanced):
  import awkward1._numba.array.listoffsetarray
+ import awkward1._numba.array.regulararray
 
  if len(wheretpe.types) == 0:
  return arrayval
@@ -400,10 +392,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
 
  nextcarry = util.newindex64(context, builder, numba.int64, lencarry)
  nextadvanced = util.newindex64(context, builder, numba.int64, lencarry)
- nextoffsets = util.newindex(arraytpe.indexname, context, builder, numba.int64, lenoffsets)
  util.call(context, builder, kernel,
- (util.arrayptr(context, builder, util.indextpe(arraytpe.indexname), nextoffsets),
- util.arrayptr(context, builder, util.index64tpe, nextcarry),
+ (util.arrayptr(context, builder, util.index64tpe, nextcarry),
  util.arrayptr(context, builder, util.index64tpe, nextadvanced),
  util.arrayptr(context, builder, arraytpe.startstpe, proxyin.starts),
  util.arrayptr(context, builder, arraytpe.stopstpe, proxyin.stops),
@@ -421,10 +411,10 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
  contenttpe = nexttpe.getitem_next(tailtpe, True)
  contentval = nexttpe.lower_getitem_next(context, builder, nexttpe, tailtpe, nextval, tailval, nextadvanced)
 
- outtpe = awkward1._numba.array.listoffsetarray.ListOffsetArrayType(util.indextpe(arraytpe.indexname), contenttpe, arraytpe.idtpe)
+ outtpe = awkward1._numba.array.regulararray.RegularArrayType(contenttpe, arraytpe.idtpe)
  proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder)
- proxyout.offsets = nextoffsets
  proxyout.content = contentval
+ proxyout.size = lenflathead
  if outtpe.idtpe != numba.none:
  proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, outtpe.idtpe, util.index64tpe, proxyin.id, flathead)
  return proxyout._getvalue()

awkward1/_numba/array/listoffsetarray.py

@@ -47,6 +47,7 @@ def getitem_tuple(self, wheretpe):
  return out.getitem_int()
 
  def getitem_next(self, wheretpe, isadvanced):
+ import awkward1._numba.array.regulararray
  if len(wheretpe.types) == 0:
  return self
  headtpe = wheretpe.types[0]
@@ -70,7 +71,7 @@ def getitem_next(self, wheretpe, isadvanced):
  raise NotImplementedError("array.ndim != 1")
  contenttpe = self.contenttpe.carry().getitem_next(tailtpe, True)
  if not isadvanced:
- return ListOffsetArrayType(util.indextpe(self.indexname), contenttpe, self.idtpe)
+ return awkward1._numba.array.regulararray.RegularArrayType(contenttpe, self.idtpe)
  else:
  return contenttpe
 
@@ -85,6 +86,10 @@ def carry(self):
  def lower_len(self):
  return lower_len
 
+ @property
+ def lower_getitem_nothing(self):
+ return content.lower_getitem_nothing
+
  @property
  def lower_getitem_int(self):
  return lower_getitem_int
@@ -219,27 +224,15 @@ def lower_getitem_range(context, builder, sig, args):
 
 @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.BaseTuple)
 def lower_getitem_tuple(context, builder, sig, args):
- rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args
- arrayval, whereval = args
-
- wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval)
- nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval)
-
- outtpe = nexttpe.getitem_next(wheretpe, False)
- outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None)
- return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0)))
+ return content.lower_getitem_tuple(context, builder, sig, args)
 
 @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.Array)
 @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.List)
 @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.ArrayCompatible)
 @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, numba.types.EllipsisType)
 @numba.extending.lower_builtin(operator.getitem, ListOffsetArrayType, type(numba.typeof(numpy.newaxis)))
 def lower_getitem_other(context, builder, sig, args):
- rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args
- arrayval, whereval = args
- wrappedtpe = numba.types.Tuple((wheretpe,))
- wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,))
- return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval))
+ return content.lower_getitem_other(context, builder, sig, args)
 
 def starts_stops(context, builder, offsetstpe, offsetsval, lenstarts, lenoffsets):
  proxyslicestarts = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder)
@@ -411,10 +404,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
 
  nextcarry = util.newindex64(context, builder, numba.int64, lencarry)
  nextadvanced = util.newindex64(context, builder, numba.int64, lencarry)
- nextoffsets = util.newindex(arraytpe.indexname, context, builder, numba.int64, lenoffsets)
  util.call(context, builder, kernel,
- (util.arrayptr(context, builder, util.indextpe(arraytpe.indexname), nextoffsets),
- util.arrayptr(context, builder, util.index64tpe, nextcarry),
+ (util.arrayptr(context, builder, util.index64tpe, nextcarry),
  util.arrayptr(context, builder, util.index64tpe, nextadvanced),
  util.arrayptr(context, builder, arraytpe.offsetstpe, starts),
  util.arrayptr(context, builder, arraytpe.offsetstpe, stops),
@@ -432,10 +423,10 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
  contenttpe = nexttpe.getitem_next(tailtpe, True)
  contentval = nexttpe.lower_getitem_next(context, builder, nexttpe, tailtpe, nextval, tailval, nextadvanced)
 
- outtpe = ListOffsetArrayType(util.indextpe(arraytpe.indexname), contenttpe, arraytpe.idtpe)
+ outtpe = awkward1._numba.array.regulararray.RegularArrayType(contenttpe, arraytpe.idtpe)
  proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder)
- proxyout.offsets = nextoffsets
  proxyout.content = contentval
+ proxyout.size = lenflathead
  if outtpe.idtpe != numba.none:
  proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, outtpe.idtpe, util.index64tpe, proxyin.id, flathead)
  return proxyout._getvalue()

awkward1/_numba/array/numpyarray.py

@@ -58,6 +58,10 @@ def carry(self):
  def lower_len(self):
  return lower_len
 
+ @property
+ def lower_getitem_nothing(self):
+ return None
+
  @property
  def lower_getitem_int(self):
  return lower_getitem
@@ -119,6 +123,40 @@ def lower_len(context, builder, sig, args):
  proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
  return numba.targets.arrayobj.array_len(context, builder, numba.intp(tpe.arraytpe), (proxyin.array,))
 
+# def lower_getitem_nothing(context, builder, tpe, val):
+# import awkward1._numba.identity
+#
+# proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val)
+#
+# proxyslice = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder)
+# proxyslice.start = context.get_constant(numba.intp, 0)
+# proxyslice.stop = context.get_constant(numba.intp, 0)
+# proxyslice.step = context.get_constant(numba.intp, 1)
+# emptyslice = proxyslice._getvalue()
+# emptyarray = numba.targets.arrayobj.getitem_arraynd_intp(context, builder, tpe.arraytpe(tpe.arraytpe, numba.types.slice2_type), (proxyin.array, emptyslice))
+#
+# if tpe.arraytpe.ndim > 1:
+# shapetpe = numba.types.Tuple((numba.intp,) * tpe.arraytpe.ndim)
+# shapeval = numba.targets.arrayobj.make_array(tpe.arraytpe)(context, builder, proxyin.array).shape
+#
+# newshapetpe = numba.types.Tuple((numba.intp,) * (tpe.arraytpe.ndim - 1))
+# newshapeval = context.make_tuple(builder, newshapetpe, tuple(builder.extract_value(shapeval, i) for i in range(tpe.arraytpe.ndim - 1)))
+#
+# arraytpe = numba.types.Array(tpe.arraytpe.dtype, tpe.arraytpe.ndim - 1, tpe.arraytpe.layout)
+# arrayval = numba.targets.arrayobj.array_reshape(context, builder, arraytpe(tpe.arraytpe, newshapetpe), (proxyin.array, newshapeval))
+#
+# else:
+# arraytpe = tpe.arraytpe
+# arrayval = emptyarray
+#
+# outtpe = NumpyArrayType(arraytpe, tpe.idtpe)
+# proxyout = numba.cgutils.create_struct_proxy(outtpe)(context, builder)
+# proxyout.array = arrayval
+# if tpe.idtpe != numba.none:
+# proxyout.id = awkward1._numba.identity.lower_getitem_any(context, builder, tpe.idtpe, numba.types.slice2_type, proxyin.id, emptyslice)
+#
+# return proxyout._getvalue()
+
 @numba.extending.lower_builtin(operator.getitem, NumpyArrayType, numba.types.Integer)
 @numba.extending.lower_builtin(operator.getitem, NumpyArrayType, numba.types.SliceType)
 @numba.extending.lower_builtin(operator.getitem, NumpyArrayType, numba.types.Array)

awkward1/_numba/array/regulararray.py

@@ -72,6 +72,10 @@ def carry(self):
  def lower_len(self):
  return lower_len
 
+ @property
+ def lower_getitem_nothing(self):
+ return content.lower_getitem_nothing
+
  @property
  def lower_getitem_int(self):
  return lower_getitem_int
@@ -188,28 +192,15 @@ def lower_getitem_range(context, builder, sig, args):
 
 @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.BaseTuple)
 def lower_getitem_tuple(context, builder, sig, args):
- rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args
- arrayval, whereval = args
-
- wheretpe, whereval = util.preprocess_slicetuple(context, builder, wheretpe, whereval)
- nexttpe, nextval = util.wrap_for_slicetuple(context, builder, arraytpe, arrayval)
-
- outtpe = nexttpe.getitem_next(wheretpe, False)
- outval = nexttpe.lower_getitem_next(context, builder, nexttpe, wheretpe, nextval, whereval, None)
-
- return outtpe.lower_getitem_int(context, builder, rettpe(outtpe, numba.int64), (outval, context.get_constant(numba.int64, 0)))
+ return content.lower_getitem_tuple(context, builder, sig, args)
 
 @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.Array)
 @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.List)
 @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.ArrayCompatible)
 @numba.extending.lower_builtin(operator.getitem, RegularArrayType, numba.types.EllipsisType)
 @numba.extending.lower_builtin(operator.getitem, RegularArrayType, type(numba.typeof(numpy.newaxis)))
 def lower_getitem_other(context, builder, sig, args):
- rettpe, (arraytpe, wheretpe) = sig.return_type, sig.args
- arrayval, whereval = args
- wrappedtpe = numba.types.Tuple((wheretpe,))
- wrappedval = context.make_tuple(builder, wrappedtpe, (whereval,))
- return lower_getitem_tuple(context, builder, rettpe(arraytpe, wrappedtpe), (arrayval, wrappedval))
+ return content.lower_getitem_other(context, builder, sig, args)
 
 def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, advanced):
  if len(wheretpe.types) == 0: