Error-handling as a struct, rather than just a string. (#13)

The `Error` struct can carry a lot of information into error messages for C++, but Numba can only raise serialized and globally registered exceptions. Therefore, the error messages from Numba have to be known at compile-time, which severely limits their specificity. Users with unhelpful error messages will have to re-run their code outside of Numba.

* Foot-in-the-door to start implementing error-handling as a struct with feedback about the index position of the error (and therefore the Identity, too, if that's available).

* Try to initiate a build from the new repository owner (scikit-hep).

* Make sure I can still trigger builds.

* Add classname() as a virtual method on all Content classes

* Changed Error from a 'const char*' to a struct.

* The 'failure' function didn't return an error (thanks, Windows compiler).

* Start methodically testing error messages.

* NumpyArray error messages are correct.

* Have to carry knowledge of fake dimensions to give the right error message.

* All cpu-kernels should return Error objects, to ensure consistency in the future.

* Locations can only be used if there are Identities.

* The 'fake' argument isn't necessary because fake dimensions will never have Identities.

* These are good error messages.

* Update the Numba side in preparation for handling errors.

* Numba can raise errors in response to the Error.str value.

* Raise errors in Numba, if the function we call returns them (too bad we can't have more specific messages, though).

* Put the 'str' member first so that success vs failure can be determined without an offset.

* Change order of 'failure' function options to highlight the leading role of the static string.

VERSION_INFO

@@ -1 +1 @@
-0.1.9
+0.1.10

awkward1/_numba/content.py

@@ -8,7 +8,9 @@
 from .._numba import cpu, util, identity
 
 class ContentType(numba.types.Type):
- pass
+ @property
+ def shortname(self):
+ return self.name[:self.name.index("Type")]
 
 @numba.typing.templates.infer_global(len)
 class type_len(numba.typing.templates.AbstractTemplate):

awkward1/_numba/cpu.py

@@ -19,6 +19,26 @@
 
 kernels = ctypes.cdll.LoadLibrary(libpath)
 
+class ErrorType(numba.types.Type):
+ def __init__(self):
+ super(ErrorType, self).__init__(name="awkward._numba.cpu.ErrorType")
+
+class Error(ctypes.Structure):
+ _fields_ = [("str", ctypes.c_char_p),
+ ("location", ctypes.c_int64),
+ ("attempt", ctypes.c_int64),
+ ("extra", ctypes.c_int64)]
+ numbatpe = ErrorType()
+
+@numba.extending.register_model(ErrorType)
+class ErrorModel(numba.datamodel.models.StructModel):
+ def __init__(self, dmm, fe_type):
+ members = [("str", numba.intp),
+ ("location", numba.int64),
+ ("attempt", numba.int64),
+ ("extra", numba.int64)]
+ super(ErrorModel, self).__init__(dmm, fe_type, members)
+
 h2ctypes = {
  "bool": ctypes.c_uint8,
  "uint8_t *": ctypes.POINTER(ctypes.c_uint8),
@@ -29,7 +49,7 @@
  "int64_t": ctypes.c_int64,
  "int64_t *": ctypes.POINTER(ctypes.c_int64),
  "const int64_t *": ctypes.POINTER(ctypes.c_int64),
- "Error": ctypes.c_void_p,
+ "Error": Error,
  "void": None,
  }
 
@@ -41,7 +61,13 @@
  if hasattr(kernels, name):
  rettype = xfcn.find("./type").text
  params = [(x.find("./declname").text, x.find("./type").text) for x in xfcn.findall("./param")]
- getattr(kernels, name).argtypes = [h2ctypes[t] for n, t in params]
- getattr(kernels, name).restype = h2ctypes[rettype]
  getattr(kernels, name).name = name
- getattr(kernels, name).numbatpe = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name))
+ getattr(kernels, name).argtypes = [h2ctypes[t] for n, t in params]
+ if rettype == "Error":
+ getattr(kernels, name).restype = None
+ tmp = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name))
+ getattr(kernels, name).numbatpe = numba.types.functions.ExternalFunctionPointer(Error.numbatpe(*tmp.sig.args), tmp.get_pointer, cconv=tmp.cconv)
+ getattr(kernels, name).restype = h2ctypes[rettype]
+ else:
+ getattr(kernels, name).restype = h2ctypes[rettype]
+ getattr(kernels, name).numbatpe = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name))

awkward1/_numba/listarray.py

@@ -293,7 +293,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
  context.get_constant(numba.int64, 0),
  lenstarts,
  lenflathead,
- lencontent))
+ lencontent),
+ "in {}, indexing error".format(arraytpe.shortname))
 
  nexttpe = arraytpe.contenttpe.carry()
  nextval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry)

awkward1/_numba/util.py

@@ -2,6 +2,9 @@
 
 import numpy
 import numba
+import llvmlite.ir.types
+
+from .._numba import cpu
 
 RefType = numba.int64
 
@@ -28,12 +31,30 @@ def arraylen(context, builder, tpe, val, totpe=None):
  else:
  return cast(context, builder, numba.intp, totpe, out)
 
-def call(context, builder, fcn, args):
+def call(context, builder, fcn, args, errormessage=None):
  fcntpe = context.get_function_pointer_type(fcn.numbatpe)
  fcnval = context.add_dynamic_addr(builder, fcn.numbatpe.get_pointer(fcn), info=fcn.name)
  fcnptr = builder.bitcast(fcnval, fcntpe)
+
  err = context.call_function_pointer(builder, fcnptr, args)
- # FIXME: handle error
+
+ if fcn.restype is cpu.Error:
+ assert errormessage is not None, "this function can return an error"
+ proxyerr = numba.cgutils.create_struct_proxy(cpu.Error.numbatpe)(context, builder, value=err)
+ with builder.if_then(builder.icmp_signed("!=", proxyerr.str, context.get_constant(numba.intp, 0)), likely=False):
+ context.call_conv.return_user_exc(builder, ValueError, (errormessage,))
+
+ # pyapi = context.get_python_api(builder)
+ # exc = pyapi.serialize_object(ValueError(errormessage))
+ # excptr = context.call_conv._get_excinfo_argument(builder.function)
+ # if excptr.name == "excinfo" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.struct([llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)), llvmlite.llvmpy.core.Type.int(32)]))):
+ # builder.store(exc, excptr)
+ # builder.ret(numba.targets.callconv.RETCODE_USEREXC)
+ # elif excptr.name == "py_args" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)):
+ # pyapi.raise_object(exc)
+ # builder.ret(llvmlite.llvmpy.core.Constant.null(context.get_value_type(numba.types.pyobject)))
+ # else:
+ # raise AssertionError("unrecognized exception calling convention: {}".format(excptr))
 
 def newindex64(context, builder, lentpe, lenval):
  return numba.targets.arrayobj.numpy_empty_nd(context, builder, index64tpe(lentpe), (lenval,))

include/awkward/Content.h

@@ -10,6 +10,7 @@
 namespace awkward {
  class Content {
  public:
+ virtual const std::string classname() const = 0;
  virtual const std::shared_ptr<Identity> id() const = 0;
  virtual void setid() = 0;
  virtual void setid(const std::shared_ptr<Identity> id) = 0;

include/awkward/Identity.h

@@ -32,6 +32,8 @@ namespace awkward {
  const int64_t width() const { return width_; }
  const int64_t length() const { return length_; }
 
+ virtual const std::string classname() const = 0;
+ virtual const std::string location(int64_t where) const = 0;
  virtual const std::shared_ptr<Identity> to64() const = 0;
  virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0;
  virtual const std::shared_ptr<Identity> getitem_range(int64_t start, int64_t stop) const = 0;
@@ -58,6 +60,8 @@ namespace awkward {
 
  const std::shared_ptr<T> ptr() const { return ptr_; }
 
+ virtual const std::string classname() const;
+ virtual const std::string location(int64_t where) const;
  virtual const std::shared_ptr<Identity> to64() const;
  virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const;
  virtual const std::shared_ptr<Identity> getitem_range(int64_t start, int64_t stop) const;

include/awkward/ListArray.h

@@ -24,6 +24,7 @@ namespace awkward {
  const IndexOf<T> stops() const { return stops_; }
  const std::shared_ptr<Content> content() const { return content_.get()->shallow_copy(); }
 
+ virtual const std::string classname() const;
  virtual const std::shared_ptr<Identity> id() const { return id_; }
  virtual void setid();
  virtual void setid(const std::shared_ptr<Identity> id);

include/awkward/ListOffsetArray.h

@@ -22,6 +22,7 @@ namespace awkward {
  const IndexOf<T> offsets() const { return offsets_; }
  const std::shared_ptr<Content> content() const { return content_.get()->shallow_copy(); }
 
+ virtual const std::string classname() const;
  virtual const std::shared_ptr<Identity> id() const { return id_; }
  virtual void setid();
  virtual void setid(const std::shared_ptr<Identity> id);

include/awkward/NumpyArray.h

@@ -40,6 +40,7 @@ namespace awkward {
  ssize_t bytelength() const;
  uint8_t getbyte(ssize_t at) const;
 
+ virtual const std::string classname() const;
  virtual const std::shared_ptr<Identity> id() const { return id_; }
  virtual void setid();
  virtual void setid(const std::shared_ptr<Identity> id);

include/awkward/RawArray.h

@@ -58,6 +58,7 @@ namespace awkward {
  T* borrow() const { return borrow(0); }
  T* borrow(int64_t at) const { return reinterpret_cast<T*>(reinterpret_cast<ssize_t>(ptr_.get()) + (ssize_t)itemsize_*(ssize_t)(offset_ + at)); }
 
+ virtual const std::string classname() const { return std::string("RawArrayOf<") + std::string(typeid(T).name()) + std::string(">"); }
  virtual const std::shared_ptr<Identity> id() const { return id_; }
  virtual void setid() {
  if (length() <= kMaxInt32) {
@@ -138,7 +139,7 @@ namespace awkward {
  std::shared_ptr<SliceItem> nexthead = where.head();
  Slice nexttail = where.tail();
  Index64 nextadvanced(0);
- return getitem_next(nexthead, nexttail, nextadvanced);
+ return getitem_next(nexthead, nexttail, nextadvanced, false);
  }
  const std::shared_ptr<Content> getitem_next(const std::shared_ptr<SliceItem> head, const Slice& tail, const Index64& advanced) const {
  if (tail.length() != 0) {

include/awkward/cpu-kernels/getitem.h

@@ -9,46 +9,47 @@ extern "C" {
  void awkward_regularize_rangeslice(int64_t* start, int64_t* stop, bool posstep, bool hasstart, bool hasstop, int64_t length);
  Error awkward_regularize_arrayslice_64(int64_t* flatheadptr, int64_t lenflathead, int64_t length);
 
- void awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides);
+ Error awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides);
 
- void awkward_carry_arange_64(int64_t* toptr, int64_t length);
+ Error awkward_carry_arange_64(int64_t* toptr, int64_t length);
 
  Error awkward_identity32_getitem_carry_64(int32_t* newidentityptr, const int32_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length);
  Error awkward_identity64_getitem_carry_64(int64_t* newidentityptr, const int64_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length);
 
- void awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride);
- void awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
- void awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride);
- void awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
- void awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at);
- void awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
- void awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
- void awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip);
- void awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip);
+ Error awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride);
+ Error awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
+ Error awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride);
+
+ Error awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
+ Error awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at);
+ Error awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
+ Error awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
+ Error awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip);
+ Error awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip);
 
  Error awkward_listarray32_getitem_next_at_64(int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at);
  Error awkward_listarray64_getitem_next_at_64(int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at);
 
- void awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
- void awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+ Error awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+ Error awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
 
- void awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
- void awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+ Error awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+ Error awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
 
- void awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts);
- void awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts);
+ Error awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts);
+ Error awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts);
 
- void awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts);
- void awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts);
+ Error awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts);
+ Error awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts);
 
  Error awkward_listarray32_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
  Error awkward_listarray64_getitem_next_array_64(int64_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
 
  Error awkward_listarray32_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
  Error awkward_listarray64_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
 
- void awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry);
- void awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry);
+ Error awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry);
+ Error awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry);
 }
 
 #endif // AWKWARDCPU_GETITEM_H_

include/awkward/cpu-kernels/identity.h

@@ -6,10 +6,10 @@
 #include "awkward/cpu-kernels/util.h"
 
 extern "C" {
- void awkward_new_identity32(int32_t* toptr, int64_t length);
- void awkward_new_identity64(int64_t* toptr, int64_t length);
+ Error awkward_new_identity32(int32_t* toptr, int64_t length);
+ Error awkward_new_identity64(int64_t* toptr, int64_t length);
 
- void awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length);
+ Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length);
 
  Error awkward_identity32_from_listarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth);
  Error awkward_identity64_from_listarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth);

include/awkward/cpu-kernels/util.h

@@ -20,9 +20,17 @@
  #include <cstdint>
 #endif
 
+#include <iostream>
+
 extern "C" {
- typedef const char* Error;
- const Error kNoError = nullptr;
+ struct Error {
+ const char* str;
+ int64_t location;
+ int64_t attempt;
+ int64_t extra;
+ };
+ Error success();
+ Error failure(const char* str, int64_t location, int64_t attempt);
 
  const int8_t kMaxInt8 = 127; // 2**7 - 1
  const uint8_t kMaxUInt8 = 255; // 2**8 - 1