diff --git a/README.md b/README.md index ae72766540..ed069bd28e 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ The following features of awkward 0.x will be features of awkward 1.x. * 2019-09-26 (PR [#11](../../pull/11)): fully implemented `ListArray` and `ListOffsetArray`'s `__getitem__`. * 2019-10-02 (PR [#12](../../pull/12)): implemented `ListArray.__getitem__(array)` in Numba, setting the pattern for all the other cases. * 2019-10-04 (PR [#13](../../pull13)): turned the `Error` type into a struct and `handle_error` into a function that provides `Identity` (if available) and attempted index information. Numba also handles errors, but with messages that must be known at compile-time. + * 2019-10-04 (PR [#14](../../pull14)): brought `NumpyArray`, `ListArray`, and `ListOffsetArray` up-to-date on `getitem` for all cases (except `newaxis`) and tested them all as well. (Not including Numba.) ## Roadmap diff --git a/VERSION_INFO b/VERSION_INFO index 9767cc98e7..20f49513e1 100644 --- a/VERSION_INFO +++ b/VERSION_INFO @@ -1 +1 @@ -0.1.10 +0.1.11 diff --git a/awkward1/_numba/listoffsetarray.py b/awkward1/_numba/listoffsetarray.py index a429f8adc0..cf7fcf736c 100644 --- a/awkward1/_numba/listoffsetarray.py +++ b/awkward1/_numba/listoffsetarray.py @@ -167,6 +167,8 @@ def lower_getitem_slice(context, builder, sig, args): proxyin = numba.cgutils.create_struct_proxy(tpe)(context, builder, value=val) proxyslicein = numba.cgutils.create_struct_proxy(wheretpe)(context, builder, value=whereval) + numba.targets.slicing.fix_slice(builder, proxyslicein, tpe.lower_len(context, builder, numba.intp(tpe), (val,))) + proxysliceout = numba.cgutils.create_struct_proxy(numba.types.slice2_type)(context, builder) proxysliceout.start = proxyslicein.start proxysliceout.stop = builder.add(proxyslicein.stop, context.get_constant(numba.intp, 1)) diff --git a/include/awkward/Content.h b/include/awkward/Content.h index 0ea6c5ce4c..1141db4b02 100644 --- a/include/awkward/Content.h +++ b/include/awkward/Content.h @@ -17,8 +17,11 @@ namespace awkward { virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0; virtual int64_t length() const = 0; virtual const std::shared_ptr shallow_copy() const = 0; + virtual void checksafe() const = 0; virtual const std::shared_ptr getitem_at(int64_t at) const = 0; + virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const = 0; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const = 0; + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const = 0; virtual const std::shared_ptr getitem(const Slice& where) const; virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const = 0; virtual const std::shared_ptr carry(const Index64& carry) const = 0; diff --git a/include/awkward/Identity.h b/include/awkward/Identity.h index 631c051d92..c2c2046bf3 100644 --- a/include/awkward/Identity.h +++ b/include/awkward/Identity.h @@ -36,10 +36,12 @@ namespace awkward { virtual const std::string location(int64_t where) const = 0; virtual const std::shared_ptr to64() const = 0; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0; - virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const = 0; + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const = 0; virtual const std::shared_ptr shallow_copy() const = 0; virtual const std::shared_ptr getitem_carry_64(const Index64& carry) const = 0; + const std::string tostring() const; + protected: const Ref ref_; const FieldLoc fieldloc_; @@ -64,12 +66,13 @@ namespace awkward { virtual const std::string location(int64_t where) const; virtual const std::shared_ptr to64() const; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; - virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; virtual const std::shared_ptr shallow_copy() const; virtual const std::shared_ptr getitem_carry_64(const Index64& carry) const; - const std::string tostring() const; - const std::vector get(int64_t at) const; + const std::vector getitem_at(int64_t at) const; + const std::vector getitem_at_unsafe(int64_t at) const; + const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; private: const std::shared_ptr ptr_; diff --git a/include/awkward/Index.h b/include/awkward/Index.h index 3293a9c717..395cbb48dd 100644 --- a/include/awkward/Index.h +++ b/include/awkward/Index.h @@ -30,10 +30,13 @@ namespace awkward { int64_t offset() const { return offset_; } int64_t length() const { return length_; } + const std::string classname() const; const std::string tostring() const; const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; T getitem_at(int64_t at) const; + T getitem_at_unsafe(int64_t at) const; IndexOf getitem_range(int64_t start, int64_t stop) const; + IndexOf getitem_range_unsafe(int64_t start, int64_t stop) const; virtual const std::shared_ptr shallow_copy() const; private: diff --git a/include/awkward/Iterator.h b/include/awkward/Iterator.h index 2082c647dd..800b8cc11a 100644 --- a/include/awkward/Iterator.h +++ b/include/awkward/Iterator.h @@ -11,13 +11,15 @@ namespace awkward { public: Iterator(const std::shared_ptr content) : content_(content) - , where_(0) { } + , where_(0) { + content.get()->checksafe(); + } const std::shared_ptr content() const { return content_; } const int64_t where() const { return where_; } - const bool isdone() const { return where_ >= content_.get()->length(); } - const std::shared_ptr next() { return content_.get()->getitem_at(where_++); } + const bool isdone() const; + const std::shared_ptr next(); const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; const std::string tostring() const; diff --git a/include/awkward/ListArray.h b/include/awkward/ListArray.h index 9bf296d5b6..d30e60e1ea 100644 --- a/include/awkward/ListArray.h +++ b/include/awkward/ListArray.h @@ -31,8 +31,11 @@ namespace awkward { virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; + virtual void checksafe() const; virtual const std::shared_ptr getitem_at(int64_t at) const; + virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; virtual const std::shared_ptr carry(const Index64& carry) const; virtual const std::pair minmax_depth() const; diff --git a/include/awkward/ListOffsetArray.h b/include/awkward/ListOffsetArray.h index a536f853ba..c3dd023950 100644 --- a/include/awkward/ListOffsetArray.h +++ b/include/awkward/ListOffsetArray.h @@ -29,8 +29,11 @@ namespace awkward { virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; + virtual void checksafe() const; virtual const std::shared_ptr getitem_at(int64_t at) const; + virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; virtual const std::shared_ptr carry(const Index64& carry) const; virtual const std::pair minmax_depth() const; diff --git a/include/awkward/NumpyArray.h b/include/awkward/NumpyArray.h index eb19e82cfa..53e8255e0f 100644 --- a/include/awkward/NumpyArray.h +++ b/include/awkward/NumpyArray.h @@ -47,8 +47,11 @@ namespace awkward { virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual int64_t length() const; virtual const std::shared_ptr shallow_copy() const; + virtual void checksafe() const; virtual const std::shared_ptr getitem_at(int64_t at) const; + virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const; virtual const std::shared_ptr getitem(const Slice& where) const; virtual const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const; virtual const std::shared_ptr carry(const Index64& carry) const; @@ -59,7 +62,7 @@ namespace awkward { const NumpyArray contiguous() const; const NumpyArray contiguous_next(Index64 bytepos) const; const NumpyArray getitem_bystrides(const std::shared_ptr& head, const Slice& tail, int64_t length) const; - const NumpyArray getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride) const; + const NumpyArray getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const; private: std::shared_ptr id_; diff --git a/include/awkward/RawArray.h b/include/awkward/RawArray.h index 34b35152c8..4be856b06d 100644 --- a/include/awkward/RawArray.h +++ b/include/awkward/RawArray.h @@ -47,19 +47,29 @@ namespace awkward { , itemsize_(sizeof(T)) { } const std::shared_ptr ptr() const { return ptr_; } + const int64_t offset() const { return offset_; } + const int64_t itemsize() const { return itemsize_; } bool isempty() const { return length_ == 0; } + ssize_t byteoffset() const { return (ssize_t)itemsize_*(ssize_t)offset_; } + uint8_t* byteptr() const { return reinterpret_cast(reinterpret_cast(ptr_.get()) + byteoffset()); } + ssize_t bytelength() const { return (ssize_t)itemsize_*(ssize_t)length_; } + uint8_t getbyte(ssize_t at) const { return *reinterpret_cast(reinterpret_cast(ptr_.get()) + (ssize_t)(byteoffset() + at)); } + T* borrow() const { return borrow(0); } + T* borrow(int64_t at) const { return reinterpret_cast(reinterpret_cast(ptr_.get()) + (ssize_t)itemsize_*(ssize_t)(offset_ + at)); } virtual const std::string classname() const { return std::string("RawArrayOf<") + std::string(typeid(T).name()) + std::string(">"); } + virtual const std::shared_ptr id() const { return id_; } + virtual void setid() { if (length() <= kMaxInt32) { Identity32* rawid = new Identity32(Identity::newref(), Identity::FieldLoc(), 1, length()); @@ -74,12 +84,14 @@ namespace awkward { setid(newid); } } + virtual void setid(const std::shared_ptr id) { if (id.get() != nullptr && length() != id.get()->length()) { throw std::invalid_argument("content and its id must have the same length"); } id_ = id; } + virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const { std::stringstream out; out << indent << pre << " shallow_copy() const { return std::shared_ptr(new RawArrayOf(id_, ptr_, offset_, length_, itemsize_)); } - virtual const std::shared_ptr getitem_at(int64_t at) const { return getitem_range(at, at + 1); } + + virtual void checksafe() const { + if (id_.get() != nullptr && id_.get()->length() < length_) { + util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); + } + } + + virtual const std::shared_ptr getitem_at(int64_t at) const { + int64_t regular_at = at; + if (regular_at < 0) { + regular_at += length_; + } + if (!(0 <= regular_at && regular_at < length_)) { + util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); + } + return getitem_range_unsafe(regular_at, regular_at + 1); + } + + virtual const std::shared_ptr getitem_at_unsafe(int64_t at) const { + return getitem_range_unsafe(at, at + 1); + } + virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const { int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(regular_start, regular_stop, true, start != Slice::none(), stop != Slice::none(), length_); + return getitem_range_unsafe(regular_start, regular_stop); + } + + virtual const std::shared_ptr getitem_range_unsafe(int64_t start, int64_t stop) const { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { if (regular_stop > id_.get()->length()) { - throw std::invalid_argument("index out of range for identity"); + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } id = id_.get()->getitem_range(regular_start, regular_stop); } return std::shared_ptr(new RawArrayOf(id, ptr_, offset_ + regular_start, regular_stop - regular_start, itemsize_)); } + virtual const std::shared_ptr getitem(const Slice& where) const { std::shared_ptr nexthead = where.head(); Slice nexttail = where.tail(); Index64 nextadvanced(0); return getitem_next(nexthead, nexttail, nextadvanced, false); } + const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { if (tail.length() != 0) { throw std::invalid_argument("too many indexes for array"); @@ -204,9 +245,11 @@ namespace awkward { throw std::runtime_error("unrecognized slice item type"); } } + virtual const std::shared_ptr carry(const Index64& carry) const { throw std::runtime_error("RawArray::carry"); } + virtual const std::pair minmax_depth() const { return std::pair(1, 1); } private: diff --git a/src/libawkward/Identity.cpp b/src/libawkward/Identity.cpp index 55472123e7..230cb3955c 100644 --- a/src/libawkward/Identity.cpp +++ b/src/libawkward/Identity.cpp @@ -8,6 +8,7 @@ #include "awkward/cpu-kernels/identity.h" #include "awkward/cpu-kernels/getitem.h" +#include "awkward/Slice.h" #include "awkward/Identity.h" @@ -40,8 +41,8 @@ namespace awkward { if (bothi != 0) { out << ", "; } - if (fieldi < (int64_t)fieldloc_.size() && fieldloc_[fieldi].first == bothi) { - out << "\"" << fieldloc_[fieldi].second << "\""; + if (fieldi < (int64_t)fieldloc_.size() && fieldloc_[(size_t)fieldi].first == bothi) { + out << "\"" << fieldloc_[(size_t)fieldi].second << "\""; fieldi++; } else { @@ -88,12 +89,7 @@ namespace awkward { } template - const std::string IdentityOf::tostring() const { - return tostring_part("", "", ""); - } - - template - const std::shared_ptr IdentityOf::getitem_range(int64_t start, int64_t stop) const { + const std::shared_ptr IdentityOf::getitem_range_unsafe(int64_t start, int64_t stop) const { assert(0 <= start && start < length_ && 0 <= stop && stop < length_); return std::shared_ptr(new IdentityOf(ref_, fieldloc_, offset_ + width_*start*(start != stop), width_, (stop - start), ptr_)); } @@ -137,16 +133,40 @@ namespace awkward { return out; } + const std::string Identity::tostring() const { + return tostring_part("", "", ""); + } + template - const std::vector IdentityOf::get(int64_t at) const { - assert(0 <= at < length_); + const std::vector IdentityOf::getitem_at(int64_t at) const { + int64_t regular_at = at; + if (regular_at < 0) { + regular_at += length_; + } + if (!(0 <= regular_at && regular_at < length_)) { + util::handle_error(failure("index out of range", kSliceNone, at), classname(), nullptr); + } + return getitem_at_unsafe(regular_at); + } + + template + const std::vector IdentityOf::getitem_at_unsafe(int64_t at) const { + assert(0 <= at && at < length_); std::vector out; - for (size_t i = (size_t)(offset() + at); i < (size_t)(offset() + at + width()); i++) { + for (size_t i = (size_t)(offset_ + at); i < (size_t)(offset_ + at + width_); i++) { out.push_back(ptr_.get()[i]); } return out; } + template + const std::shared_ptr IdentityOf::getitem_range(int64_t start, int64_t stop) const { + int64_t regular_start = start; + int64_t regular_stop = stop; + awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), length_); + return getitem_range_unsafe(regular_start, regular_stop); + } + template class IdentityOf; template class IdentityOf; } diff --git a/src/libawkward/Index.cpp b/src/libawkward/Index.cpp index 5a1d9fe7d3..085583dddc 100644 --- a/src/libawkward/Index.cpp +++ b/src/libawkward/Index.cpp @@ -9,6 +9,22 @@ #include "awkward/Index.h" namespace awkward { + template + const std::string IndexOf::classname() const { + if (std::is_same::value) { + return "Index8"; + } + else if (std::is_same::value) { + return "Index32"; + } + else if (std::is_same::value) { + return "Index64"; + } + else { + return "UnrecognizedIndex"; + } + } + template const std::string IndexOf::tostring() const { return tostring_part("", "", ""); @@ -17,23 +33,13 @@ namespace awkward { template const std::string IndexOf::tostring_part(const std::string indent, const std::string pre, const std::string post) const { std::stringstream out; - std::string name = "Unrecognized Index"; - if (std::is_same::value) { - name = "Index8"; - } - else if (std::is_same::value) { - name = "Index32"; - } - else if (std::is_same::value) { - name = "Index64"; - } - out << indent << pre << "<" << name << " i=\"["; + out << indent << pre << "<" << classname() << " i=\"["; if (length_ <= 10) { for (int64_t i = 0; i < length_; i++) { if (i != 0) { out << " "; } - out << (int64_t)getitem_at(i); + out << (int64_t)getitem_at_unsafe(i); } } else { @@ -41,14 +47,14 @@ namespace awkward { if (i != 0) { out << " "; } - out << (int64_t)getitem_at(i); + out << (int64_t)getitem_at_unsafe(i); } out << " ... "; for (int64_t i = length_ - 5; i < length_; i++) { if (i != length_ - 5) { out << " "; } - out << (int64_t)getitem_at(i); + out << (int64_t)getitem_at_unsafe(i); } } out << "]\" offset=\"" << offset_ << "\" at=\"0x"; @@ -58,12 +64,32 @@ namespace awkward { template T IndexOf::getitem_at(int64_t at) const { + int64_t regular_at = at; + if (regular_at < 0) { + regular_at += length_; + } + if (!(0 <= regular_at && regular_at < length_)) { + util::handle_error(failure("index out of range", kSliceNone, at), classname(), nullptr); + } + return getitem_at_unsafe(regular_at); + } + + template + T IndexOf::getitem_at_unsafe(int64_t at) const { assert(0 <= at && at < length_); return ptr_.get()[(size_t)(offset_ + at)]; } template IndexOf IndexOf::getitem_range(int64_t start, int64_t stop) const { + int64_t regular_start = start; + int64_t regular_stop = stop; + awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), length_); + return getitem_range_unsafe(regular_start, regular_stop); + } + + template + IndexOf IndexOf::getitem_range_unsafe(int64_t start, int64_t stop) const { assert(0 <= start && start < length_ && 0 <= stop && stop < length_); return IndexOf(ptr_, offset_ + start*(start != stop), stop - start); } diff --git a/src/libawkward/Iterator.cpp b/src/libawkward/Iterator.cpp index 72e3f99665..dc1064bc7f 100644 --- a/src/libawkward/Iterator.cpp +++ b/src/libawkward/Iterator.cpp @@ -5,6 +5,14 @@ #include "awkward/Iterator.h" namespace awkward { + const bool Iterator::isdone() const { + return where_ >= content_.get()->length(); + } + + const std::shared_ptr Iterator::next() { + return content_.get()->getitem_at_unsafe(where_++); + } + const std::string Iterator::tostring_part(const std::string indent, const std::string pre, const std::string post) const { std::stringstream out; out << indent << pre << "\n"; diff --git a/src/libawkward/ListArray.cpp b/src/libawkward/ListArray.cpp index 00e1b7ff25..70ca3593da 100644 --- a/src/libawkward/ListArray.cpp +++ b/src/libawkward/ListArray.cpp @@ -31,7 +31,7 @@ namespace awkward { } else { if (length() != id.get()->length()) { - throw std::invalid_argument("content and its id must have the same length"); + util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } std::shared_ptr bigid = id; if (content_.get()->length() > kMaxInt32) { @@ -85,7 +85,7 @@ namespace awkward { } else { if (length() != id.get()->length()) { - throw std::invalid_argument("content and its id must have the same length"); + util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } std::shared_ptr bigid = id.get()->to64(); if (Identity64* rawid = dynamic_cast(bigid.get())) { @@ -154,19 +154,43 @@ namespace awkward { return std::shared_ptr(new ListArrayOf(id_, starts_, stops_, content_)); } + template + void ListArrayOf::checksafe() const { + if (stops_.length() < starts_.length()) { + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); + } + if (id_.get() != nullptr && id_.get()->length() < starts_.length()) { + util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); + } + } + template const std::shared_ptr ListArrayOf::getitem_at(int64_t at) const { int64_t regular_at = at; if (regular_at < 0) { regular_at += starts_.length(); } - if (regular_at < 0 || regular_at >= starts_.length()) { + if (!(0 <= regular_at && regular_at < starts_.length())) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } if (regular_at >= stops_.length()) { - throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } - return content_.get()->getitem_range(starts_.getitem_at(regular_at), stops_.getitem_at(regular_at)); + return getitem_at_unsafe(regular_at); + } + + template + const std::shared_ptr ListArrayOf::getitem_at_unsafe(int64_t at) const { + int64_t start = (int64_t)starts_.getitem_at_unsafe(at); + int64_t stop = (int64_t)stops_.getitem_at_unsafe(at); + int64_t lencontent = content_.get()->length(); + if (start == stop) { + start = stop = 0; + } + if (!(0 <= start && start < lencontent) || !(start <= stop && stop <= lencontent)) { + util::handle_error(failure("not 0 <= starts[i] < len(content) or not starts[i] <= stops[i] <= len(content)", kSliceNone, at), classname(), id_.get()); + } + return content_.get()->getitem_range_unsafe(start, stop); } template @@ -175,25 +199,28 @@ namespace awkward { int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), starts_.length()); if (regular_stop > stops_.length()) { - throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } + if (id_.get() != nullptr && regular_stop > id_.get()->length()) { + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); + } + return getitem_range_unsafe(regular_start, regular_stop); + } + template + const std::shared_ptr ListArrayOf::getitem_range_unsafe(int64_t start, int64_t stop) const { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { - if (regular_stop > id_.get()->length()) { - util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); - } - id = id_.get()->getitem_range(regular_start, regular_stop); + id = id_.get()->getitem_range_unsafe(start, stop); } - - return std::shared_ptr(new ListArrayOf(id, starts_.getitem_range(regular_start, regular_stop), stops_.getitem_range(regular_start, regular_stop), content_)); + return std::shared_ptr(new ListArrayOf(id, starts_.getitem_range_unsafe(start, stop), stops_.getitem_range_unsafe(start, stop), content_)); } template <> const std::shared_ptr ListArrayOf::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { int64_t lenstarts = starts_.length(); if (stops_.length() < lenstarts) { - throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } if (head.get() == nullptr) { @@ -341,7 +368,7 @@ namespace awkward { const std::shared_ptr ListArrayOf::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { int64_t lenstarts = starts_.length(); if (stops_.length() < lenstarts) { - throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } if (head.get() == nullptr) { @@ -489,7 +516,7 @@ namespace awkward { const std::shared_ptr ListArrayOf::carry(const Index64& carry) const { int64_t lenstarts = starts_.length(); if (stops_.length() < lenstarts) { - throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } Index32 nextstarts(carry.length()); Index32 nextstops(carry.length()); @@ -515,7 +542,7 @@ namespace awkward { const std::shared_ptr ListArrayOf::carry(const Index64& carry) const { int64_t lenstarts = starts_.length(); if (stops_.length() < lenstarts) { - throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); + util::handle_error(failure("len(stops) < len(starts)", kSliceNone, kSliceNone), classname(), id_.get()); } Index64 nextstarts(carry.length()); Index64 nextstops(carry.length()); diff --git a/src/libawkward/ListOffsetArray.cpp b/src/libawkward/ListOffsetArray.cpp index 62a165d037..11fb91e925 100644 --- a/src/libawkward/ListOffsetArray.cpp +++ b/src/libawkward/ListOffsetArray.cpp @@ -4,6 +4,8 @@ #include #include "awkward/cpu-kernels/identity.h" +#include "awkward/cpu-kernels/getitem.h" +#include "awkward/Slice.h" #include "awkward/ListArray.h" #include "awkward/ListOffsetArray.h" @@ -39,7 +41,7 @@ namespace awkward { } else { if (length() != id.get()->length()) { - throw std::invalid_argument("content and its id must have the same length"); + util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } Index32 starts = make_starts(offsets_); Index32 stops = make_stops(offsets_); @@ -95,7 +97,7 @@ namespace awkward { } else { if (length() != id.get()->length()) { - throw std::invalid_argument("content and its id must have the same length"); + util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } Index64 starts = make_starts(offsets_); Index64 stops = make_stops(offsets_); @@ -165,11 +167,37 @@ namespace awkward { return std::shared_ptr(new ListOffsetArrayOf(id_, offsets_, content_)); } + template + void ListOffsetArrayOf::checksafe() const { + if (id_.get() != nullptr && id_.get()->length() < offsets_.length() - 1) { + util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); + } + } + template const std::shared_ptr ListOffsetArrayOf::getitem_at(int64_t at) const { - int64_t start = (int64_t)offsets_.getitem_at(at); - int64_t stop = (int64_t)offsets_.getitem_at(at + 1); - return content_.get()->getitem_range(start, stop); + int64_t regular_at = at; + if (regular_at < 0) { + regular_at += offsets_.length() - 1; + } + if (!(0 <= regular_at && regular_at < offsets_.length() - 1)) { + util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); + } + return getitem_at_unsafe(regular_at); + } + + template + const std::shared_ptr ListOffsetArrayOf::getitem_at_unsafe(int64_t at) const { + int64_t start = (int64_t)offsets_.getitem_at_unsafe(at); + int64_t stop = (int64_t)offsets_.getitem_at_unsafe(at + 1); + int64_t lencontent = content_.get()->length(); + if (start == stop) { + start = stop = 0; + } + if (!(0 <= start && start < lencontent) || !(start <= stop && stop <= lencontent)) { + util::handle_error(failure("not 0 <= offsets[i] < len(content) or not offsets[i] <= offsets[i + 1] <= len(content)", kSliceNone, at), classname(), id_.get()); + } + return content_.get()->getitem_range_unsafe(start, stop); } template @@ -177,16 +205,19 @@ namespace awkward { int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), offsets_.length() - 1); + if (id_.get() != nullptr && regular_stop > id_.get()->length()) { + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); + } + return getitem_range_unsafe(regular_start, regular_stop); + } + template + const std::shared_ptr ListOffsetArrayOf::getitem_range_unsafe(int64_t start, int64_t stop) const { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { - if (regular_stop > id_.get()->length()) { - util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); - } - id = id_.get()->getitem_range(regular_start, regular_stop); + id = id_.get()->getitem_range_unsafe(start, stop); } - - return std::shared_ptr(new ListOffsetArrayOf(id, offsets_.getitem_range(regular_start, regular_stop + 1), content_)); + return std::shared_ptr(new ListOffsetArrayOf(id, offsets_.getitem_range_unsafe(start, stop + 1), content_)); } template <> diff --git a/src/libawkward/NumpyArray.cpp b/src/libawkward/NumpyArray.cpp index b34a4f880e..4a1a9c4906 100644 --- a/src/libawkward/NumpyArray.cpp +++ b/src/libawkward/NumpyArray.cpp @@ -190,6 +190,12 @@ namespace awkward { return std::shared_ptr(new NumpyArray(id_, ptr_, shape_, strides_, byteoffset_, itemsize_, format_)); } + void NumpyArray::checksafe() const { + if (id_.get() != nullptr && id_.get()->length() < shape_[0]) { + util::handle_error(failure("len(id) < len(array)", kSliceNone, kSliceNone), id_.get()->classname(), nullptr); + } + } + const std::shared_ptr NumpyArray::getitem_at(int64_t at) const { assert(!isscalar()); int64_t regular_at = at; @@ -199,15 +205,19 @@ namespace awkward { if (regular_at < 0 || regular_at >= shape_[0]) { util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } - ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)regular_at); + return getitem_at_unsafe(regular_at); + } + + const std::shared_ptr NumpyArray::getitem_at_unsafe(int64_t at) const { + ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)at); const std::vector shape(shape_.begin() + 1, shape_.end()); const std::vector strides(strides_.begin() + 1, strides_.end()); std::shared_ptr id; if (id_.get() != nullptr) { - if (regular_at >= id_.get()->length()) { + if (at >= id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, at), id_.get()->classname(), nullptr); } - id = id_.get()->getitem_range(regular_at, regular_at + 1); + id = id_.get()->getitem_range_unsafe(at, at + 1); } return std::shared_ptr(new NumpyArray(id, ptr_, shape, strides, byteoffset, itemsize_, format_)); } @@ -217,16 +227,20 @@ namespace awkward { int64_t regular_start = start; int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), shape_[0]); - ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)regular_start); + return getitem_range_unsafe(regular_start, regular_stop); + } + + const std::shared_ptr NumpyArray::getitem_range_unsafe(int64_t start, int64_t stop) const { + ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)start); std::vector shape; - shape.push_back((ssize_t)(regular_stop - regular_start)); + shape.push_back((ssize_t)(stop - start)); shape.insert(shape.end(), shape_.begin() + 1, shape_.end()); std::shared_ptr id; if (id_.get() != nullptr) { - if (regular_stop > id_.get()->length()) { + if (stop > id_.get()->length()) { util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } - id = id_.get()->getitem_range(regular_start, regular_stop); + id = id_.get()->getitem_range_unsafe(start, stop); } return std::shared_ptr(new NumpyArray(id, ptr_, shape, strides_, byteoffset, itemsize_, format_)); } @@ -264,7 +278,7 @@ namespace awkward { Index64 nextcarry(1); nextcarry.ptr().get()[0] = 0; Index64 nextadvanced(0); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, 1, next.strides_[0]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, 1, next.strides_[0], true); std::vector outshape(out.shape_.begin() + 1, out.shape_.end()); std::vector outstrides(out.strides_.begin() + 1, out.strides_.end()); @@ -277,7 +291,7 @@ namespace awkward { Index64 carry(shape_[0]); Error err = awkward_carry_arange_64(carry.ptr().get(), shape_[0]); util::handle_error(err, classname(), id_.get()); - return getitem_next(head, tail, carry, advanced, shape_[0], strides_[0]).shallow_copy(); + return getitem_next(head, tail, carry, advanced, shape_[0], strides_[0], false).shallow_copy(); } const std::shared_ptr NumpyArray::carry(const Index64& carry) const { @@ -505,7 +519,7 @@ namespace awkward { } } - const NumpyArray NumpyArray::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride) const { + const NumpyArray NumpyArray::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride, bool first) const { if (head.get() == nullptr) { std::shared_ptr ptr(new uint8_t[(size_t)(carry.length()*stride)], awkward::util::array_deleter()); Error err = awkward_numpyarray_getitem_next_null_64( @@ -534,7 +548,7 @@ namespace awkward { util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } - NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); @@ -558,7 +572,7 @@ namespace awkward { regular_at); util::handle_error(err, classname(), id_.get()); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length, next.strides_[0]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length, next.strides_[0], false); std::vector outshape = { (ssize_t)length }; outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); @@ -584,7 +598,7 @@ namespace awkward { int64_t m = numer % denom; int64_t lenhead = d + (m != 0 ? 1 : 0); - NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); @@ -600,7 +614,7 @@ namespace awkward { step); util::handle_error(err, classname(), id_.get()); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*lenhead, next.strides_[0]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*lenhead, next.strides_[0], false); std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); std::vector outstrides = { (ssize_t)lenhead*out.strides_[0] }; @@ -623,7 +637,7 @@ namespace awkward { step); util::handle_error(err, classname(), id_.get()); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*lenhead, next.strides_[0]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*lenhead, next.strides_[0], false); std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); std::vector outstrides = { (ssize_t)lenhead*out.strides_[0] }; @@ -640,7 +654,7 @@ namespace awkward { if (tail.length() == 0 || mindepth - 1 == tail.dimlength()) { std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); - return getitem_next(nexthead, nexttail, carry, advanced, length, stride); + return getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); } else { std::vector> tailitems = tail.items(); @@ -648,14 +662,14 @@ namespace awkward { items.insert(items.end(), tailitems.begin(), tailitems.end()); std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); Slice nexttail(items); - return getitem_next(nexthead, nexttail, carry, advanced, length, stride); + return getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); } } else if (SliceNewAxis* newaxis = dynamic_cast(head.get())) { std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); - NumpyArray out = getitem_next(nexthead, nexttail, carry, advanced, length, stride); + NumpyArray out = getitem_next(nexthead, nexttail, carry, advanced, length, stride, false); std::vector outshape = { (ssize_t)length, 1 }; outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); @@ -669,7 +683,7 @@ namespace awkward { util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } - NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); + NumpyArray next(first ? id_ : std::shared_ptr(nullptr), ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); std::shared_ptr nexthead = tail.head(); Slice nexttail = tail.tail(); @@ -693,7 +707,7 @@ namespace awkward { shape_[1]); // because this is contiguous util::handle_error(err, classname(), id_.get()); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*flathead.length(), next.strides_[0]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*flathead.length(), next.strides_[0], false); std::vector outshape = { (ssize_t)length }; std::vector arrayshape = array->shape(); @@ -721,7 +735,7 @@ namespace awkward { shape_[1]); // because this is contiguous util::handle_error(err, classname(), id_.get()); - NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*array->length(), next.strides_[0]); + NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*array->length(), next.strides_[0], false); std::vector outshape = { (ssize_t)length }; outshape.insert(outshape.end(), out.shape_.begin() + 1, out.shape_.end()); diff --git a/src/libawkward/Slice.cpp b/src/libawkward/Slice.cpp index 111624da9b..1448c06545 100644 --- a/src/libawkward/Slice.cpp +++ b/src/libawkward/Slice.cpp @@ -40,7 +40,7 @@ namespace awkward { if (i != 0) { out << ", "; } - out << (T)index_.getitem_at(i*strides_[0]); + out << (T)index_.getitem_at_unsafe(i*strides_[0]); } } else { @@ -48,14 +48,14 @@ namespace awkward { if (i != 0) { out << ", "; } - out << (T)index_.getitem_at(i*strides_[0]); + out << (T)index_.getitem_at_unsafe(i*strides_[0]); } out << ", ..., "; for (int64_t i = shape_[0] - 3; i < shape_[0]; i++) { if (i != shape_[0] - 3) { out << ", "; } - out << (T)index_.getitem_at(i*strides_[0]); + out << (T)index_.getitem_at_unsafe(i*strides_[0]); } } } diff --git a/src/pyawkward.cpp b/src/pyawkward.cpp index ac76080281..9678477688 100644 --- a/src/pyawkward.cpp +++ b/src/pyawkward.cpp @@ -215,7 +215,7 @@ py::class_> make_IdentityOf(py::handle m, std::string name) { .def("__repr__", &ak::IdentityOf::tostring) .def("__len__", &ak::IdentityOf::length) - .def("__getitem__", &ak::IdentityOf::get) + .def("__getitem__", &ak::IdentityOf::getitem_at) .def("__getitem__", &ak::IdentityOf::getitem_range) .def_property_readonly("ref", &ak::IdentityOf::ref) diff --git a/tests/test_refcount.py b/tests/test_PR001_refcount.py similarity index 100% rename from tests/test_refcount.py rename to tests/test_PR001_refcount.py diff --git a/tests/test_PR2_minimal_listarray.py b/tests/test_PR002_minimal_listarray.py similarity index 100% rename from tests/test_PR2_minimal_listarray.py rename to tests/test_PR002_minimal_listarray.py diff --git a/tests/test_PR3_minimal_numba_listarray.py b/tests/test_PR003_minimal_numba_listarray.py similarity index 100% rename from tests/test_PR3_minimal_numba_listarray.py rename to tests/test_PR003_minimal_numba_listarray.py diff --git a/tests/test_PR4_design_surrogate_key.py b/tests/test_PR004_design_surrogate_key.py similarity index 100% rename from tests/test_PR4_design_surrogate_key.py rename to tests/test_PR004_design_surrogate_key.py diff --git a/tests/test_PR5_surrogate_key_in_numba.py b/tests/test_PR005_surrogate_key_in_numba.py similarity index 100% rename from tests/test_PR5_surrogate_key_in_numba.py rename to tests/test_PR005_surrogate_key_in_numba.py diff --git a/tests/test_PR6_deep_iteration.py b/tests/test_PR006_deep_iteration.py similarity index 100% rename from tests/test_PR6_deep_iteration.py rename to tests/test_PR006_deep_iteration.py diff --git a/tests/test_PR8_slices_and_getitem.py b/tests/test_PR008_slices_and_getitem.py similarity index 100% rename from tests/test_PR8_slices_and_getitem.py rename to tests/test_PR008_slices_and_getitem.py diff --git a/tests/test_PR9_identity_and_getitem.py b/tests/test_PR009_identity_and_getitem.py similarity index 100% rename from tests/test_PR9_identity_and_getitem.py rename to tests/test_PR009_identity_and_getitem.py diff --git a/tests/test_PR11_listarray.py b/tests/test_PR011_listarray.py similarity index 100% rename from tests/test_PR11_listarray.py rename to tests/test_PR011_listarray.py diff --git a/tests/test_PR12_listarray_in_numba.py b/tests/test_PR012_listarray_in_numba.py similarity index 100% rename from tests/test_PR12_listarray_in_numba.py rename to tests/test_PR012_listarray_in_numba.py diff --git a/tests/test_PR13_error_handling_struct.py b/tests/test_PR013_error_handling_struct.py similarity index 100% rename from tests/test_PR13_error_handling_struct.py rename to tests/test_PR013_error_handling_struct.py diff --git a/tests/test_PR014_finish_up_getitem.py b/tests/test_PR014_finish_up_getitem.py new file mode 100644 index 0000000000..953ff9ec13 --- /dev/null +++ b/tests/test_PR014_finish_up_getitem.py @@ -0,0 +1,236 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import sys +import itertools + +import pytest +import numpy + +import awkward1 + +py27 = (sys.version_info[0] < 3) + +content = awkward1.layout.NumpyArray(numpy.arange(2*3*5*7).reshape(-1, 7)) +offsetsA = numpy.arange(0, 2*3*5 + 5, 5) +offsetsB = numpy.arange(0, 2*3 + 3, 3) +startsA, stopsA = offsetsA[:-1], offsetsA[1:] +startsB, stopsB = offsetsB[:-1], offsetsB[1:] + +listoffsetarrayA64 = awkward1.layout.ListOffsetArray64(awkward1.layout.Index64(offsetsA), content) +listoffsetarrayA32 = awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(offsetsA), content) +listarrayA64 = awkward1.layout.ListArray64(awkward1.layout.Index64(startsA), awkward1.layout.Index64(stopsA), content) +listarrayA32 = awkward1.layout.ListArray32(awkward1.layout.Index32(startsA), awkward1.layout.Index32(stopsA), content) +modelA = numpy.arange(2*3*5*7).reshape(2*3, 5, 7) + +listoffsetarrayB64 = awkward1.layout.ListOffsetArray64(awkward1.layout.Index64(offsetsB), listoffsetarrayA64) +listoffsetarrayB32 = awkward1.layout.ListOffsetArray32(awkward1.layout.Index32(offsetsB), listoffsetarrayA64) +listarrayB64 = awkward1.layout.ListArray64(awkward1.layout.Index64(startsB), awkward1.layout.Index64(stopsB), listarrayA64) +listarrayB32 = awkward1.layout.ListArray32(awkward1.layout.Index32(startsB), awkward1.layout.Index32(stopsB), listarrayA64) +modelB = numpy.arange(2*3*5*7).reshape(2, 3, 5, 7) + +listoffsetarrayB64.setid() +listoffsetarrayB32.setid() +listarrayB64.setid() +listarrayB32.setid() + +def test_basic(): + assert awkward1.tolist(modelA) == awkward1.tolist(listoffsetarrayA64) + assert awkward1.tolist(modelA) == awkward1.tolist(listoffsetarrayA32) + assert awkward1.tolist(modelA) == awkward1.tolist(listarrayA64) + assert awkward1.tolist(modelA) == awkward1.tolist(listarrayA32) + assert awkward1.tolist(modelB) == awkward1.tolist(listoffsetarrayB64) + assert awkward1.tolist(modelB) == awkward1.tolist(listoffsetarrayB32) + assert awkward1.tolist(modelB) == awkward1.tolist(listarrayB64) + assert awkward1.tolist(modelB) == awkward1.tolist(listarrayB32) + +def test_listoffsetarrayA64(): + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((0, 1, 4, -5), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(None), slice(1, None), slice(None, -1), slice(None, None, 2)), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(1, None), slice(None, -1), 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], slice(1, None), slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA64[cuts]) + +def test_listoffsetarrayA32(): + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((0, 1, 4, -5), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(None), slice(1, None), slice(None, -1), slice(None, None, 2)), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(1, None), slice(None, -1), 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], slice(1, None), slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listoffsetarrayA32[cuts]) + +def test_listarrayA64(): + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((0, 1, 4, -5), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(None), slice(1, None), slice(None, -1), slice(None, None, 2)), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(1, None), slice(None, -1), 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA64[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], slice(1, None), slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA64[cuts]) + +def test_listarrayA32(): + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((0, 1, 4, -5), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(None), slice(1, None), slice(None, -1), slice(None, None, 2)), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations((slice(1, None), slice(None, -1), 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], 2, -2), depth): + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA32[cuts]) + + for depth in 0, 1, 2, 3: + for cuts in itertools.permutations(([2, 0, 0, 1], [1, -2, 0, -1], slice(1, None), slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelA[cuts]) == awkward1.tolist(listarrayA32[cuts]) + +def test_listoffsetarrayB64(): + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-2, -1, 0, 1, 1), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listoffsetarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, 1, slice(1, None), slice(None, -1)), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listoffsetarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, [1, 0, 0, 1], [0, 1, -1, 1], slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listoffsetarrayB64[cuts]) + +def test_listoffsetarrayB32(): + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-2, -1, 0, 1, 1), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listoffsetarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, 1, slice(1, None), slice(None, -1)), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listoffsetarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, [1, 0, 0, 1], [0, 1, -1, 1], slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listoffsetarrayB64[cuts]) + +def test_listarrayB64(): + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-2, -1, 0, 1, 1), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, 1, slice(1, None), slice(None, -1)), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, [1, 0, 0, 1], [0, 1, -1, 1], slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listarrayB64[cuts]) + +def test_listarrayB32(): + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-2, -1, 0, 1, 1), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, 1, slice(1, None), slice(None, -1)), depth): + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listarrayB64[cuts]) + + for depth in 0, 1, 2, 3, 4: + for cuts in itertools.permutations((-1, 0, [1, 0, 0, 1], [0, 1, -1, 1], slice(None, -1)), depth): + cuts = cuts + while len(cuts) > 0 and isinstance(cuts[0], slice): + cuts = cuts[1:] + while len(cuts) > 0 and isinstance(cuts[-1], slice): + cuts = cuts[:-1] + if any(isinstance(x, slice) for x in cuts): + continue + assert awkward1.tolist(modelB[cuts]) == awkward1.tolist(listarrayB64[cuts])