Merge pull request #400 from scikit-hep/issue-399

Added interpretation of vector<class*>

tests/test_issues.py

@@ -240,3 +240,9 @@ def test_issue390(self):
  t = uproot.open("tests/samples/issue390.root")["E"]
  t.pandas.df("hits.*")
  t.pandas.df("trks.*")
+
+ def test_issue399(self):
+ t = uproot.open("tests/samples/issue399.root")["Event"]
+ a = t["Histos.histograms1D"].array()
+ for i in range(t.numentries):
+ assert [x.title for x in a[i]] == [b"Primary Hits", b"Primary Loss", b"Energy Loss", b"Primary Hits per Element", b"Primary Loss per Element", b"Energy Loss per Element"]

uproot/__init__.py

@@ -142,6 +142,7 @@
 from uproot.interp.objects import STLVector
 from uproot.interp.objects import STLMap
 from uproot.interp.objects import STLString
+from uproot.interp.objects import Pointer
 asdebug = asjagged(asdtype("u1"))
 
 from uproot import pandas
@@ -157,4 +158,4 @@
 # don't expose uproot.uproot; it's ugly
 del uproot
 
-__all__ = ["open", "xrootd", "http", "iterate", "numentries", "lazyarray", "lazyarrays", "daskarray", "daskframe", "create", "recreate", "update", "ZLIB", "LZMA", "LZ4", "newtree", "newbranch", "MemmapSource", "FileSource", "XRootDSource", "HTTPSource", "ArrayCache", "ThreadSafeArrayCache", "interpret", "asdtype", "asarray", "asdouble32", "asstlbitset", "asjagged", "astable", "asobj", "asgenobj", "asstring", "asdebug", "SimpleArray", "STLVector", "STLMap", "STLString", "pandas", "__version__"]
+__all__ = ["open", "xrootd", "http", "iterate", "numentries", "lazyarray", "lazyarrays", "daskarray", "daskframe", "create", "recreate", "update", "ZLIB", "LZMA", "LZ4", "newtree", "newbranch", "MemmapSource", "FileSource", "XRootDSource", "HTTPSource", "ArrayCache", "ThreadSafeArrayCache", "interpret", "asdtype", "asarray", "asdouble32", "asstlbitset", "asjagged", "astable", "asobj", "asgenobj", "asstring", "asdebug", "SimpleArray", "STLVector", "STLMap", "STLString", "Pointer", "pandas", "__version__"]

uproot/_help.py

@@ -1913,7 +1913,7 @@ def _method(x):
  **source_numitems(self, source)**
  calculate the number of "items" given a ``source`` instance.
 
- **fromroot(self, data, offsets, local_entrystart, local_entrystop)**
+ **fromroot(self, data, offsets, local_entrystart, local_entrystop, keylen)**
  produce a source from one basket ``data`` array (dtype ``numpy.uint8``) and its corresponding ``offsets`` array (dtype **numpy.int32** or ``None`` if not present) that has *n + 1* elements for *n* entries: ``offsets[0] == 0 and offsets[-1] == numentries``. The ``local_entrystart`` and ``local_entrystop`` are entry start (inclusive) and stop (exclusive), in which the first entry in the basket is number zero (hence "local"). The result of this operation may be a zero-copy cast of the basket data.
 
  **destination(self, numitems, numentries)**

uproot/interp/auto.py

@@ -25,6 +25,7 @@
 from uproot.interp.objects import STLVector
 from uproot.interp.objects import STLMap
 from uproot.interp.objects import STLString
+from uproot.interp.objects import Pointer
 
 class _NotNumerical(Exception): pass
 
@@ -380,6 +381,11 @@ def transform(node, tofloat=True):
  return asjagged(asdtype("f8"), skipbytes=10)
  elif getattr(branch._streamer, "_fTypeName", None) == b"vector<string>":
  return asgenobj(STLVector(STLString(awkward)), branch._context, 6)
+ else:
+ m = interpret._vectorpointer.match(getattr(branch._streamer, "_fTypeName", b""))
+ if m is not None and m.group(1) in branch._context.streamerinfosmap:
+ streamer = branch._context.streamerinfosmap[m.group(1)]
+ return asgenobj(STLVector(Pointer(streamer.pyclass)), branch._context, skipbytes=6)
 
  if getattr(branch._streamer, "_fTypeName", None) == b"map<string,bool>" or getattr(branch._streamer, "_fTypeName", None) == b"map<string,Bool_t>":
  return asgenobj(STLMap(STLString(awkward), asdtype(awkward.numpy.bool_)), branch._context, 6)
@@ -550,4 +556,5 @@ def transform(node, tofloat=True):
 interpret._titlehasdims = re.compile(br"^([^\[\]]+)(\[[^\[\]]+\])+")
 interpret._itemdimpattern = re.compile(br"\[([1-9][0-9]*)\]")
 interpret._itemanypattern = re.compile(br"\[(.*)\]")
+interpret._vectorpointer = re.compile(br"vector\<([^<>]*)\*\>")
 interpret._pairsecond = re.compile(br"pair\<[^<>]*,(.*) \>")

uproot/interp/interp.py

@@ -41,7 +41,7 @@ def numitems(self, numbytes, numentries):
  def source_numitems(self, source):
  raise NotImplementedError
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
  raise NotImplementedError
 
  def destination(self, numitems, numentries):

uproot/interp/jagged.py

@@ -66,15 +66,15 @@ def numitems(self, numbytes, numentries):
  def source_numitems(self, source):
  return self.content.source_numitems(source.content)
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
  if local_entrystart == local_entrystop:
- return self.awkward.JaggedArray.fromoffsets([0], self.content.fromroot(data, None, local_entrystart, local_entrystop))
+ return self.awkward.JaggedArray.fromoffsets([0], self.content.fromroot(data, None, local_entrystart, local_entrystop, keylen))
  else:
  if self.skipbytes == 0:
  offsets = _destructive_divide(byteoffsets, self.content.itemsize, self.awkward)
  starts = offsets[local_entrystart : local_entrystop ]
  stops = offsets[local_entrystart + 1 : local_entrystop + 1]
- content = self.content.fromroot(data, None, starts[0], stops[-1])
+ content = self.content.fromroot(data, None, starts[0], stops[-1], keylen)
  return self.awkward.JaggedArray(starts, stops, content)
 
  else:
@@ -87,7 +87,7 @@ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
  self.awkward.numpy.cumsum(mask, out=mask)
  data = data[mask.view(self.awkward.numpy.bool_)]
 
- content = self.content.fromroot(data, None, 0, bytestops[-1])
+ content = self.content.fromroot(data, None, 0, bytestops[-1], keylen)
 
  itemsize = 1
  sub = self.content

uproot/interp/numerical.py

@@ -152,7 +152,7 @@ def numitems(self, numbytes, numentries):
  assert remainder == 0
  return quotient
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
  dtype, shape = _dtypeshape(self.fromdtype)
  return data.view(dtype).reshape((-1,) + shape)[local_entrystart:local_entrystop]
 
@@ -269,7 +269,7 @@ def numitems(self, numbytes, numentries):
  assert remainder == 0
  return quotient
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
  # Interpret input data using proper type
  array = data.view(dtype=self.fromdtypeflat)
  # Make sure the interpreted data has correct shape
@@ -359,7 +359,7 @@ def numitems(self, numbytes, numentries):
  def source_numitems(self, source):
  return int(self.awkward.numpy.prod(source.shape))
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
  return data.view(self.todtype).reshape((-1, self.numbytes + 4))[:, 4:]
 
  def destination(self, numitems, numentries):

uproot/interp/objects.py

@@ -6,7 +6,11 @@
 
 import copy
 import struct
+import numbers
 
+import numpy
+
+import uproot.rootio
 import uproot.interp.interp
 import uproot.interp.numerical
 import uproot.interp.jagged
@@ -129,6 +133,25 @@ def read(self, source, cursor, context, parent):
  numitems = cursor.field(source, self._format2)
  return cursor.array(source, numitems, self.awkward.ObjectArray.CHARTYPE).tostring()
 
+class Pointer(object):
+ def __init__(self, cls):
+ self.cls = cls
+
+ @property
+ def __name__(self):
+ return "Pointer"
+
+ def __repr__(self):
+ if isinstance(self.cls, type):
+ return "Pointer({0})".format(self.cls.__name__)
+ else:
+ return "Pointer({0})".format(repr(self.cls))
+
+ def read(self, source, cursor, context, parent):
+ return uproot.rootio._readobjany(source, cursor, context, parent)
+
+ _format1 = struct.Struct(">II")
+
 class astable(uproot.interp.interp.Interpretation):
  # makes __doc__ attribute mutable before Python 3.3
  __metaclass__ = type.__new__(type, "type", (uproot.interp.interp.Interpretation.__metaclass__,), {})
@@ -180,8 +203,8 @@ def numitems(self, numbytes, numentries):
  def source_numitems(self, source):
  return self.content.source_numitems(source)
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
- return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+ return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
 
  def destination(self, numitems, numentries):
  return self.content.destination(numitems, numentries)
@@ -233,8 +256,8 @@ def numitems(self, numbytes, numentries):
  def source_numitems(self, source):
  return self.content.source_numitems(source)
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
- return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+ return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
 
  def destination(self, numitems, numentries):
  return self.content.destination(numitems, numentries)
@@ -286,14 +309,14 @@ def numitems(self, numbytes, numentries):
  def source_numitems(self, source):
  return self.content.source_numitems(source)
 
- def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop):
- return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+ return self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
 
  def destination(self, numitems, numentries):
  return self.content.destination(numitems, numentries)
 
  def fill(self, source, destination, itemstart, itemstop, entrystart, entrystop):
- return self.content.fill(source, destination, itemstart, itemstop, entrystart, entrystop)
+ self.content.fill(source, destination, itemstart, itemstop, entrystart, entrystop)
 
  def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
  return self.content.clip(destination, itemstart, itemstop, entrystart, entrystop)
@@ -304,17 +327,55 @@ def finalize(self, destination, branch):
  print("reading {0}".format(repr(out)))
  return out
 
-class asgenobj(_variable):
+class _variable_withoffsets(_variable):
+ def fromroot(self, data, byteoffsets, local_entrystart, local_entrystop, keylen):
+ out = self.content.fromroot(data, byteoffsets, local_entrystart, local_entrystop, keylen)
+ out.byteoffsets = byteoffsets[local_entrystart:local_entrystop] + keylen + self.content.skipbytes
+ return out
+
+ def destination(self, numitems, numentries):
+ out = self.content.destination(numitems, numentries)
+ out.byteoffsets = self.awkward.numpy.empty(numentries, dtype=self.awkward.numpy.int32)
+ return out
+
+ def fill(self, source, destination, itemstart, itemstop, entrystart, entrystop):
+ self.content.fill(source, destination, itemstart, itemstop, entrystart, entrystop)
+ destination.byteoffsets[entrystart:entrystop] = source.byteoffsets
+
+ def clip(self, destination, itemstart, itemstop, entrystart, entrystop):
+ out = self.content.clip(destination, itemstart, itemstop, entrystart, entrystop)
+ out.byteoffsets = destination.byteoffsets[entrystart:entrystop]
+ return out
+
+ def finalize(self, destination, branch):
+ out = self.awkward.ObjectArray(JaggedWithByteOffsets(self.content.finalize(destination, branch), destination.byteoffsets), self.generator, *self.args, **self.kwargs)
+ if self.debug_reading:
+ print("reading {0}".format(repr(out)))
+ return out
+
+class JaggedWithByteOffsets(object):
+ def __init__(self, jagged, byteoffsets):
+ self.jagged = jagged
+ self.byteoffsets = byteoffsets
+
+ def __len__(self):
+ return len(self.jagged)
+
+ def __getitem__(self, where):
+ return self.jagged[where], -self.byteoffsets[where]
+
+class asgenobj(_variable_withoffsets):
  # makes __doc__ attribute mutable before Python 3.3
  __metaclass__ = type.__new__(type, "type", (_variable.__metaclass__,), {})
 
  class _Wrapper(object):
  def __init__(self, cls, context):
  self.cls = cls
  self.context = context
- def __call__(self, bytes):
+ def __call__(self, arg):
+ bytes, origin = arg
  source = uproot.source.source.Source(bytes)
- cursor = uproot.source.cursor.Cursor(0)
+ cursor = uproot.source.cursor.Cursor(0, origin=origin)
  return self.cls.read(source, cursor, self.context, None)
  def __repr__(self):
  if isinstance(self.cls, type):

uproot/tree.py

@@ -1194,7 +1194,7 @@ def _basket(self, i, interpretation, local_entrystart, local_entrystop, awkward,
  byteoffsets[-1] = key._fLast
  awkward.numpy.subtract(byteoffsets, key._fKeylen, byteoffsets)
 
- return interpretation.fromroot(data, byteoffsets, local_entrystart, local_entrystop)
+ return interpretation.fromroot(data, byteoffsets, local_entrystart, local_entrystop, key._fKeylen)
 
  def basket(self, i, interpretation=None, entrystart=None, entrystop=None, flatten=False, awkwardlib=None, cache=None, basketcache=None, keycache=None):
  awkward = _normalize_awkwardlib(awkwardlib)

uproot/version.py

@@ -6,7 +6,7 @@
 
 import re
 
-__version__ = "3.10.10"
+__version__ = "3.10.11"
 version = __version__
 version_info = tuple(re.split(r"[-\.]", __version__))