support lazy batching again, support general iterators

CHANGES.rst

@@ -6,6 +6,9 @@ Changelog
 
 - Drop support for Python 3.7.
 
+- Support lazy batching again, support general iterators
+  (`#75 <https://github.com/zopefoundation/DocumentTemplate/issues/75>`_)
+
 
 4.6 (2023-11-13)
 ----------------

src/DocumentTemplate/DT_In.py

@@ -347,6 +347,7 @@
 from .DT_Util import add_with_prefix
 from .DT_Util import name_param
 from .DT_Util import parse_params
+from .DT_Util import sequence_ensure_subscription
 from .DT_Util import simple_name
 
 
@@ -459,25 +460,25 @@ def renderwb(self, md):
         expr = self.expr
         name = self.__name__
         if expr is None:
-            sequence = md[name]
+            sequence = sequence_ensure_subscription(md[name])
             cache = {name: sequence}
         else:
-            sequence = expr(md)
+            sequence = sequence_ensure_subscription(expr(md))
             cache = None
 
-        if not sequence:
-            if self.elses:
-                return render_blocks(self.elses, md, encoding=self.encoding)
-            return ''
-
         if isinstance(sequence, str):
             raise ValueError(
                 'Strings are not allowed as input to the in tag.')
 
-        # Turn iterable like dict.keys() into a list.
-        sequence = list(sequence)
-        if cache is not None:
-            cache[name] = sequence
+        # below we do not use ``not sequence`` because the
+        # implied ``__len__`` is expensive for some (lazy) sequences
+        # if not sequence:
+        try:
+            sequence[0]
+        except IndexError:
+            if self.elses:
+                return render_blocks(self.elses, md, encoding=self.encoding)
+            return ''
 
         section = self.section
         params = self.args
@@ -667,25 +668,25 @@ def renderwob(self, md):
         expr = self.expr
         name = self.__name__
         if expr is None:
-            sequence = md[name]
+            sequence = sequence_ensure_subscription(md[name])
             cache = {name: sequence}
         else:
-            sequence = expr(md)
+            sequence = sequence_ensure_subscription(expr(md))
             cache = None
 
-        if not sequence:
-            if self.elses:
-                return render_blocks(self.elses, md, encoding=self.encoding)
-            return ''
-
         if isinstance(sequence, str):
             raise ValueError(
                 'Strings are not allowed as input to the in tag.')
 
-        # Turn iterable like dict.keys() into a list.
-        sequence = list(sequence)
-        if cache is not None:
-            cache[name] = sequence
+        # below we do not use ``not sequence`` because the
+        # implied ``__len__`` is expensive for some (lazy) sequences
+        # if not sequence:
+        try:
+            sequence[0]
+        except IndexError:
+            if self.elses:
+                return render_blocks(self.elses, md, encoding=self.encoding)
+            return ''
 
         section = self.section
         mapping = self.mapping

src/DocumentTemplate/DT_InSV.py

@@ -17,6 +17,8 @@
 
 import roman
 
+from .DT_Util import sequence_ensure_subscription
+
 
 try:
     import Missing
@@ -37,8 +39,7 @@ def __init__(self,
                  start_name_re=None,
                  alt_prefix=''):
         if items is not None:
-            # Turn iterable into a list, to support key lookup
-            items = list(items)
+            items = sequence_ensure_subscription(items)
         self.items = items
         self.query_string = query_string
         self.start_name_re = start_name_re

src/DocumentTemplate/DT_Util.py

@@ -1,6 +1,6 @@
 ##############################################################################
 #
-# Copyright (c) 2002 Zope Foundation and Contributors.
+# Copyright (c) 2002-2024 Zope Foundation and Contributors.
 #
 # This software is subject to the provisions of the Zope Public License,
 # Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
@@ -450,3 +450,67 @@ def parse_params(text,
         return parse_params(text, result, **parms)
     else:
         return result
+
+
+def sequence_supports_subscription(obj):
+    """Check whether *obj* supports sequence subscription.
+
+    We are using a heuristics.
+    """
+    # check wether *obj* might support sequence subscription
+    try:
+        obj[0]
+    except IndexError:
+        return True
+    except (AttributeError, TypeError, KeyError):
+        return False
+    # check that *obj* is not a mapping
+    try:
+        obj[None]
+    except (TypeError, ValueError, RuntimeError):
+        # we may need more exceptions above
+        # (to support sequence like objects using strange exceptions)
+        return True
+    except KeyError:
+        pass
+    return False
+
+
+def sequence_ensure_subscription(obj):
+    """return an *obj* wrapper supporting sequence subscription.
+
+    *obj* must either support sequence subscription itself
+    (and then is returned unwrapped) or be iterable.
+    """
+    if sequence_supports_subscription(obj):
+        return obj
+    return SequenceFromIter(iter(obj))
+
+
+class SequenceFromIter:
+    """Iterator wrapper supporting lazy sequence subscription."""
+
+    finished = False
+
+    def __init__(self, it):
+        self.it = it
+        self.data = []
+
+    def __getitem__(self, idx):
+        if idx < 0:
+            raise IndexError(f"negative indexes are not supported {idx}")
+        while not self.finished and idx >= len(self.data):
+            try:
+                self.data.append(next(self.it))
+            except StopIteration:
+                self.finished = True
+        return self.data[idx]
+
+    def __len__(self):
+        """the size -- ATT: expensive!"""
+        while not self.finished:
+            try:
+                self[len(self.data)]
+            except IndexError:
+                pass
+        return len(self.data)

src/DocumentTemplate/tests/test_Util.py

@@ -0,0 +1,67 @@
+from unittest import TestCase
+
+from ..DT_Util import SequenceFromIter
+from ..DT_Util import sequence_ensure_subscription
+from ..DT_Util import sequence_supports_subscription
+
+
+class SequenceTests(TestCase):
+    def test_supports_str(self):
+        self.assertTrue(sequence_supports_subscription(""))
+
+    def test_supports_sequence(self):
+        self.assertTrue(sequence_supports_subscription([]))
+        self.assertTrue(sequence_supports_subscription([0]))
+
+    def test_supports_mapping(self):
+        self.assertFalse(sequence_supports_subscription({}))
+        self.assertFalse(sequence_supports_subscription({0: 0}))
+        self.assertFalse(sequence_supports_subscription({0: 0, None: None}))
+
+    def test_supports_iter(self):
+        self.assertFalse(sequence_supports_subscription((i for i in range(0))))
+        self.assertFalse(sequence_supports_subscription((i for i in range(1))))
+
+    def test_supports_SequenceFromIter(self):
+        S = SequenceFromIter
+        self.assertTrue(
+            sequence_supports_subscription(S((i for i in range(0)))))
+        self.assertTrue(
+            sequence_supports_subscription(S((i for i in range(1)))))
+
+    def test_supports_RuntimeError(self):
+        # check that ``ZTUtils.Lazy.Lazy`` is recognized
+        class RTSequence(list):
+            def __getitem__(self, idx):
+                if not isinstance(idx, int):
+                    raise RuntimeError
+
+        s = RTSequence(i for i in range(0))
+        self.assertTrue(sequence_supports_subscription(s))
+        s = RTSequence(i for i in range(2))
+        self.assertTrue(sequence_supports_subscription(s))
+
+    def test_ensure_sequence(self):
+        s = []
+        self.assertIs(s, sequence_ensure_subscription(s))
+
+    def test_ensure_iter(self):
+        self.assertIsInstance(
+            sequence_ensure_subscription(i for i in range(0)),
+            SequenceFromIter)
+
+    def test_FromIter(self):
+        S = SequenceFromIter
+        with self.assertRaises(IndexError):
+            S(i for i in range(0))[0]
+        s = S(i for i in range(2))
+        with self.assertRaises(IndexError):
+            s[-1]
+        self.assertEqual(s[0], 0)
+        self.assertEqual(s[0], 0)  # ensure nothing bad happens
+        self.assertEqual(s[1], 1)
+        with self.assertRaises(IndexError):
+            s[2]
+        self.assertEqual(list(s), [0, 1])
+        self.assertEqual(len(s), 2)
+        self.assertEqual(len(S(i for i in range(2))), 2)