[pypy-commit] pypy py3.5: CPython's ``sys.settrace()`` sometimes reports an ``exception`` at the

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89045:d86691a0c3ca
Date: 2016-12-13 15:48 +0100
http://bitbucket.org/pypy/pypy/changeset/d86691a0c3ca/

Log:	CPython's ``sys.settrace()`` sometimes reports an ``exception`` at
	the end of ``for`` or ``yield from`` lines for the
	``StopIteration``, and sometimes not. The problem is that it occurs
	in an ill-defined subset of cases. PyPy attempts to emulate that but
	the precise set of cases is not exactly the same.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -478,6 +478,12 @@
   from the Makefile used to build the interpreter. PyPy should bake the values
   in during compilation, but does not do that yet.
 
+* CPython's ``sys.settrace()`` sometimes reports an ``exception`` at the
+  end of ``for`` or ``yield from`` lines for the ``StopIteration``, and
+  sometimes not.  The problem is that it occurs in an ill-defined subset
+  of cases.  PyPy attempts to emulate that but the precise set of cases
+  is not exactly the same.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -309,6 +309,9 @@
             tb.frame.mark_as_escaped()
         return tb
 
+    def has_any_traceback(self):
+        return self._application_traceback is not None
+
     def set_cause(self, space, w_cause):
         if w_cause is None:
             return
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -185,7 +185,7 @@
             if not e.match(space, space.w_StopIteration):
                 raise
             e.normalize_exception(space)
-            space.getexecutioncontext().exception_trace(frame, e)
+            frame._report_stopiteration_sometimes(w_yf, e)
             try:
                 w_stop_value = space.getattr(e.get_w_value(space),
                                              space.wrap("value"))
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1154,13 +1154,31 @@
             if not e.match(self.space, self.space.w_StopIteration):
                 raise
             # iterator exhausted
-            self.space.getexecutioncontext().exception_trace(self, e)
+            self._report_stopiteration_sometimes(w_iterator, e)
             self.popvalue()
             next_instr += jumpby
         else:
             self.pushvalue(w_nextitem)
         return next_instr
 
+    def _report_stopiteration_sometimes(self, w_iterator, operr):
+        # CPython 3.5 calls the exception trace in an ill-defined subset
+        # of cases: only if tp_iternext returned NULL and set a
+        # StopIteration exception, but not if tp_iternext returned NULL
+        # *without* setting an exception.  We can't easily emulate that
+        # behavior at this point.  For example, the generator's
+        # tp_iternext uses one or other case depending on whether the
+        # generator is already exhausted or just exhausted now.  We'll
+        # classify that as a CPython incompatibility and use an
+        # approximative rule: if w_iterator is a generator-iterator,
+        # we always report it; if operr has already a stack trace
+        # attached (likely from a custom __iter__() method), we also
+        # report it; in other cases, we don't.
+        from pypy.interpreter.generator import GeneratorOrCoroutine
+        if (isinstance(w_iterator, GeneratorOrCoroutine) or
+                operr.has_any_traceback()):
+            self.space.getexecutioncontext().exception_trace(self, operr)
+
     def FOR_LOOP(self, oparg, next_instr):
         raise BytecodeCorruption("old opcode, no longer in use")
 
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -666,10 +666,33 @@
         sys.settrace(None)
         print('seen:', seen)
         # on Python 3 we get an extra 'exception' when 'for' catches
-        # StopIteration
+        # StopIteration (but not always! mess)
         assert seen == ['call', 'line', 'call', 'return', 'exception', 'return']
         assert frames[-2].f_code.co_name == 'g'
 
+    def test_nongenerator_trace_stopiteration(self):
+        import sys
+        gen = iter([5])
+        assert next(gen) == 5
+        seen = []
+        frames = []
+        def trace_func(frame, event, *args):
+            print('TRACE:', frame, event, args)
+            seen.append(event)
+            frames.append(frame)
+            return trace_func
+        def g():
+            for x in gen:
+                never_entered
+        sys.settrace(trace_func)
+        g()
+        sys.settrace(None)
+        print('seen:', seen)
+        # hack: don't report the StopIteration for some "simple"
+        # iterators.
+        assert seen == ['call', 'line', 'return']
+        assert frames[-2].f_code.co_name == 'g'
+
     def test_yieldfrom_trace_stopiteration(self): """
         import sys
         def f2():
@@ -726,6 +749,30 @@
         assert frames[-2].f_code.co_name == 'g'
         """
 
+    def test_yieldfrom_trace_stopiteration_3(self): """
+        import sys
+        def f():
+            yield from []
+        gen = f()
+        seen = []
+        frames = []
+        def trace_func(frame, event, *args):
+            print('TRACE:', frame, event, args)
+            seen.append(event)
+            frames.append(frame)
+            return trace_func
+        def g():
+            for x in gen:
+                never_entered
+        sys.settrace(trace_func)
+        g()      # invokes next_yield_from() from YIELD_FROM()
+        sys.settrace(None)
+        print('seen:', seen)
+        assert seen == ['call', 'line', 'call', 'line',
+                        'return', 'exception', 'return']
+        assert frames[-4].f_code.co_name == 'f'
+        """
+
     def test_clear_locals(self):
         def make_frames():
             def outer():