[Python-checkins] [3.9] bpo-43710: Rollback the 3.9 bpo-42500 fix, it broke the ABI in 3.9.3 (#25179)
ambv
webhook-mailer at python.org
Sun Apr 4 07:02:51 EDT 2021
https://github.com/python/cpython/commit/c7b0feca25fc68ec3e0884b82e5f45a4da011e8e
commit: c7b0feca25fc68ec3e0884b82e5f45a4da011e8e
branch: 3.9
author: Gregory P. Smith <greg at krypto.org>
committer: ambv <lukasz at langa.pl>
date: 2021-04-04T13:02:29+02:00
summary:
[3.9] bpo-43710: Rollback the 3.9 bpo-42500 fix, it broke the ABI in 3.9.3 (#25179)
This reverts commit 8b795ab5541d8a4e69be4137dfdc207714270b77.
It changed the PyThreadState structure size, breaking the ABI in 3.9.3.
files:
A Misc/NEWS.d/next/Core and Builtins/2021-04-03-21-12-15.bpo-43710._Mj01v.rst
M Include/cpython/pystate.h
M Include/internal/pycore_ceval.h
M Lib/test/test_exceptions.py
M Lib/test/test_sys.py
M Python/ceval.c
M Python/errors.c
M Python/pystate.c
M Python/sysmodule.c
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 29f6bf5a662dc..f292da1d3c6c5 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -58,7 +58,8 @@ struct _ts {
/* Borrowed reference to the current frame (it can be NULL) */
PyFrameObject *frame;
int recursion_depth;
- int recursion_headroom; /* Allow 50 more calls to handle any errors. */
+ char overflowed; /* The stack has overflowed. Allow 50 more calls
+ to handle the runtime error. */
char recursion_critical; /* The current calls must not cause
a stack overflow. */
int stackcheck_counter;
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index e7ace9bd01c2a..18c8f027af16e 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -90,8 +90,24 @@ static inline int _Py_EnterRecursiveCall_inline(const char *where) {
#define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where)
+/* Compute the "lower-water mark" for a recursion limit. When
+ * Py_LeaveRecursiveCall() is called with a recursion depth below this mark,
+ * the overflowed flag is reset to 0. */
+static inline int _Py_RecursionLimitLowerWaterMark(int limit) {
+ if (limit > 200) {
+ return (limit - 50);
+ }
+ else {
+ return (3 * (limit >> 2));
+ }
+}
+
static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) {
tstate->recursion_depth--;
+ int limit = tstate->interp->ceval.recursion_limit;
+ if (tstate->recursion_depth < _Py_RecursionLimitLowerWaterMark(limit)) {
+ tstate->overflowed = 0;
+ }
}
static inline void _Py_LeaveRecursiveCall_inline(void) {
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 9c09bdc317dac..8d125b57ad6d5 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -1043,7 +1043,7 @@ def gen():
# tstate->recursion_depth is equal to (recursion_limit - 1)
# and is equal to recursion_limit when _gen_throw() calls
# PyErr_NormalizeException().
- recurse(setrecursionlimit(depth + 2) - depth)
+ recurse(setrecursionlimit(depth + 2) - depth - 1)
finally:
sys.setrecursionlimit(recursionlimit)
print('Done.')
@@ -1073,54 +1073,6 @@ def test_recursion_normalizing_infinite_exception(self):
b'while normalizing an exception', err)
self.assertIn(b'Done.', out)
-
- def test_recursion_in_except_handler(self):
-
- def set_relative_recursion_limit(n):
- depth = 1
- while True:
- try:
- sys.setrecursionlimit(depth)
- except RecursionError:
- depth += 1
- else:
- break
- sys.setrecursionlimit(depth+n)
-
- def recurse_in_except():
- try:
- 1/0
- except:
- recurse_in_except()
-
- def recurse_after_except():
- try:
- 1/0
- except:
- pass
- recurse_after_except()
-
- def recurse_in_body_and_except():
- try:
- recurse_in_body_and_except()
- except:
- recurse_in_body_and_except()
-
- recursionlimit = sys.getrecursionlimit()
- try:
- set_relative_recursion_limit(10)
- for func in (recurse_in_except, recurse_after_except, recurse_in_body_and_except):
- with self.subTest(func=func):
- try:
- func()
- except RecursionError:
- pass
- else:
- self.fail("Should have raised a RecursionError")
- finally:
- sys.setrecursionlimit(recursionlimit)
-
-
@cpython_only
def test_recursion_normalizing_with_no_memory(self):
# Issue #30697. Test that in the abort that occurs when there is no
@@ -1157,7 +1109,7 @@ def raiseMemError():
except MemoryError as e:
tb = e.__traceback__
else:
- self.fail("Should have raised a MemoryError")
+ self.fail("Should have raises a MemoryError")
return traceback.format_tb(tb)
tb1 = raiseMemError()
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index fce6b98299f78..ed9b1770ab55f 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -219,7 +219,7 @@ def test_recursionlimit_recovery(self):
def f():
f()
try:
- for depth in (50, 75, 100, 250, 1000):
+ for depth in (10, 25, 50, 75, 100, 250, 1000):
try:
sys.setrecursionlimit(depth)
except RecursionError:
@@ -229,17 +229,17 @@ def f():
# Issue #5392: test stack overflow after hitting recursion
# limit twice
- with self.assertRaises(RecursionError):
- f()
- with self.assertRaises(RecursionError):
- f()
+ self.assertRaises(RecursionError, f)
+ self.assertRaises(RecursionError, f)
finally:
sys.setrecursionlimit(oldlimit)
@test.support.cpython_only
def test_setrecursionlimit_recursion_depth(self):
# Issue #25274: Setting a low recursion limit must be blocked if the
- # current recursion depth is already higher than limit.
+ # current recursion depth is already higher than the "lower-water
+ # mark". Otherwise, it may not be possible anymore to
+ # reset the overflowed flag to 0.
from _testinternalcapi import get_recursion_depth
@@ -260,10 +260,42 @@ def set_recursion_limit_at_depth(depth, limit):
sys.setrecursionlimit(1000)
for limit in (10, 25, 50, 75, 100, 150, 200):
- set_recursion_limit_at_depth(limit, limit)
+ # formula extracted from _Py_RecursionLimitLowerWaterMark()
+ if limit > 200:
+ depth = limit - 50
+ else:
+ depth = limit * 3 // 4
+ set_recursion_limit_at_depth(depth, limit)
finally:
sys.setrecursionlimit(oldlimit)
+ # The error message is specific to CPython
+ @test.support.cpython_only
+ def test_recursionlimit_fatalerror(self):
+ # A fatal error occurs if a second recursion limit is hit when recovering
+ # from a first one.
+ code = textwrap.dedent("""
+ import sys
+
+ def f():
+ try:
+ f()
+ except RecursionError:
+ f()
+
+ sys.setrecursionlimit(%d)
+ f()""")
+ with test.support.SuppressCrashReport():
+ for i in (50, 1000):
+ sub = subprocess.Popen([sys.executable, '-c', code % i],
+ stderr=subprocess.PIPE)
+ err = sub.communicate()[1]
+ self.assertTrue(sub.returncode, sub.returncode)
+ self.assertIn(
+ b"Fatal Python error: _Py_CheckRecursiveCall: "
+ b"Cannot recover from stack overflow",
+ err)
+
def test_getwindowsversion(self):
# Raise SkipTest if sys doesn't have getwindowsversion attribute
test.support.get_attribute(sys, "getwindowsversion")
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-03-21-12-15.bpo-43710._Mj01v.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-03-21-12-15.bpo-43710._Mj01v.rst
new file mode 100644
index 0000000000000..cc32efc543924
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-04-03-21-12-15.bpo-43710._Mj01v.rst
@@ -0,0 +1,4 @@
+Reverted the fix for https://bugs.python.org/issue42500 as it changed the
+PyThreadState struct size and broke the 3.9.x ABI in the 3.9.3 release
+(visible on 32-bit platforms using binaries compiled using an earlier
+version of Python 3.9.x headers).
diff --git a/Python/ceval.c b/Python/ceval.c
index b7176dc0f6b8a..91e879e804204 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -793,22 +793,23 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
_Py_CheckRecursionLimit = recursion_limit;
}
#endif
- if (tstate->recursion_headroom) {
+ if (tstate->recursion_critical)
+ /* Somebody asked that we don't check for recursion. */
+ return 0;
+ if (tstate->overflowed) {
if (tstate->recursion_depth > recursion_limit + 50) {
/* Overflowing while handling an overflow. Give up. */
Py_FatalError("Cannot recover from stack overflow.");
}
+ return 0;
}
- else {
- if (tstate->recursion_depth > recursion_limit) {
- tstate->recursion_headroom++;
- _PyErr_Format(tstate, PyExc_RecursionError,
- "maximum recursion depth exceeded%s",
- where);
- tstate->recursion_headroom--;
- --tstate->recursion_depth;
- return -1;
- }
+ if (tstate->recursion_depth > recursion_limit) {
+ --tstate->recursion_depth;
+ tstate->overflowed = 1;
+ _PyErr_Format(tstate, PyExc_RecursionError,
+ "maximum recursion depth exceeded%s",
+ where);
+ return -1;
}
return 0;
}
diff --git a/Python/errors.c b/Python/errors.c
index d8c2d8b9236ed..87af39d527a51 100644
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -290,14 +290,12 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
PyObject **val, PyObject **tb)
{
int recursion_depth = 0;
- tstate->recursion_headroom++;
PyObject *type, *value, *initial_tb;
restart:
type = *exc;
if (type == NULL) {
/* There was no exception, so nothing to do. */
- tstate->recursion_headroom--;
return;
}
@@ -349,7 +347,6 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc,
}
*exc = type;
*val = value;
- tstate->recursion_headroom--;
return;
error:
diff --git a/Python/pystate.c b/Python/pystate.c
index 71aac6f695d6e..9beefa8e20c44 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -576,7 +576,7 @@ new_threadstate(PyInterpreterState *interp, int init)
tstate->frame = NULL;
tstate->recursion_depth = 0;
- tstate->recursion_headroom = 0;
+ tstate->overflowed = 0;
tstate->recursion_critical = 0;
tstate->stackcheck_counter = 0;
tstate->tracing = 0;
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 4c7f7b67fa837..3e4115fe8e1f9 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1160,6 +1160,7 @@ static PyObject *
sys_setrecursionlimit_impl(PyObject *module, int new_limit)
/*[clinic end generated code: output=35e1c64754800ace input=b0f7a23393924af3]*/
{
+ int mark;
PyThreadState *tstate = _PyThreadState_GET();
if (new_limit < 1) {
@@ -1177,7 +1178,8 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit)
Reject too low new limit if the current recursion depth is higher than
the new low-water mark. Otherwise it may not be possible anymore to
reset the overflowed flag to 0. */
- if (tstate->recursion_depth >= new_limit) {
+ mark = _Py_RecursionLimitLowerWaterMark(new_limit);
+ if (tstate->recursion_depth >= mark) {
_PyErr_Format(tstate, PyExc_RecursionError,
"cannot set the recursion limit to %i at "
"the recursion depth %i: the limit is too low",
More information about the Python-checkins
mailing list