[Python-checkins] cpython (merge 3.3 -> default): Issue #5765: Merge from 3.3
nick.coghlan
python-checkins at python.org
Sun Nov 4 14:54:24 CET 2012
http://hg.python.org/cpython/rev/bd1db93d76e1
changeset: 80240:bd1db93d76e1
parent: 80238:f02555353544
parent: 80239:ab02cd145f56
user: Nick Coghlan <ncoghlan at gmail.com>
date: Sun Nov 04 23:53:15 2012 +1000
summary:
Issue #5765: Merge from 3.3
files:
Include/symtable.h | 2 +
Lib/test/crashers/compiler_recursion.py | 13 -
Lib/test/test_compile.py | 27 +++
Misc/ACKS | 1 +
Misc/NEWS | 3 +
Python/compile.c | 5 +
Python/symtable.c | 101 ++++++++---
7 files changed, 108 insertions(+), 44 deletions(-)
diff --git a/Include/symtable.h b/Include/symtable.h
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -30,6 +30,8 @@
PyObject *st_private; /* name of current class or NULL */
PyFutureFeatures *st_future; /* module's future features that affect
the symbol table */
+ int recursion_depth; /* current recursion depth */
+ int recursion_limit; /* recursion limit */
};
typedef struct _symtable_entry {
diff --git a/Lib/test/crashers/compiler_recursion.py b/Lib/test/crashers/compiler_recursion.py
deleted file mode 100644
--- a/Lib/test/crashers/compiler_recursion.py
+++ /dev/null
@@ -1,13 +0,0 @@
-"""
-The compiler (>= 2.5) recurses happily until it blows the stack.
-
-Recorded on the tracker as http://bugs.python.org/issue11383
-"""
-
-# The variant below blows up in compiler_call, but there are assorted
-# other variations that blow up in other functions
-# e.g. '1*'*10**5+'1' will die in compiler_visit_expr
-
-# The exact limit to destroy the stack will vary by platform
-# but 10M should do the trick even with huge stack allocations
-compile('()'*10**7, '?', 'exec')
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -474,6 +474,33 @@
self.assertInvalidSingle('f()\nxy # blah\nblah()')
self.assertInvalidSingle('x = 5 # comment\nx = 6\n')
+ @support.cpython_only
+ def test_compiler_recursion_limit(self):
+ # Expected limit is sys.getrecursionlimit() * the scaling factor
+ # in symtable.c (currently 3)
+ # We expect to fail *at* that limit, because we use up some of
+ # the stack depth limit in the test suite code
+ # So we check the expected limit and 75% of that
+ # XXX (ncoghlan): duplicating the scaling factor here is a little
+ # ugly. Perhaps it should be exposed somewhere...
+ fail_depth = sys.getrecursionlimit() * 3
+ success_depth = int(fail_depth * 0.75)
+
+ def check_limit(prefix, repeated):
+ expect_ok = prefix + repeated * success_depth
+ self.compile_single(expect_ok)
+ broken = prefix + repeated * fail_depth
+ details = "Compiling ({!r} + {!r} * {})".format(
+ prefix, repeated, fail_depth)
+ with self.assertRaises(RuntimeError, msg=details):
+ self.compile_single(broken)
+
+ check_limit("a", "()")
+ check_limit("a", ".b")
+ check_limit("a", "[0]")
+ check_limit("a", "*a")
+
+
def test_main():
support.run_unittest(TestSpecifics)
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -431,6 +431,7 @@
Nathaniel Gray
Eddy De Greef
Grant Griffin
+Andrea Griffini
Duncan Grisby
Fabian Groffen
Eric Groo
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
Core and Builtins
-----------------
+- Issue #5765: Apply a hard recursion limit in the compiler instead of
+ blowing the stack and segfaulting.
+
- Issue #16402: When slicing a range, fix shadowing of exceptions from
__index__.
diff --git a/Python/compile.c b/Python/compile.c
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -141,6 +141,11 @@
The u pointer points to the current compilation unit, while units
for enclosing blocks are stored in c_stack. The u and c_stack are
managed by compiler_enter_scope() and compiler_exit_scope().
+
+Note that we don't track recursion levels during compilation - the
+task of detecting and rejecting excessive levels of nesting is
+handled by the symbol analysis pass.
+
*/
struct compiler {
diff --git a/Python/symtable.c b/Python/symtable.c
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -223,17 +223,40 @@
return NULL;
}
+/* When compiling the use of C stack is probably going to be a lot
+ lighter than when executing Python code but still can overflow
+ and causing a Python crash if not checked (e.g. eval("()"*300000)).
+ Using the current recursion limit for the compiler seems too
+ restrictive (it caused at least one test to fail) so a factor is
+ used to allow deeper recursion when compiling an expression.
+
+ Using a scaling factor means this should automatically adjust when
+ the recursion limit is adjusted for small or large C stack allocations.
+*/
+#define COMPILER_STACK_FRAME_SCALE 3
+
struct symtable *
PySymtable_Build(mod_ty mod, const char *filename, PyFutureFeatures *future)
{
struct symtable *st = symtable_new();
asdl_seq *seq;
int i;
+ PyThreadState *tstate;
if (st == NULL)
return st;
st->st_filename = filename;
st->st_future = future;
+
+ /* Setup recursion depth check counters */
+ tstate = PyThreadState_GET();
+ if (!tstate) {
+ PySymtable_Free(st);
+ return NULL;
+ }
+ st->recursion_depth = tstate->recursion_depth * COMPILER_STACK_FRAME_SCALE;
+ st->recursion_limit = Py_GetRecursionLimit() * COMPILER_STACK_FRAME_SCALE;
+
/* Make the initial symbol information gathering pass */
if (!GET_IDENTIFIER(top) ||
!symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0)) {
@@ -1031,11 +1054,17 @@
VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
useful if the first node in the sequence requires special treatment.
+
+ VISIT_QUIT macro returns the specified value exiting from the function but
+ first adjusts current recursion counter depth.
*/
+#define VISIT_QUIT(ST, X) \
+ return --(ST)->recursion_depth,(X)
+
#define VISIT(ST, TYPE, V) \
if (!symtable_visit_ ## TYPE((ST), (V))) \
- return 0;
+ VISIT_QUIT((ST), 0);
#define VISIT_SEQ(ST, TYPE, SEQ) { \
int i; \
@@ -1043,7 +1072,7 @@
for (i = 0; i < asdl_seq_LEN(seq); i++) { \
TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
if (!symtable_visit_ ## TYPE((ST), elt)) \
- return 0; \
+ VISIT_QUIT((ST), 0); \
} \
}
@@ -1053,7 +1082,7 @@
for (i = (START); i < asdl_seq_LEN(seq); i++) { \
TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
if (!symtable_visit_ ## TYPE((ST), elt)) \
- return 0; \
+ VISIT_QUIT((ST), 0); \
} \
}
@@ -1064,7 +1093,7 @@
expr_ty elt = (expr_ty)asdl_seq_GET(seq, i); \
if (!elt) continue; /* can be NULL */ \
if (!symtable_visit_expr((ST), elt)) \
- return 0; \
+ VISIT_QUIT((ST), 0); \
} \
}
@@ -1108,32 +1137,37 @@
static int
symtable_visit_stmt(struct symtable *st, stmt_ty s)
{
+ if (++st->recursion_depth > st->recursion_limit) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "maximum recursion depth exceeded during compilation");
+ VISIT_QUIT(st, 0);
+ }
switch (s->kind) {
case FunctionDef_kind:
if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL))
- return 0;
+ VISIT_QUIT(st, 0);
if (s->v.FunctionDef.args->defaults)
VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
if (s->v.FunctionDef.args->kw_defaults)
VISIT_KWONLYDEFAULTS(st,
s->v.FunctionDef.args->kw_defaults);
if (!symtable_visit_annotations(st, s))
- return 0;
+ VISIT_QUIT(st, 0);
if (s->v.FunctionDef.decorator_list)
VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
if (!symtable_enter_block(st, s->v.FunctionDef.name,
FunctionBlock, (void *)s, s->lineno,
s->col_offset))
- return 0;
+ VISIT_QUIT(st, 0);
VISIT(st, arguments, s->v.FunctionDef.args);
VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
if (!symtable_exit_block(st, s))
- return 0;
+ VISIT_QUIT(st, 0);
break;
case ClassDef_kind: {
PyObject *tmp;
if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL))
- return 0;
+ VISIT_QUIT(st, 0);
VISIT_SEQ(st, expr, s->v.ClassDef.bases);
VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
if (s->v.ClassDef.starargs)
@@ -1144,20 +1178,20 @@
VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
(void *)s, s->lineno, s->col_offset))
- return 0;
+ VISIT_QUIT(st, 0);
if (!GET_IDENTIFIER(__class__) ||
!symtable_add_def(st, __class__, DEF_LOCAL) ||
!GET_IDENTIFIER(__locals__) ||
!symtable_add_def(st, __locals__, DEF_PARAM)) {
symtable_exit_block(st, s);
- return 0;
+ VISIT_QUIT(st, 0);
}
tmp = st->st_private;
st->st_private = s->v.ClassDef.name;
VISIT_SEQ(st, stmt, s->v.ClassDef.body);
st->st_private = tmp;
if (!symtable_exit_block(st, s))
- return 0;
+ VISIT_QUIT(st, 0);
break;
}
case Return_kind:
@@ -1241,7 +1275,7 @@
identifier name = (identifier)asdl_seq_GET(seq, i);
long cur = symtable_lookup(st, name);
if (cur < 0)
- return 0;
+ VISIT_QUIT(st, 0);
if (cur & (DEF_LOCAL | USE)) {
char buf[256];
char *c_name = _PyUnicode_AsString(name);
@@ -1256,12 +1290,12 @@
GLOBAL_AFTER_USE,
c_name);
if (!symtable_warn(st, buf, s->lineno))
- return 0;
+ VISIT_QUIT(st, 0);
}
if (!symtable_add_def(st, name, DEF_GLOBAL))
- return 0;
+ VISIT_QUIT(st, 0);
if (!symtable_record_directive(st, name, s))
- return 0;
+ VISIT_QUIT(st, 0);
}
break;
}
@@ -1272,7 +1306,7 @@
identifier name = (identifier)asdl_seq_GET(seq, i);
long cur = symtable_lookup(st, name);
if (cur < 0)
- return 0;
+ VISIT_QUIT(st, 0);
if (cur & (DEF_LOCAL | USE)) {
char buf[256];
char *c_name = _PyUnicode_AsString(name);
@@ -1287,12 +1321,12 @@
NONLOCAL_AFTER_USE,
c_name);
if (!symtable_warn(st, buf, s->lineno))
- return 0;
+ VISIT_QUIT(st, 0);
}
if (!symtable_add_def(st, name, DEF_NONLOCAL))
- return 0;
+ VISIT_QUIT(st, 0);
if (!symtable_record_directive(st, name, s))
- return 0;
+ VISIT_QUIT(st, 0);
}
break;
}
@@ -1309,12 +1343,17 @@
VISIT_SEQ(st, stmt, s->v.With.body);
break;
}
- return 1;
+ VISIT_QUIT(st, 1);
}
static int
symtable_visit_expr(struct symtable *st, expr_ty e)
{
+ if (++st->recursion_depth > st->recursion_limit) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "maximum recursion depth exceeded during compilation");
+ VISIT_QUIT(st, 0);
+ }
switch (e->kind) {
case BoolOp_kind:
VISIT_SEQ(st, expr, e->v.BoolOp.values);
@@ -1328,7 +1367,7 @@
break;
case Lambda_kind: {
if (!GET_IDENTIFIER(lambda))
- return 0;
+ VISIT_QUIT(st, 0);
if (e->v.Lambda.args->defaults)
VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
if (e->v.Lambda.args->kw_defaults)
@@ -1337,11 +1376,11 @@
if (!symtable_enter_block(st, lambda,
FunctionBlock, (void *)e, e->lineno,
e->col_offset))
- return 0;
+ VISIT_QUIT(st, 0);
VISIT(st, arguments, e->v.Lambda.args);
VISIT(st, expr, e->v.Lambda.body);
if (!symtable_exit_block(st, (void *)e))
- return 0;
+ VISIT_QUIT(st, 0);
break;
}
case IfExp_kind:
@@ -1358,19 +1397,19 @@
break;
case GeneratorExp_kind:
if (!symtable_visit_genexp(st, e))
- return 0;
+ VISIT_QUIT(st, 0);
break;
case ListComp_kind:
if (!symtable_visit_listcomp(st, e))
- return 0;
+ VISIT_QUIT(st, 0);
break;
case SetComp_kind:
if (!symtable_visit_setcomp(st, e))
- return 0;
+ VISIT_QUIT(st, 0);
break;
case DictComp_kind:
if (!symtable_visit_dictcomp(st, e))
- return 0;
+ VISIT_QUIT(st, 0);
break;
case Yield_kind:
case YieldFrom_kind: {
@@ -1414,14 +1453,14 @@
case Name_kind:
if (!symtable_add_def(st, e->v.Name.id,
e->v.Name.ctx == Load ? USE : DEF_LOCAL))
- return 0;
+ VISIT_QUIT(st, 0);
/* Special-case super: it counts as a use of __class__ */
if (e->v.Name.ctx == Load &&
st->st_cur->ste_type == FunctionBlock &&
!PyUnicode_CompareWithASCIIString(e->v.Name.id, "super")) {
if (!GET_IDENTIFIER(__class__) ||
!symtable_add_def(st, __class__, USE))
- return 0;
+ VISIT_QUIT(st, 0);
}
break;
/* child nodes of List and Tuple will have expr_context set */
@@ -1432,7 +1471,7 @@
VISIT_SEQ(st, expr, e->v.Tuple.elts);
break;
}
- return 1;
+ VISIT_QUIT(st, 1);
}
static int
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list