[Python-checkins] gh-98461: Fix source location in comprehensions bytecode (GH-98464)
iritkatriel
webhook-mailer at python.org
Thu Oct 20 11:58:44 EDT 2022
https://github.com/python/cpython/commit/4ec9ed8fde8e285a3eea97c199f7bbf7c95c8881
commit: 4ec9ed8fde8e285a3eea97c199f7bbf7c95c8881
branch: main
author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com>
committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com>
date: 2022-10-20T16:58:37+01:00
summary:
gh-98461: Fix source location in comprehensions bytecode (GH-98464)
files:
A Misc/NEWS.d/next/Core and Builtins/2022-10-19-20-53-38.gh-issue-98461.iNmPDV.rst
M Lib/test/test_compile.py
M Python/compile.c
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 85f05a8dd257..a59d692876b6 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -1249,6 +1249,165 @@ def test_multiline_assert(self):
self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS',
line=1, end_line=3, column=0, end_column=30, occurrence=1)
+ def test_multiline_generator_expression(self):
+ snippet = """\
+((x,
+ 2*x)
+ for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50))
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ compiled_code = compiled_code.co_consts[0]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE',
+ line=1, end_line=2, column=1, end_column=8, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=1, end_line=2, column=1, end_column=8, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=1, end_line=6, column=0, end_column=32, occurrence=1)
+
+ def test_multiline_async_generator_expression(self):
+ snippet = """\
+((x,
+ 2*x)
+ async for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50))
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ compiled_code = compiled_code.co_consts[0]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE',
+ line=1, end_line=2, column=1, end_column=8, occurrence=2)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=6, end_line=6, column=23, end_column=30, occurrence=1)
+
+ def test_multiline_list_comprehension(self):
+ snippet = """\
+[(x,
+ 2*x)
+ for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50)]
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ compiled_code = compiled_code.co_consts[0]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
+ line=1, end_line=2, column=1, end_column=8, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=1, end_line=2, column=1, end_column=8, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=1, end_line=6, column=0, end_column=32, occurrence=1)
+
+ def test_multiline_async_list_comprehension(self):
+ snippet = """\
+async def f():
+ [(x,
+ 2*x)
+ async for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50)]
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ g = {}
+ eval(compiled_code, g)
+ compiled_code = g['f'].__code__.co_consts[1]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
+ line=2, end_line=3, column=5, end_column=12, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=2, end_line=3, column=5, end_column=12, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=2, end_line=7, column=4, end_column=36, occurrence=1)
+
+ def test_multiline_set_comprehension(self):
+ snippet = """\
+{(x,
+ 2*x)
+ for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50)}
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ compiled_code = compiled_code.co_consts[0]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
+ line=1, end_line=2, column=1, end_column=8, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=1, end_line=2, column=1, end_column=8, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=1, end_line=6, column=0, end_column=32, occurrence=1)
+
+ def test_multiline_async_set_comprehension(self):
+ snippet = """\
+async def f():
+ {(x,
+ 2*x)
+ async for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50)}
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ g = {}
+ eval(compiled_code, g)
+ compiled_code = g['f'].__code__.co_consts[1]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD',
+ line=2, end_line=3, column=5, end_column=12, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=2, end_line=3, column=5, end_column=12, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=2, end_line=7, column=4, end_column=36, occurrence=1)
+
+ def test_multiline_dict_comprehension(self):
+ snippet = """\
+{x:
+ 2*x
+ for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50)}
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ compiled_code = compiled_code.co_consts[0]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
+ line=1, end_line=2, column=1, end_column=7, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=1, end_line=2, column=1, end_column=7, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=1, end_line=6, column=0, end_column=32, occurrence=1)
+
+ def test_multiline_async_dict_comprehension(self):
+ snippet = """\
+async def f():
+ {x:
+ 2*x
+ async for x
+ in [1,2,3] if (x > 0
+ and x < 100
+ and x != 50)}
+"""
+ compiled_code, _ = self.check_positions_against_ast(snippet)
+ g = {}
+ eval(compiled_code, g)
+ compiled_code = g['f'].__code__.co_consts[1]
+ self.assertIsInstance(compiled_code, types.CodeType)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD',
+ line=2, end_line=3, column=5, end_column=11, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
+ line=2, end_line=3, column=5, end_column=11, occurrence=1)
+ self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
+ line=2, end_line=7, column=4, end_column=36, occurrence=1)
+
def test_very_long_line_end_offset(self):
# Make sure we get the correct column offset for offsets
# too large to store in a byte.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-19-20-53-38.gh-issue-98461.iNmPDV.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-20-53-38.gh-issue-98461.iNmPDV.rst
new file mode 100644
index 000000000000..6289f208609e
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-19-20-53-38.gh-issue-98461.iNmPDV.rst
@@ -0,0 +1 @@
+Fix source location in bytecode for list, set and dict comprehensions as well as generator expressions.
diff --git a/Python/compile.c b/Python/compile.c
index c888f4064c79..70f38d4d0cda 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -482,13 +482,13 @@ static int compiler_try_star_except(struct compiler *, stmt_ty);
static int compiler_set_qualname(struct compiler *);
static int compiler_sync_comprehension_generator(
- struct compiler *c, location *ploc,
+ struct compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type);
static int compiler_async_comprehension_generator(
- struct compiler *c, location *ploc,
+ struct compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type);
@@ -5190,7 +5190,7 @@ compiler_call_helper(struct compiler *c, location loc,
static int
-compiler_comprehension_generator(struct compiler *c, location *ploc,
+compiler_comprehension_generator(struct compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type)
@@ -5199,35 +5199,33 @@ compiler_comprehension_generator(struct compiler *c, location *ploc,
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
if (gen->is_async) {
return compiler_async_comprehension_generator(
- c, ploc, generators, gen_index, depth, elt, val, type);
+ c, loc, generators, gen_index, depth, elt, val, type);
} else {
return compiler_sync_comprehension_generator(
- c, ploc, generators, gen_index, depth, elt, val, type);
+ c, loc, generators, gen_index, depth, elt, val, type);
}
}
static int
-compiler_sync_comprehension_generator(struct compiler *c, location *ploc,
- asdl_comprehension_seq *generators, int gen_index,
- int depth,
+compiler_sync_comprehension_generator(struct compiler *c, location loc,
+ asdl_comprehension_seq *generators,
+ int gen_index, int depth,
expr_ty elt, expr_ty val, int type)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
- comprehension_ty gen;
- Py_ssize_t i, n;
-
NEW_JUMP_TARGET_LABEL(c, start);
NEW_JUMP_TARGET_LABEL(c, if_cleanup);
NEW_JUMP_TARGET_LABEL(c, anchor);
- gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
+ comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
+ gen_index);
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_argcount = 1;
- ADDOP_I(c, *ploc, LOAD_FAST, 0);
+ ADDOP_I(c, loc, LOAD_FAST, 0);
}
else {
/* Sub-iter - calculate on the fly */
@@ -5254,29 +5252,34 @@ compiler_sync_comprehension_generator(struct compiler *c, location *ploc,
}
if (IS_LABEL(start)) {
VISIT(c, expr, gen->iter);
- ADDOP(c, *ploc, GET_ITER);
+ ADDOP(c, loc, GET_ITER);
}
}
if (IS_LABEL(start)) {
depth++;
USE_LABEL(c, start);
- ADDOP_JUMP(c, *ploc, FOR_ITER, anchor);
+ ADDOP_JUMP(c, loc, FOR_ITER, anchor);
}
VISIT(c, expr, gen->target);
/* XXX this needs to be cleaned up...a lot! */
- n = asdl_seq_LEN(gen->ifs);
- for (i = 0; i < n; i++) {
+ Py_ssize_t n = asdl_seq_LEN(gen->ifs);
+ for (Py_ssize_t i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
- if (!compiler_jump_if(c, ploc, e, if_cleanup, 0))
+ if (!compiler_jump_if(c, &loc, e, if_cleanup, 0)) {
return 0;
+ }
}
- if (++gen_index < asdl_seq_LEN(generators))
- if (!compiler_comprehension_generator(c, ploc,
+ if (++gen_index < asdl_seq_LEN(generators)) {
+ if (!compiler_comprehension_generator(c, loc,
generators, gen_index, depth,
- elt, val, type))
- return 0;
+ elt, val, type)) {
+ return 0;
+ }
+ }
+
+ location elt_loc = LOC(elt);
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
@@ -5284,23 +5287,27 @@ compiler_sync_comprehension_generator(struct compiler *c, location *ploc,
switch (type) {
case COMP_GENEXP:
VISIT(c, expr, elt);
- ADDOP_YIELD(c, *ploc);
- ADDOP(c, *ploc, POP_TOP);
+ ADDOP_YIELD(c, elt_loc);
+ ADDOP(c, elt_loc, POP_TOP);
break;
case COMP_LISTCOMP:
VISIT(c, expr, elt);
- ADDOP_I(c, *ploc, LIST_APPEND, depth + 1);
+ ADDOP_I(c, elt_loc, LIST_APPEND, depth + 1);
break;
case COMP_SETCOMP:
VISIT(c, expr, elt);
- ADDOP_I(c, *ploc, SET_ADD, depth + 1);
+ ADDOP_I(c, elt_loc, SET_ADD, depth + 1);
break;
case COMP_DICTCOMP:
/* With '{k: v}', k is evaluated before v, so we do
the same. */
VISIT(c, expr, elt);
VISIT(c, expr, val);
- ADDOP_I(c, *ploc, MAP_ADD, depth + 1);
+ elt_loc = LOCATION(elt->lineno,
+ val->end_lineno,
+ elt->col_offset,
+ val->end_col_offset);
+ ADDOP_I(c, elt_loc, MAP_ADD, depth + 1);
break;
default:
return 0;
@@ -5309,7 +5316,7 @@ compiler_sync_comprehension_generator(struct compiler *c, location *ploc,
USE_LABEL(c, if_cleanup);
if (IS_LABEL(start)) {
- ADDOP_JUMP(c, *ploc, JUMP, start);
+ ADDOP_JUMP(c, elt_loc, JUMP, start);
USE_LABEL(c, anchor);
}
@@ -5318,81 +5325,88 @@ compiler_sync_comprehension_generator(struct compiler *c, location *ploc,
}
static int
-compiler_async_comprehension_generator(struct compiler *c, location *ploc,
- asdl_comprehension_seq *generators, int gen_index,
- int depth,
+compiler_async_comprehension_generator(struct compiler *c, location loc,
+ asdl_comprehension_seq *generators,
+ int gen_index, int depth,
expr_ty elt, expr_ty val, int type)
{
- comprehension_ty gen;
- Py_ssize_t i, n;
NEW_JUMP_TARGET_LABEL(c, start);
NEW_JUMP_TARGET_LABEL(c, except);
NEW_JUMP_TARGET_LABEL(c, if_cleanup);
- gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
+ comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
+ gen_index);
if (gen_index == 0) {
/* Receive outermost iter as an implicit argument */
c->u->u_argcount = 1;
- ADDOP_I(c, *ploc, LOAD_FAST, 0);
+ ADDOP_I(c, loc, LOAD_FAST, 0);
}
else {
/* Sub-iter - calculate on the fly */
VISIT(c, expr, gen->iter);
- ADDOP(c, *ploc, GET_AITER);
+ ADDOP(c, loc, GET_AITER);
}
USE_LABEL(c, start);
/* Runtime will push a block here, so we need to account for that */
- if (!compiler_push_fblock(c, *ploc, ASYNC_COMPREHENSION_GENERATOR,
+ if (!compiler_push_fblock(c, loc, ASYNC_COMPREHENSION_GENERATOR,
start, NO_LABEL, NULL)) {
return 0;
}
- ADDOP_JUMP(c, *ploc, SETUP_FINALLY, except);
- ADDOP(c, *ploc, GET_ANEXT);
- ADDOP_LOAD_CONST(c, *ploc, Py_None);
- ADD_YIELD_FROM(c, *ploc, 1);
- ADDOP(c, *ploc, POP_BLOCK);
+ ADDOP_JUMP(c, loc, SETUP_FINALLY, except);
+ ADDOP(c, loc, GET_ANEXT);
+ ADDOP_LOAD_CONST(c, loc, Py_None);
+ ADD_YIELD_FROM(c, loc, 1);
+ ADDOP(c, loc, POP_BLOCK);
VISIT(c, expr, gen->target);
- n = asdl_seq_LEN(gen->ifs);
- for (i = 0; i < n; i++) {
+ Py_ssize_t n = asdl_seq_LEN(gen->ifs);
+ for (Py_ssize_t i = 0; i < n; i++) {
expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
- if (!compiler_jump_if(c, ploc, e, if_cleanup, 0))
+ if (!compiler_jump_if(c, &loc, e, if_cleanup, 0)) {
return 0;
+ }
}
depth++;
- if (++gen_index < asdl_seq_LEN(generators))
- if (!compiler_comprehension_generator(c, ploc,
+ if (++gen_index < asdl_seq_LEN(generators)) {
+ if (!compiler_comprehension_generator(c, loc,
generators, gen_index, depth,
- elt, val, type))
- return 0;
+ elt, val, type)) {
+ return 0;
+ }
+ }
+ location elt_loc = LOC(elt);
/* only append after the last for generator */
if (gen_index >= asdl_seq_LEN(generators)) {
/* comprehension specific code */
switch (type) {
case COMP_GENEXP:
VISIT(c, expr, elt);
- ADDOP_YIELD(c, *ploc);
- ADDOP(c, *ploc, POP_TOP);
+ ADDOP_YIELD(c, elt_loc);
+ ADDOP(c, elt_loc, POP_TOP);
break;
case COMP_LISTCOMP:
VISIT(c, expr, elt);
- ADDOP_I(c, *ploc, LIST_APPEND, depth + 1);
+ ADDOP_I(c, elt_loc, LIST_APPEND, depth + 1);
break;
case COMP_SETCOMP:
VISIT(c, expr, elt);
- ADDOP_I(c, *ploc, SET_ADD, depth + 1);
+ ADDOP_I(c, elt_loc, SET_ADD, depth + 1);
break;
case COMP_DICTCOMP:
/* With '{k: v}', k is evaluated before v, so we do
the same. */
VISIT(c, expr, elt);
VISIT(c, expr, val);
- ADDOP_I(c, *ploc, MAP_ADD, depth + 1);
+ elt_loc = LOCATION(elt->lineno,
+ val->end_lineno,
+ elt->col_offset,
+ val->end_col_offset);
+ ADDOP_I(c, elt_loc, MAP_ADD, depth + 1);
break;
default:
return 0;
@@ -5400,14 +5414,13 @@ compiler_async_comprehension_generator(struct compiler *c, location *ploc,
}
USE_LABEL(c, if_cleanup);
- ADDOP_JUMP(c, *ploc, JUMP, start);
+ ADDOP_JUMP(c, elt_loc, JUMP, start);
compiler_pop_fblock(c, ASYNC_COMPREHENSION_GENERATOR, start);
USE_LABEL(c, except);
- //UNSET_LOC(c);
- ADDOP(c, *ploc, END_ASYNC_FOR);
+ ADDOP(c, loc, END_ASYNC_FOR);
return 1;
}
@@ -5466,12 +5479,13 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
ADDOP_I(c, loc, op, 0);
}
- if (!compiler_comprehension_generator(c, &loc, generators, 0, 0,
- elt, val, type))
+ if (!compiler_comprehension_generator(c, loc, generators, 0, 0,
+ elt, val, type)) {
goto error_in_scope;
+ }
if (type != COMP_GENEXP) {
- ADDOP(c, loc, RETURN_VALUE);
+ ADDOP(c, LOC(e), RETURN_VALUE);
}
co = assemble(c, 1);
More information about the Python-checkins
mailing list