[pypy-commit] pypy default: Issue #2526: fix for a corner case of __future__ imports
arigo
pypy.commits at gmail.com
Sat Apr 1 04:01:18 EDT 2017
Author: Armin Rigo <arigo at tunes.org>
Branch:
Changeset: r90886:49dcf5e4c5a1
Date: 2017-04-01 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/49dcf5e4c5a1/
Log: Issue #2526: fix for a corner case of __future__ imports
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -78,6 +78,7 @@
from pypy.interpreter.pyparser import pygram
it = TokenIterator(tokens)
result = 0
+ last_position = (0, 0)
#
# The only things that can precede a future statement are another
# future statement and a doc string (only one). This is a very
@@ -92,6 +93,11 @@
it.skip_name("__future__") and
it.skip_name("import")):
it.skip(pygram.tokens.LPAR) # optionally
+ # return in 'last_position' any line-column pair that points
+ # somewhere inside the last __future__ import statement
+ # (at the start would be fine too, but it's easier to grab a
+ # random position inside)
+ last_position = (it.tok[2], it.tok[3])
result |= future_flags.get_compiler_feature(it.next_feature_name())
while it.skip(pygram.tokens.COMMA):
result |= future_flags.get_compiler_feature(it.next_feature_name())
@@ -99,5 +105,4 @@
it.skip(pygram.tokens.SEMI) # optionally
it.skip_newlines()
- position = (it.tok[2], it.tok[3])
- return result, position
+ return result, last_position
diff --git a/pypy/interpreter/pyparser/test/test_future.py b/pypy/interpreter/pyparser/test/test_future.py
--- a/pypy/interpreter/pyparser/test/test_future.py
+++ b/pypy/interpreter/pyparser/test/test_future.py
@@ -2,10 +2,9 @@
from pypy.interpreter.pyparser import future, pytokenizer
from pypy.tool import stdlib___future__ as fut
-def run(s, expected_last_future=None):
+def run(s, expected_last_future=(0, 0)):
source_lines = s.splitlines(True)
tokens = pytokenizer.generate_tokens(source_lines, 0)
- expected_last_future = expected_last_future or tokens[-1][2:4]
#
flags, last_future_import = future.add_future_flags(
future.futureFlags_2_7, tokens)
@@ -14,7 +13,7 @@
def test_docstring():
s = '"Docstring\\" "\nfrom __future__ import division\n'
- f = run(s)
+ f = run(s, (2, 24))
assert f == fut.CO_FUTURE_DIVISION
def test_comment():
@@ -45,167 +44,167 @@
def test_from():
s = 'from __future__ import division\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == fut.CO_FUTURE_DIVISION
def test_froms():
s = 'from __future__ import division, generators, with_statement\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_from_as():
s = 'from __future__ import division as b\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == fut.CO_FUTURE_DIVISION
def test_froms_as():
s = 'from __future__ import division as b, generators as c\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED)
def test_from_paren():
s = 'from __future__ import (division)\n'
- f = run(s)
+ f = run(s, (1, 25))
assert f == fut.CO_FUTURE_DIVISION
def test_froms_paren():
s = 'from __future__ import (division, generators)\n'
- f = run(s)
+ f = run(s, (1, 25))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED)
def test_froms_paren_as():
s = 'from __future__ import (division as b, generators,)\n'
- f = run(s)
+ f = run(s, (1, 25))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED)
def test_paren_with_newline():
s = 'from __future__ import (division,\nabsolute_import)\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
def test_paren_with_newline_2():
s = 'from __future__ import (\ndivision,\nabsolute_import)\n'
- f = run(s)
+ f = run(s, (2, 0))
assert f == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
def test_multiline():
s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n'
- f = run(s)
+ f = run(s, (4, 23))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_windows_style_lineendings():
s = '"abc" #def\r\n #ghi\r\nfrom __future__ import (division as b, generators,)\r\nfrom __future__ import with_statement\r\n'
- f = run(s)
+ f = run(s, (4, 23))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_mac_style_lineendings():
s = '"abc" #def\r #ghi\rfrom __future__ import (division as b, generators,)\rfrom __future__ import with_statement\r'
- f = run(s)
+ f = run(s, (4, 23))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_semicolon():
s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n'
- f = run(s)
+ f = run(s, (3, 78))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_semicolon_2():
s = 'from __future__ import division; from foo import bar'
- f = run(s, expected_last_future=(1, 39))
+ f = run(s, expected_last_future=(1, 24))
assert f == fut.CO_FUTURE_DIVISION
def test_full_chain():
s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n'
- f = run(s)
+ f = run(s, (3, 78))
assert f == (fut.CO_FUTURE_DIVISION |
fut.CO_GENERATOR_ALLOWED |
fut.CO_FUTURE_WITH_STATEMENT)
def test_intervening_code():
s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n'
- f = run(s, expected_last_future=(2, 5))
+ f = run(s, expected_last_future=(1, 25))
assert f == (fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED)
def test_nonexisting():
s = 'from __future__ import non_existing_feature\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == 0
def test_nonexisting_2():
s = 'from __future__ import non_existing_feature, with_statement\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_from_import_abs_import():
s = 'from __future__ import absolute_import\n'
- f = run(s)
+ f = run(s, (1, 24))
assert f == fut.CO_FUTURE_ABSOLUTE_IMPORT
def test_raw_doc():
s = 'r"Doc"\nfrom __future__ import with_statement\n'
- f = run(s)
+ f = run(s, (2, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_unicode_doc():
s = 'u"Doc"\nfrom __future__ import with_statement\n'
- f = run(s)
+ f = run(s, (2, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_raw_unicode_doc():
s = 'ru"Doc"\nfrom __future__ import with_statement\n'
- f = run(s)
+ f = run(s, (2, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_continuation_line():
s = "\\\nfrom __future__ import with_statement\n"
- f = run(s)
+ f = run(s, (2, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_continuation_lines():
s = "\\\n \t\\\nfrom __future__ import with_statement\n"
- f = run(s)
+ f = run(s, (3, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_lots_of_continuation_lines():
s = "\\\n\\\n\\\n\\\n\\\n\\\n\nfrom __future__ import with_statement\n"
- f = run(s)
+ f = run(s, (8, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT
def test_continuation_lines_raise():
s = " \\\n \t\\\nfrom __future__ import with_statement\n"
- f = run(s, expected_last_future=(1, 0))
+ f = run(s)
assert f == 0 # because of the INDENT
def test_continuation_lines_in_docstring_single_quoted():
s = '"\\\n\\\n\\\n\\\n\\\n\\\n"\nfrom __future__ import division\n'
- f = run(s)
+ f = run(s, (8, 24))
assert f == fut.CO_FUTURE_DIVISION
def test_continuation_lines_in_docstring_triple_quoted():
s = '"""\\\n\\\n\\\n\\\n\\\n\\\n"""\nfrom __future__ import division\n'
- f = run(s)
+ f = run(s, (8, 24))
assert f == fut.CO_FUTURE_DIVISION
def test_blank_lines():
s = ('\n\t\n\nfrom __future__ import with_statement'
' \n \n \nfrom __future__ import division')
- f = run(s)
+ f = run(s, (7, 23))
assert f == fut.CO_FUTURE_WITH_STATEMENT | fut.CO_FUTURE_DIVISION
def test_dummy_semicolons():
s = ('from __future__ import division;\n'
'from __future__ import with_statement;')
- f = run(s)
+ f = run(s, (2, 23))
assert f == fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_WITH_STATEMENT
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -366,6 +366,15 @@
assert isinstance(ns["b"], str)
assert isinstance(ns["c"], str)
+ def test_both_futures_with_semicolon(self):
+ # Issue #2526: a corner case which crashes only if the file
+ # contains *nothing more* than two __future__ imports separated
+ # by a semicolon.
+ s = """
+from __future__ import unicode_literals; from __future__ import print_function
+"""
+ exec s in {}
+
class AppTestComprehensions:
More information about the pypy-commit
mailing list