[pypy-commit] pypy py3.7: switch to the 3.7 grammar, which makes it possible to stop using the hacks in
cfbolz
pypy.commits at gmail.com
Mon Jan 6 13:52:02 EST 2020
Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.7
Changeset: r98458:c345fa010218
Date: 2020-01-06 19:45 +0100
http://bitbucket.org/pypy/pypy/changeset/c345fa010218/
Log: switch to the 3.7 grammar, which makes it possible to stop using the
hacks in the tokenizer
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -993,7 +993,8 @@
def handle_atom_expr(self, atom_node):
start = 0
num_ch = atom_node.num_children()
- if atom_node.get_child(0).type == tokens.AWAIT:
+ if atom_node.get_child(0).type == tokens.NAME:
+ # await
start = 1
atom_expr = self.handle_atom(atom_node.get_child(start))
if num_ch == 1:
@@ -1323,9 +1324,11 @@
current_for = comp_node
while True:
count += 1
- is_async = current_for.get_child(0).type == tokens.ASYNC
- if current_for.num_children() == 5 + is_async:
- current_iter = current_for.get_child(4 + is_async)
+ is_async = current_for.get_child(0).type == tokens.NAME
+ current_for = current_for.get_child(int(is_async))
+ assert current_for.type == syms.sync_comp_for
+ if current_for.num_children() == 5:
+ current_iter = current_for.get_child(4)
else:
return count
while True:
@@ -1353,13 +1356,16 @@
iter_node = first_child.get_child(2)
def comprehension_helper(self, comp_node):
+ assert comp_node.type == syms.comp_for
fors_count = self.count_comp_fors(comp_node)
comps = []
for i in range(fors_count):
- is_async = comp_node.get_child(0).type == tokens.ASYNC
- for_node = comp_node.get_child(1 + is_async)
+ is_async = comp_node.get_child(0).type == tokens.NAME
+ comp_node = comp_node.get_child(int(is_async))
+ assert comp_node.type == syms.sync_comp_for
+ for_node = comp_node.get_child(1)
for_targets = self.handle_exprlist(for_node, ast.Store)
- expr = self.handle_expr(comp_node.get_child(3 + is_async))
+ expr = self.handle_expr(comp_node.get_child(3))
assert isinstance(expr, ast.expr)
if for_node.num_children() == 1:
comp = ast.comprehension(for_targets[0], expr, None, is_async)
@@ -1372,8 +1378,8 @@
line = expr_node.lineno
target = ast.Tuple(for_targets, ast.Store, line, col)
comp = ast.comprehension(target, expr, None, is_async)
- if comp_node.num_children() == 5 + is_async:
- comp_node = comp_iter = comp_node.get_child(4 + is_async)
+ if comp_node.num_children() == 5:
+ comp_node = comp_iter = comp_node.get_child(4)
assert comp_iter.type == syms.comp_iter
ifs_count = self.count_comp_ifs(comp_iter)
if ifs_count:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1323,14 +1323,6 @@
# compiling the produced AST previously triggered a crash
compile(ast, '', 'exec')
- def test_await_warning(self):
- import warnings
- source = "def f(): await = 5"
- with warnings.catch_warnings(record=True) as l:
- warnings.simplefilter("always")
- compile(source, '', 'exec')
- assert isinstance(l[0].message, DeprecationWarning)
-
class TestOptimizations:
def count_instructions(self, source):
diff --git a/pypy/interpreter/pyparser/pygram.py b/pypy/interpreter/pyparser/pygram.py
--- a/pypy/interpreter/pyparser/pygram.py
+++ b/pypy/interpreter/pyparser/pygram.py
@@ -9,7 +9,7 @@
def _get_python_grammar():
here = os.path.dirname(__file__)
- fp = open(os.path.join(here, "data", "Grammar3.6"))
+ fp = open(os.path.join(here, "data", "Grammar3.7"))
try:
gram_source = fp.read()
finally:
diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py
--- a/pypy/interpreter/pyparser/pytoken.py
+++ b/pypy/interpreter/pyparser/pytoken.py
@@ -65,8 +65,8 @@
_add_tok('RARROW', "->")
_add_tok('ELLIPSIS', "...")
_add_tok('OP')
-_add_tok('ASYNC')
-_add_tok('AWAIT')
+_add_tok('ASYNC') # unused, just keywords now
+_add_tok('AWAIT') # unused, just keywords now
_add_tok('ERRORTOKEN')
# extra PyPy-specific tokens
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -111,9 +111,6 @@
altindents = [0]
last_comment = ''
parenstack = []
- async_def = False
- async_def_nl = False
- async_def_indent = 0
# make the annotator happy
endDFA = DUMMY_DFA
@@ -205,10 +202,6 @@
raise TokenIndentationError(err, line, lnum, column+1, token_list)
if altcolumn != altindents[-1]:
raise TabError(lnum, pos, line)
- if async_def_nl and async_def_indent >= indents[-1]:
- async_def = False
- async_def_nl = False
- async_def_indent = 0
else: # continued statement
if not line:
@@ -242,8 +235,6 @@
last_comment = ''
elif initial in '\r\n':
if not parenstack:
- if async_def:
- async_def_nl = True
tok = Token(tokens.NEWLINE, last_comment, lnum, start, line)
token_list.append(tok)
last_comment = ''
@@ -292,30 +283,6 @@
# be used in identifiers
raise TokenError("invalid character in identifier",
line, lnum, start + 1, token_list)
-
- if async_def: # inside 'async def' function
- if token == 'async':
- token_list.append(Token(tokens.ASYNC, token, lnum, start, line))
- elif token == 'await':
- token_list.append(Token(tokens.AWAIT, token, lnum, start, line))
- else:
- token_list.append(Token(tokens.NAME, token, lnum, start, line))
- elif token == 'async': # async token, look ahead
- #ahead token
- if pos < max:
- async_end = pseudoDFA.recognize(line, pos)
- assert async_end >= 3
- async_start = async_end - 3
- assert async_start >= 0
- ahead_token = line[async_start:async_end]
- if ahead_token == 'def':
- async_def = True
- async_def_indent = indents[-1]
- token_list.append(Token(tokens.ASYNC, token, lnum, start, line))
- else:
- token_list.append(Token(tokens.NAME, token, lnum, start, line))
- else:
- token_list.append(Token(tokens.NAME, token, lnum, start, line))
else:
token_list.append(Token(tokens.NAME, token, lnum, start, line))
last_comment = ''
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import py
+import pytest
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.pyparser.pygram import syms, tokens
from pypy.interpreter.pyparser.error import SyntaxError, IndentationError, TabError
@@ -215,21 +216,28 @@
def test_async_await(self):
self.parse("async def coro(): await func")
- py.test.raises(SyntaxError, self.parse, 'await x')
+ self.parse("await x")
#Test as var and func name
- self.parse("async = 1")
- self.parse("await = 1")
- self.parse("def async(): pass")
+ with pytest.raises(SyntaxError):
+ self.parse("async = 1")
+ with pytest.raises(SyntaxError):
+ self.parse("await = 1")
+ with pytest.raises(SyntaxError):
+ self.parse("def async(): pass")
#async for
self.parse("""async def foo():
async for a in b:
pass""")
- py.test.raises(SyntaxError, self.parse, 'def foo(): async for a in b: pass')
+ self.parse("""def foo():
+ async for a in b:
+ pass""")
#async with
self.parse("""async def foo():
async with a:
pass""")
- py.test.raises(SyntaxError, self.parse, 'def foo(): async with a: pass')
+ self.parse('''def foo():
+ async with a:
+ pass''')
def test_number_underscores(self):
VALID_UNDERSCORE_LITERALS = [
More information about the pypy-commit
mailing list