[pypy-svn] r78892 - in pypy/branch/fast-forward/pypy/interpreter: astcompiler/test pyparser
afa at codespeak.net
afa at codespeak.net
Mon Nov 8 21:34:01 CET 2010
Author: afa
Date: Mon Nov 8 21:33:57 2010
New Revision: 78892
Modified:
pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py
pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py
pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py
Log:
"0x", "0b" are not valid tokens and should raise SyntaxError
Modified: pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py (original)
+++ pypy/branch/fast-forward/pypy/interpreter/astcompiler/test/test_astbuilder.py Mon Nov 8 21:33:57 2010
@@ -1147,6 +1147,10 @@
for num in ("0b00101", "0B00101", "0b101", "0B101"):
assert space.eq_w(get_num(num), space.wrap(5))
+ py.test.raises(SyntaxError, self.get_ast, "0x")
+ py.test.raises(SyntaxError, self.get_ast, "0b")
+ py.test.raises(SyntaxError, self.get_ast, "0o")
+
def check_comprehension(self, brackets, ast_type):
def brack(s):
return brackets % s
Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py (original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/genpytokenize.py Mon Nov 8 21:33:57 2010
@@ -59,17 +59,21 @@
hexNumber = chain(states,
newArcPair(states, "0"),
groupStr(states, "xX"),
- any(states, groupStr(states, "0123456789abcdefABCDEF")),
+ atleastonce(states,
+ groupStr(states, "0123456789abcdefABCDEF")),
maybe(states, groupStr(states, "lL")))
octNumber = chain(states,
newArcPair(states, "0"),
- maybe(states, groupStr(states, "oO")),
+ maybe(states,
+ chain(states,
+ groupStr(states, "oO"),
+ groupStr(states, "01234567"))),
any(states, groupStr(states, "01234567")),
maybe(states, groupStr(states, "lL")))
binNumber = chain(states,
newArcPair(states, "0"),
groupStr(states, "bB"),
- any(states, groupStr(states, "01")),
+ atleastonce(states, groupStr(states, "01")),
maybe(states, groupStr(states, "lL")))
decNumber = chain(states,
groupStr(states, "123456789"),
Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py
==============================================================================
--- pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py (original)
+++ pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenize.py Mon Nov 8 21:33:57 2010
@@ -25,10 +25,10 @@
accepts = [True, True, True, True, True, True, True, True,
True, True, False, True, True, True, True, False,
- False, False, True, True, True, True, True, False,
- True, False, True, False, True, False, False,
- True, False, False, False, False, True, False,
- False, False, True]
+ False, False, True, False, False, True, False,
+ False, True, False, True, False, True, False,
+ False, True, False, False, True, True, True,
+ False, False, True, False, False, False, True]
states = [
# 0
{'\t': 0, '\n': 13, '\x0c': 0,
@@ -153,16 +153,15 @@
# 18
{automata.DEFAULT: 18, '\n': 27, '\r': 27},
# 19
- {'0': 19, '1': 19, '2': 19, '3': 19,
- '4': 19, '5': 19, '6': 19, '7': 19,
- '8': 19, '9': 19, 'A': 19, 'B': 19,
- 'C': 19, 'D': 19, 'E': 19, 'F': 19,
- 'L': 13, 'a': 19, 'b': 19, 'c': 19,
- 'd': 19, 'e': 19, 'f': 19, 'l': 13},
+ {'0': 34, '1': 34, '2': 34, '3': 34,
+ '4': 34, '5': 34, '6': 34, '7': 34,
+ '8': 34, '9': 34, 'A': 34, 'B': 34,
+ 'C': 34, 'D': 34, 'E': 34, 'F': 34,
+ 'a': 34, 'b': 34, 'c': 34, 'd': 34,
+ 'e': 34, 'f': 34},
# 20
- {'0': 20, '1': 20, '2': 20, '3': 20,
- '4': 20, '5': 20, '6': 20, '7': 20,
- 'L': 13, 'l': 13},
+ {'0': 35, '1': 35, '2': 35, '3': 35,
+ '4': 35, '5': 35, '6': 35, '7': 35},
# 21
{'.': 24, '0': 21, '1': 21, '2': 21,
'3': 21, '4': 21, '5': 21, '6': 21,
@@ -170,7 +169,7 @@
'J': 13, 'L': 13, 'e': 25, 'j': 13,
'l': 13},
# 22
- {'0': 22, '1': 22, 'L': 13, 'l': 13},
+ {'0': 36, '1': 36},
# 23
{'.': 24, '0': 23, '1': 23, '2': 23,
'3': 23, '4': 23, '5': 23, '6': 23,
@@ -179,59 +178,72 @@
# 24
{'0': 24, '1': 24, '2': 24, '3': 24,
'4': 24, '5': 24, '6': 24, '7': 24,
- '8': 24, '9': 24, 'E': 34, 'J': 13,
- 'e': 34, 'j': 13},
+ '8': 24, '9': 24, 'E': 37, 'J': 13,
+ 'e': 37, 'j': 13},
# 25
- {'+': 35, '-': 35, '0': 36, '1': 36,
- '2': 36, '3': 36, '4': 36, '5': 36,
- '6': 36, '7': 36, '8': 36, '9': 36},
+ {'+': 38, '-': 38, '0': 39, '1': 39,
+ '2': 39, '3': 39, '4': 39, '5': 39,
+ '6': 39, '7': 39, '8': 39, '9': 39},
# 26
{'0': 26, '1': 26, '2': 26, '3': 26,
'4': 26, '5': 26, '6': 26, '7': 26,
- '8': 26, '9': 26, 'E': 34, 'J': 13,
- 'e': 34, 'j': 13},
+ '8': 26, '9': 26, 'E': 37, 'J': 13,
+ 'e': 37, 'j': 13},
# 27
{},
# 28
{"'": 13},
# 29
- {automata.DEFAULT: 37, '\n': 13, '\r': 14},
+ {automata.DEFAULT: 40, '\n': 13, '\r': 14},
# 30
{automata.DEFAULT: 30, '\n': 27,
'\r': 27, "'": 13, '\\': 29},
# 31
{'"': 13},
# 32
- {automata.DEFAULT: 38, '\n': 13, '\r': 14},
+ {automata.DEFAULT: 41, '\n': 13, '\r': 14},
# 33
{automata.DEFAULT: 33, '\n': 27,
'\r': 27, '"': 13, '\\': 32},
# 34
- {'+': 39, '-': 39, '0': 40, '1': 40,
- '2': 40, '3': 40, '4': 40, '5': 40,
- '6': 40, '7': 40, '8': 40, '9': 40},
+ {'0': 34, '1': 34, '2': 34, '3': 34,
+ '4': 34, '5': 34, '6': 34, '7': 34,
+ '8': 34, '9': 34, 'A': 34, 'B': 34,
+ 'C': 34, 'D': 34, 'E': 34, 'F': 34,
+ 'L': 13, 'a': 34, 'b': 34, 'c': 34,
+ 'd': 34, 'e': 34, 'f': 34, 'l': 13},
# 35
- {'0': 36, '1': 36, '2': 36, '3': 36,
- '4': 36, '5': 36, '6': 36, '7': 36,
- '8': 36, '9': 36},
+ {'0': 35, '1': 35, '2': 35, '3': 35,
+ '4': 35, '5': 35, '6': 35, '7': 35,
+ 'L': 13, 'l': 13},
# 36
- {'0': 36, '1': 36, '2': 36, '3': 36,
- '4': 36, '5': 36, '6': 36, '7': 36,
- '8': 36, '9': 36, 'J': 13, 'j': 13},
+ {'0': 36, '1': 36, 'L': 13, 'l': 13},
# 37
- {automata.DEFAULT: 37, '\n': 27,
- '\r': 27, "'": 13, '\\': 29},
+ {'+': 42, '-': 42, '0': 43, '1': 43,
+ '2': 43, '3': 43, '4': 43, '5': 43,
+ '6': 43, '7': 43, '8': 43, '9': 43},
# 38
- {automata.DEFAULT: 38, '\n': 27,
- '\r': 27, '"': 13, '\\': 32},
+ {'0': 39, '1': 39, '2': 39, '3': 39,
+ '4': 39, '5': 39, '6': 39, '7': 39,
+ '8': 39, '9': 39},
# 39
- {'0': 40, '1': 40, '2': 40, '3': 40,
- '4': 40, '5': 40, '6': 40, '7': 40,
- '8': 40, '9': 40},
+ {'0': 39, '1': 39, '2': 39, '3': 39,
+ '4': 39, '5': 39, '6': 39, '7': 39,
+ '8': 39, '9': 39, 'J': 13, 'j': 13},
# 40
- {'0': 40, '1': 40, '2': 40, '3': 40,
- '4': 40, '5': 40, '6': 40, '7': 40,
- '8': 40, '9': 40, 'J': 13, 'j': 13},
+ {automata.DEFAULT: 40, '\n': 27,
+ '\r': 27, "'": 13, '\\': 29},
+ # 41
+ {automata.DEFAULT: 41, '\n': 27,
+ '\r': 27, '"': 13, '\\': 32},
+ # 42
+ {'0': 43, '1': 43, '2': 43, '3': 43,
+ '4': 43, '5': 43, '6': 43, '7': 43,
+ '8': 43, '9': 43},
+ # 43
+ {'0': 43, '1': 43, '2': 43, '3': 43,
+ '4': 43, '5': 43, '6': 43, '7': 43,
+ '8': 43, '9': 43, 'J': 13, 'j': 13},
]
pseudoDFA = automata.DFA(states, accepts)
@@ -307,9 +319,6 @@
'b' : None,
'B' : None}
-#_______________________________________________________________________
-# End of automatically generated DFA's
-
for uniPrefix in ("", "u", "U", "b", "B"):
for rawPrefix in ("", "r", "R"):
prefix = uniPrefix + rawPrefix
More information about the Pypy-commit
mailing list