[pypy-svn] r66238 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test
benjamin at codespeak.net
benjamin at codespeak.net
Wed Jul 15 16:34:17 CEST 2009
Author: benjamin
Date: Wed Jul 15 16:34:16 2009
New Revision: 66238
Modified:
pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py
pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
some fiddling to get identation handled properly in the tokenizer
Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py Wed Jul 15 16:34:16 2009
@@ -102,9 +102,14 @@
raise error.SyntaxError("Unknown encoding: %s" % enc)
raise
+ flags = compile_info.flags
+ source_lines = textsrc.splitlines(True)
+ if textsrc and textsrc[-1] == "\n":
+ flags &= ~codeop.PyCF_DONT_IMPLY_DEDENT
+
self.prepare(_targets[compile_info.mode])
try:
- tokens = pytokenizer.generate_tokens(textsrc, compile_info.flags)
+ tokens = pytokenizer.generate_tokens(source_lines, flags)
for tp, value, lineno, column, line in tokens:
if self.add_token(tp, value, lineno, column, line):
break
Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py Wed Jul 15 16:34:16 2009
@@ -44,7 +44,7 @@
return None
-def generate_tokens(textsrc, flags):
+def generate_tokens(lines, flags):
"""
This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
the original function is not RPYTHON (uses yield)
@@ -83,8 +83,7 @@
endDFA = automata.DFA([], [])
# make the annotator happy
line = ''
- lines = textsrc.splitlines(True)
- lines.append('')
+ lines.append("")
for line in lines:
lnum = lnum + 1
pos, max = 0, len(line)
@@ -157,9 +156,12 @@
end = pseudomatch
if start == end:
- # Nothing matched!!!
- raise TokenError("Unknown character", line,
- lnum, start, token_list)
+ if line[pos] == ' ':
+ break
+ else:
+ # Nothing matched!!!
+ raise TokenError("Unknown character", line,
+ lnum, start, token_list)
pos = end
token, initial = line[start:end], line[start]
Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py (original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py Wed Jul 15 16:34:16 2009
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
+import codeop
import py
from pypy.interpreter.pyparser import pyparse
from pypy.interpreter.pyparser.pygram import syms, tokens
@@ -15,6 +16,12 @@
info = pyparse.CompileInfo("<test>", mode)
return self.parser.parse_source(source, info)
+ def test_dont_imply_dedent(self):
+ info = pyparse.CompileInfo("<test>", "single",
+ codeop.PyCF_DONT_IMPLY_DEDENT)
+ self.parse('if 1:\n x\n', info=info)
+ self.parse('x = 5 ', info=info)
+
def test_clear_state(self):
assert self.parser.root is None
tree = self.parse("name = 32")
More information about the Pypy-commit
mailing list