[pypy-svn] r66238 - in pypy/branch/parser-compiler/pypy/interpreter/pyparser: . test

[benjamin at codespeak.net]

Author: benjamin
Date: Wed Jul 15 16:34:16 2009
New Revision: 66238

Modified:
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py
   pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
some fiddling to get identation handled properly in the tokenizer

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pyparse.py	Wed Jul 15 16:34:16 2009
@@ -102,9 +102,14 @@
                         raise error.SyntaxError("Unknown encoding: %s" % enc)
                     raise
 
+        flags = compile_info.flags
+        source_lines = textsrc.splitlines(True)
+        if textsrc and textsrc[-1] == "\n":
+            flags &= ~codeop.PyCF_DONT_IMPLY_DEDENT
+
         self.prepare(_targets[compile_info.mode])
         try:
-            tokens = pytokenizer.generate_tokens(textsrc, compile_info.flags)
+            tokens = pytokenizer.generate_tokens(source_lines, flags)
             for tp, value, lineno, column, line in tokens:
                 if self.add_token(tp, value, lineno, column, line):
                     break

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/pytokenizer.py	Wed Jul 15 16:34:16 2009
@@ -44,7 +44,7 @@
     return None
 
 
-def generate_tokens(textsrc, flags):
+def generate_tokens(lines, flags):
     """
     This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
     the original function is not RPYTHON (uses yield)
@@ -83,8 +83,7 @@
     endDFA = automata.DFA([], [])
     # make the annotator happy
     line = ''
-    lines = textsrc.splitlines(True)
-    lines.append('')
+    lines.append("")
     for line in lines:
         lnum = lnum + 1
         pos, max = 0, len(line)
@@ -157,9 +156,12 @@
                 end = pseudomatch
 
                 if start == end:
-                    # Nothing matched!!!
-                    raise TokenError("Unknown character", line,
-                                 lnum, start, token_list)
+                    if line[pos] == ' ':
+                        break
+                    else:
+                        # Nothing matched!!!
+                        raise TokenError("Unknown character", line,
+                                         lnum, start, token_list)
 
                 pos = end
                 token, initial = line[start:end], line[start]

Modified: pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py	(original)
+++ pypy/branch/parser-compiler/pypy/interpreter/pyparser/test/test_pyparse.py	Wed Jul 15 16:34:16 2009
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import codeop
 import py
 from pypy.interpreter.pyparser import pyparse
 from pypy.interpreter.pyparser.pygram import syms, tokens
@@ -15,6 +16,12 @@
             info = pyparse.CompileInfo("<test>", mode)
         return self.parser.parse_source(source, info)
 
+    def test_dont_imply_dedent(self):
+        info = pyparse.CompileInfo("<test>", "single",
+                                   codeop.PyCF_DONT_IMPLY_DEDENT)
+        self.parse('if 1:\n  x\n', info=info)
+        self.parse('x = 5 ', info=info)
+
     def test_clear_state(self):
         assert self.parser.root is None
         tree = self.parse("name = 32")