[pypy-svn] r18610 - in pypy/dist/pypy/interpreter: pyparser test

[arigo at codespeak.net]

Author: arigo
Date: Sat Oct 15 13:07:29 2005
New Revision: 18610

Modified:
   pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
   pypy/dist/pypy/interpreter/test/test_compiler.py
Log:
Make the lexer generate "sensible" line numbers:
continuation lines really increase the current line number.



Modified: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
==============================================================================
--- pypy/dist/pypy/interpreter/pyparser/pythonlexer.py	(original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonlexer.py	Sat Oct 15 13:07:29 2005
@@ -67,7 +67,12 @@
     This is a rewrite of pypy.module.parser.pytokenize.generate_tokens since
     the original function is not RPYTHON (uses yield)
     It was also slightly modified to generate Token instances instead
-    of the original 5-tuples
+    of the original 5-tuples -- it's now a 4-tuple of
+    
+    * the Token instance
+    * the whole line as a string
+    * the line number (the real one, counting continuation lines)
+    * the position on the line of the end of the token.
 
     Original docstring ::
     
@@ -189,7 +194,7 @@
                     raise TokenError("Unknown character", line,
                                  (lnum, start), token_list)
 
-                spos, epos, pos = (lnum, start), (lnum, end), end
+                pos = end
                 token, initial = line[start:end], line[start]
                 if initial in numchars or \
                    (initial == '.' and token != '.'):      # ordinary number
@@ -244,7 +249,7 @@
                     last_comment = ''
                 elif initial == '\\':                      # continued stmt
                     continued = 1
-                    lnum -= 1
+                    # lnum -= 1  disabled: count continuation lines separately
                 else:
                     if initial in '([{':
                         parenlev = parenlev + 1

Modified: pypy/dist/pypy/interpreter/test/test_compiler.py
==============================================================================
--- pypy/dist/pypy/interpreter/test/test_compiler.py	(original)
+++ pypy/dist/pypy/interpreter/test/test_compiler.py	Sat Oct 15 13:07:29 2005
@@ -336,7 +336,7 @@
         w_fline = space.getitem(w_d, space.wrap('fline'))
         w_gline = space.getitem(w_d, space.wrap('gline'))
         assert space.int_w(w_fline) == 2
-        #IN-PROGRESS assert space.int_w(w_gline) == 6
+        assert space.int_w(w_gline) == 6
 
 class TestECCompiler(BaseTestCompiler):
     def setup_method(self, method):