[pypy-svn] r78651 - in pypy/trunk/pypy/interpreter/pyparser: . test

[afa at codespeak.net]

Author: afa
Date: Tue Nov  2 22:11:26 2010
New Revision: 78651

Modified:
   pypy/trunk/pypy/interpreter/pyparser/genpytokenize.py
   pypy/trunk/pypy/interpreter/pyparser/pytokenize.py
   pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py
Log:
Merge r78320 from the fast-foward branch:
"""
Allow \r as a newline;
also tokens that go to the end of the line
should stop on either \r or \n.
"""

It should fix some line ending on windows,
like the extra blank lines in the test_gzip output for
http://buildbot.pypy.org/summary?category=windows


Modified: pypy/trunk/pypy/interpreter/pyparser/genpytokenize.py
==============================================================================
--- pypy/trunk/pypy/interpreter/pyparser/genpytokenize.py	(original)
+++ pypy/trunk/pypy/interpreter/pyparser/genpytokenize.py	Tue Nov  2 22:11:26 2010
@@ -17,12 +17,17 @@
 def makePyPseudoDFA ():
     import string
     states = []
+    def makeEOL():
+        return group(states,
+                     newArcPair(states, "\n"),
+                     chain(states,
+                           newArcPair(states, "\r"),
+                           maybe(states, newArcPair(states, "\n"))))
     # ____________________________________________________________
     def makeLineCont ():
         return chain(states,
                      newArcPair(states, "\\"),
-                     maybe(states, newArcPair(states, "\r")),
-                     newArcPair(states, "\n"))
+                     makeEOL())
     # ____________________________________________________________
     # Ignore stuff
     def makeWhitespace ():
@@ -124,9 +129,7 @@
                      newArcPair(states, "~"))
     bracket = groupStr(states, "[](){}")
     special = group(states,
-                    chain(states,
-                          maybe(states, newArcPair(states, "\r")),
-                          newArcPair(states, "\n")),
+                    makeEOL(),
                     groupStr(states, "@:;.,`"))
     funny = group(states, operator, bracket, special)
     # ____________________________________________________________
@@ -140,13 +143,13 @@
                           makeStrPrefix(),
                           newArcPair(states, "'"),
                           any(states,
-                              notGroupStr(states, "\n'\\")),
+                              notGroupStr(states, "\r\n'\\")),
                           any(states,
                               chain(states,
                                     newArcPair(states, "\\"),
                                     newArcPair(states, DEFAULT),
                                     any(states,
-                                        notGroupStr(states, "\n'\\")))),
+                                        notGroupStr(states, "\r\n'\\")))),
                           group(states,
                                 newArcPair(states, "'"),
                                 makeLineCont())),
@@ -154,13 +157,13 @@
                           makeStrPrefix(),
                           newArcPair(states, '"'),
                           any(states,
-                              notGroupStr(states, '\n"\\')),
+                              notGroupStr(states, '\r\n"\\')),
                           any(states,
                               chain(states,
                                     newArcPair(states, "\\"),
                                     newArcPair(states, DEFAULT),
                                     any(states,
-                                        notGroupStr(states, '\n"\\')))),
+                                        notGroupStr(states, '\r\n"\\')))),
                           group(states,
                                 newArcPair(states, '"'),
                                 makeLineCont())))

Modified: pypy/trunk/pypy/interpreter/pyparser/pytokenize.py
==============================================================================
--- pypy/trunk/pypy/interpreter/pyparser/pytokenize.py	(original)
+++ pypy/trunk/pypy/interpreter/pyparser/pytokenize.py	Tue Nov  2 22:11:26 2010
@@ -24,7 +24,7 @@
 # Automatically generated DFA's
 
 accepts = [True, True, True, True, True, True, True, True,
-           True, True, False, True, True, True, False, False,
+           True, True, False, True, True, True, True, False,
            False, False, True, True, True, False, True,
            False, True, False, True, False, False, True,
            False, False, False, False, True, False, False,
@@ -142,9 +142,11 @@
     # 14
     {'\n': 13},
     # 15
-    {automata.DEFAULT: 28, '\n': 25, "'": 26, '\\': 27},
+    {automata.DEFAULT: 28, '\n': 25,
+     '\r': 25, "'": 26, '\\': 27},
     # 16
-    {automata.DEFAULT: 31, '\n': 25, '"': 29, '\\': 30},
+    {automata.DEFAULT: 31, '\n': 25,
+     '\r': 25, '"': 29, '\\': 30},
     # 17
     {'\n': 13, '\r': 14},
     # 18
@@ -188,13 +190,15 @@
     # 27
     {automata.DEFAULT: 35, '\n': 13, '\r': 14},
     # 28
-    {automata.DEFAULT: 28, '\n': 25, "'": 13, '\\': 27},
+    {automata.DEFAULT: 28, '\n': 25,
+     '\r': 25, "'": 13, '\\': 27},
     # 29
     {'"': 13},
     # 30
     {automata.DEFAULT: 36, '\n': 13, '\r': 14},
     # 31
-    {automata.DEFAULT: 31, '\n': 25, '"': 13, '\\': 30},
+    {automata.DEFAULT: 31, '\n': 25,
+     '\r': 25, '"': 13, '\\': 30},
     # 32
     {'+': 37, '-': 37, '0': 38, '1': 38,
      '2': 38, '3': 38, '4': 38, '5': 38,
@@ -208,9 +212,11 @@
      '4': 34, '5': 34, '6': 34, '7': 34,
      '8': 34, '9': 34, 'J': 13, 'j': 13},
     # 35
-    {automata.DEFAULT: 35, '\n': 25, "'": 13, '\\': 27},
+    {automata.DEFAULT: 35, '\n': 25,
+     '\r': 25, "'": 13, '\\': 27},
     # 36
-    {automata.DEFAULT: 36, '\n': 25, '"': 13, '\\': 30},
+    {automata.DEFAULT: 36, '\n': 25,
+     '\r': 25, '"': 13, '\\': 30},
     # 37
     {'0': 38, '1': 38, '2': 38, '3': 38,
      '4': 38, '5': 38, '6': 38, '7': 38,
@@ -282,7 +288,6 @@
     ]
 doubleDFA = automata.DFA(states, accepts)
 
-
 #_______________________________________________________________________
 # End of automatically generated DFA's
 

Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py
==============================================================================
--- pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py	(original)
+++ pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py	Tue Nov  2 22:11:26 2010
@@ -92,6 +92,9 @@
         exc = py.test.raises(IndentationError, parse, input).value
         assert exc.msg == "unindent does not match any outer indentation level"
 
+    def test_mac_newline(self):
+        self.parse("this_is\ra_mac\rfile")
+
     def test_mode(self):
         assert self.parse("x = 43*54").type == syms.file_input
         tree = self.parse("43**54", "eval")