[Python-checkins] r57746 - in sandbox/trunk/2to3: pgen2/tokenize.py tests/test_parser.py

[collin.winter]

Author: collin.winter
Date: Thu Aug 30 20:12:11 2007
New Revision: 57746

Modified:
   sandbox/trunk/2to3/   (props changed)
   sandbox/trunk/2to3/pgen2/tokenize.py
   sandbox/trunk/2to3/tests/test_parser.py
Log:
Fix another parser problem with bytes literals.


Modified: sandbox/trunk/2to3/pgen2/tokenize.py
==============================================================================
--- sandbox/trunk/2to3/pgen2/tokenize.py	(original)
+++ sandbox/trunk/2to3/pgen2/tokenize.py	Thu Aug 30 20:12:11 2007
@@ -66,7 +66,7 @@
 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
 # Tail end of """ string.
 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-Triple = group("[uU]?[rR]?'''", '[uU]?[rR]?"""')
+Triple = group("[ubUB]?[rR]?'''", '[ubUB]?[rR]?"""')
 # Single-line ' or " string.
 String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
                r'[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
@@ -120,8 +120,11 @@
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "u'''", 'u"""', "U'''", 'U"""',
+          "b'''", 'b"""', "B'''", 'B"""',
           "ur'''", 'ur"""', "Ur'''", 'Ur"""',
-          "uR'''", 'uR"""', "UR'''", 'UR"""'):
+          "uR'''", 'uR"""', "UR'''", 'UR"""',
+          "br'''", 'br"""', "Br'''", 'Br"""',
+          "bR'''", 'bR"""', "BR'''", 'BR"""',):
     triple_quoted[t] = t
 single_quoted = {}
 for t in ("'", '"',

Modified: sandbox/trunk/2to3/tests/test_parser.py
==============================================================================
--- sandbox/trunk/2to3/tests/test_parser.py	(original)
+++ sandbox/trunk/2to3/tests/test_parser.py	Thu Aug 30 20:12:11 2007
@@ -166,6 +166,15 @@
             """
         self.validate(s)
 
+    def test_multiline_bytes_tripquote_literals(self):
+        s = '''
+            b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
+            """
+            '''
+        self.validate(s)
+
     def test_multiline_str_literals(self):
         s = """
             md5test("\xaa" * 80,