[Python-checkins] r67714 - in python/branches/release30-maint: Lib/test/test_tokenize.py Lib/tokenize.py Misc/NEWS

[benjamin.peterson]

Author: benjamin.peterson
Date: Fri Dec 12 02:40:00 2008
New Revision: 67714

Log:
Merged revisions 67711 via svnmerge from 
svn+ssh://pythondev@svn.python.org/python/branches/py3k

........
  r67711 | benjamin.peterson | 2008-12-11 19:25:05 -0600 (Thu, 11 Dec 2008) | 1 line
  
  raise a SyntaxError in detect_encoding() when a codec lookup fails like the builtin parser #4021
........


Modified:
   python/branches/release30-maint/   (props changed)
   python/branches/release30-maint/Lib/test/test_tokenize.py
   python/branches/release30-maint/Lib/tokenize.py
   python/branches/release30-maint/Misc/NEWS

Modified: python/branches/release30-maint/Lib/test/test_tokenize.py
==============================================================================
--- python/branches/release30-maint/Lib/test/test_tokenize.py	(original)
+++ python/branches/release30-maint/Lib/test/test_tokenize.py	Fri Dec 12 02:40:00 2008
@@ -795,6 +795,8 @@
         self.assertEquals(encoding, 'utf-8')
         self.assertEquals(consumed_lines, [])
 
+        readline = self.get_readline((b'# coding: bad\n',))
+        self.assertRaises(SyntaxError, detect_encoding, readline)
 
 class TestTokenize(TestCase):
 

Modified: python/branches/release30-maint/Lib/tokenize.py
==============================================================================
--- python/branches/release30-maint/Lib/tokenize.py	(original)
+++ python/branches/release30-maint/Lib/tokenize.py	Fri Dec 12 02:40:00 2008
@@ -26,7 +26,7 @@
 
 import re, string, sys
 from token import *
-from codecs import lookup
+from codecs import lookup, BOM_UTF8
 from itertools import chain, repeat
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
@@ -251,11 +251,11 @@
 
     It detects the encoding from the presence of a utf-8 bom or an encoding
     cookie as specified in pep-0263. If both a bom and a cookie are present,
-    but disagree, a SyntaxError will be raised.
+    but disagree, a SyntaxError will be raised. If the encoding cookie is an
+    invalid charset, raise a SyntaxError.
 
     If no encoding is specified, then the default of 'utf-8' will be returned.
     """
-    utf8_bom = b'\xef\xbb\xbf'
     bom_found = False
     encoding = None
     def read_or_stop():
@@ -268,18 +268,25 @@
         try:
             line_string = line.decode('ascii')
         except UnicodeDecodeError:
-            pass
-        else:
-            matches = cookie_re.findall(line_string)
-            if matches:
-                encoding = matches[0]
-                if bom_found and lookup(encoding).name != 'utf-8':
-                    # This behaviour mimics the Python interpreter
-                    raise SyntaxError('encoding problem: utf-8')
-                return encoding
+            return None
+
+        matches = cookie_re.findall(line_string)
+        if not matches:
+            return None
+        encoding = matches[0]
+        try:
+            codec = lookup(encoding)
+        except LookupError:
+            # This behaviour mimics the Python interpreter
+            raise SyntaxError("unknown encoding: " + encoding)
+
+        if bom_found and codec.name != 'utf-8':
+            # This behaviour mimics the Python interpreter
+            raise SyntaxError('encoding problem: utf-8')
+        return encoding
 
     first = read_or_stop()
-    if first.startswith(utf8_bom):
+    if first.startswith(BOM_UTF8):
         bom_found = True
         first = first[3:]
     if not first:

Modified: python/branches/release30-maint/Misc/NEWS
==============================================================================
--- python/branches/release30-maint/Misc/NEWS	(original)
+++ python/branches/release30-maint/Misc/NEWS	Fri Dec 12 02:40:00 2008
@@ -35,6 +35,9 @@
 Library
 -------
 
+- Issue #4021: tokenize.detect_encoding() now raises a SyntaxError when the
+  codec cannot be found.  This is for compatibility with the builtin behavior.
+
 - Issue #4084: Fix max, min, max_mag and min_mag Decimal methods to
   give correct results in the case where one argument is a quiet NaN
   and the other is a finite number that requires rounding.