[Python-checkins] cpython (2.7): Issue #18873: IDLE, 2to3, and the findnocoding.py script now detect Python
serhiy.storchaka
python-checkins at python.org
Mon Sep 16 23:05:28 CEST 2013
http://hg.python.org/cpython/rev/3d46ef0c62c5
changeset: 85733:3d46ef0c62c5
branch: 2.7
parent: 85729:2e19c65d6688
user: Serhiy Storchaka <storchaka at gmail.com>
date: Tue Sep 17 00:00:46 2013 +0300
summary:
Issue #18873: IDLE, 2to3, and the findnocoding.py script now detect Python
source code encoding only in comment lines.
files:
Lib/idlelib/IOBinding.py | 13 ++++++-----
Lib/lib2to3/pgen2/tokenize.py | 9 +++----
Lib/lib2to3/tests/data/false_encoding.py | 2 +
Lib/lib2to3/tests/test_refactor.py | 4 +++
Misc/NEWS | 6 +++++
Tools/scripts/findnocoding.py | 6 ++--
6 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py
--- a/Lib/idlelib/IOBinding.py
+++ b/Lib/idlelib/IOBinding.py
@@ -71,7 +71,7 @@
encoding = encoding.lower()
-coding_re = re.compile("coding[:=]\s*([-\w_.]+)")
+coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
class EncodingMessage(SimpleDialog):
"Inform user that an encoding declaration is needed."
@@ -125,11 +125,12 @@
Raise LookupError if the encoding is declared but unknown.
"""
# Only consider the first two lines
- str = str.split("\n")[:2]
- str = "\n".join(str)
-
- match = coding_re.search(str)
- if not match:
+ str = str.split("\n", 2)[:2]
+ for line in lst:
+ match = coding_re.match(line)
+ if match is not None:
+ break
+ else:
return None
name = match.group(1)
# Check whether the encoding is known
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -236,7 +236,7 @@
startline = False
toks_append(tokval)
-cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
def _get_normal_name(orig_enc):
"""Imitates get_normal_name in tokenizer.c."""
@@ -281,11 +281,10 @@
line_string = line.decode('ascii')
except UnicodeDecodeError:
return None
-
- matches = cookie_re.findall(line_string)
- if not matches:
+ match = cookie_re.match(line_string)
+ if not match:
return None
- encoding = _get_normal_name(matches[0])
+ encoding = _get_normal_name(match.group(1))
try:
codec = lookup(encoding)
except LookupError:
diff --git a/Lib/lib2to3/tests/data/false_encoding.py b/Lib/lib2to3/tests/data/false_encoding.py
new file mode 100644
--- /dev/null
+++ b/Lib/lib2to3/tests/data/false_encoding.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+print '#coding=0'
diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py
--- a/Lib/lib2to3/tests/test_refactor.py
+++ b/Lib/lib2to3/tests/test_refactor.py
@@ -271,6 +271,10 @@
fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
self.check_file_refactoring(fn)
+ def test_false_file_encoding(self):
+ fn = os.path.join(TEST_DATA_DIR, "false_encoding.py")
+ data = self.check_file_refactoring(fn)
+
def test_bom(self):
fn = os.path.join(TEST_DATA_DIR, "bom.py")
data = self.check_file_refactoring(fn)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -183,6 +183,9 @@
Tools/Demos
-----------
+- Issue #18873: 2to3 and the findnocoding.py script now detect Python source
+ code encoding only in comment lines.
+
- Issue #18817: Fix a resource warning in Lib/aifc.py demo.
- Issue #18439: Make patchcheck work on Windows for ACKS, NEWS.
@@ -207,6 +210,9 @@
IDLE
----
+- Issue #18873: IDLE now detects Python source code encoding only in comment
+ lines.
+
- Issue #18988: The "Tab" key now works when a word is already autocompleted.
- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster.
diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py
--- a/Tools/scripts/findnocoding.py
+++ b/Tools/scripts/findnocoding.py
@@ -32,13 +32,13 @@
"no sophisticated Python source file search will be done.")
-decl_re = re.compile(r"coding[=:]\s*([-\w.]+)")
+decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
def get_declaration(line):
- match = decl_re.search(line)
+ match = decl_re.match(line)
if match:
return match.group(1)
- return ''
+ return b''
def has_correct_encoding(text, codec):
try:
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list