[Python-checkins] r67746 - in python/trunk: Lib/test/test_textwrap.py Lib/textwrap.py Misc/NEWS
antoine.pitrou
python-checkins at python.org
Sun Dec 14 00:12:31 CET 2008
Author: antoine.pitrou
Date: Sun Dec 14 00:12:30 2008
New Revision: 67746
Log:
Issue #4163: Use unicode-friendly word splitting in the textwrap functions when given an unicode string.
Modified:
python/trunk/Lib/test/test_textwrap.py
python/trunk/Lib/textwrap.py
python/trunk/Misc/NEWS
Modified: python/trunk/Lib/test/test_textwrap.py
==============================================================================
--- python/trunk/Lib/test/test_textwrap.py (original)
+++ python/trunk/Lib/test/test_textwrap.py Sun Dec 14 00:12:30 2008
@@ -174,7 +174,7 @@
text = ("Python 1.0.0 was released on 1994-01-26. Python 1.0.1 was\n"
"released on 1994-02-15.")
- self.check_wrap(text, 30, ['Python 1.0.0 was released on',
+ self.check_wrap(text, 35, ['Python 1.0.0 was released on',
'1994-01-26. Python 1.0.1 was',
'released on 1994-02-15.'])
self.check_wrap(text, 40, ['Python 1.0.0 was released on 1994-01-26.',
@@ -353,6 +353,14 @@
otext = self.wrapper.fill(text)
assert isinstance(otext, unicode)
+ def test_no_split_at_umlaut(self):
+ text = u"Die Empf\xe4nger-Auswahl"
+ self.check_wrap(text, 13, [u"Die", u"Empf\xe4nger-", u"Auswahl"])
+
+ def test_umlaut_followed_by_dash(self):
+ text = u"aa \xe4\xe4-\xe4\xe4"
+ self.check_wrap(text, 7, [u"aa \xe4\xe4-", u"\xe4\xe4"])
+
def test_split(self):
# Ensure that the standard _split() method works as advertised
# in the comments
Modified: python/trunk/Lib/textwrap.py
==============================================================================
--- python/trunk/Lib/textwrap.py (original)
+++ python/trunk/Lib/textwrap.py Sun Dec 14 00:12:30 2008
@@ -84,16 +84,16 @@
# splits into
# Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option!
# (after stripping out empty strings).
- wordsep_re = re.compile(
+ wordsep_re = (
r'(\s+|' # any whitespace
- r'[^\s\w]*\w+[a-zA-Z]-(?=\w+[a-zA-Z])|' # hyphenated words
+ r'[^\s\w]*\w+[^0-9\W]-(?=\w+[^0-9\W])|' # hyphenated words
r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash
# This less funky little regex just split on recognized spaces. E.g.
# "Hello there -- you goof-ball, use the -b option!"
# splits into
# Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/
- wordsep_simple_re = re.compile(r'(\s+)')
+ wordsep_simple_re = r'(\s+)'
# XXX this is not locale- or charset-aware -- string.lowercase
# is US-ASCII only (and therefore English-only)
@@ -160,10 +160,12 @@
'use', ' ', 'the', ' ', '-b', ' ', option!'
otherwise.
"""
- if self.break_on_hyphens is True:
- chunks = self.wordsep_re.split(text)
+ flags = re.UNICODE if isinstance(text, unicode) else 0
+ if self.break_on_hyphens:
+ pat = self.wordsep_re
else:
- chunks = self.wordsep_simple_re.split(text)
+ pat = self.wordsep_simple_re
+ chunks = re.compile(pat, flags).split(text)
chunks = filter(None, chunks) # remove empty chunks
return chunks
Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS (original)
+++ python/trunk/Misc/NEWS Sun Dec 14 00:12:30 2008
@@ -74,6 +74,9 @@
Library
-------
+- Issue #4163: Use unicode-friendly word splitting in the textwrap functions
+ when given an unicode string.
+
- Issue #4616: TarFile.utime(): Restore directory times on Windows.
- Issue #4084: Fix max, min, max_mag and min_mag Decimal methods to
More information about the Python-checkins
mailing list