[Python-checkins] r76266 - python/branches/py3k/Lib/tokenize.py

Sat Nov 14 19:09:17 CET 2009

Author: benjamin.peterson
Date: Sat Nov 14 19:09:17 2009
New Revision: 76266

Log:
use some more itertools magic to make '' be yielded after readline is done

Modified:
   python/branches/py3k/Lib/tokenize.py

Modified: python/branches/py3k/Lib/tokenize.py
==============================================================================

--- python/branches/py3k/Lib/tokenize.py	(original)
+++ python/branches/py3k/Lib/tokenize.py	Sat Nov 14 19:09:17 2009
@@ -379,10 +379,11 @@
     """
     # This import is here to avoid problems when the itertools module is not
     # built yet and tokenize is imported.
-    from itertools import chain
+    from itertools import chain, repeat
     encoding, consumed = detect_encoding(readline)
-    rl_iter = iter(readline, "")
-    return _tokenize(chain(consumed, rl_iter).__next__, encoding)
+    rl_gen = iter(readline, b"")
+    empty = repeat(b"")
+    return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
 
 
 def _tokenize(readline, encoding):
