[Python-checkins] CVS: python/dist/src/Lib tokenize.py,1.21,1.22
Ka-Ping Yee
ping@users.sourceforge.net
Thu, 22 Mar 2001 21:22:51 -0800
Update of /cvsroot/python/python/dist/src/Lib
In directory usw-pr-cvs1:/tmp/cvs-serv22253
Modified Files:
tokenize.py
Log Message:
Provide a StopTokenizing exception for conveniently exiting the loop.
Index: tokenize.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/tokenize.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -r1.21 -r1.22
*** tokenize.py 2001/03/01 17:11:17 1.21
--- tokenize.py 2001/03/23 05:22:49 1.22
***************
*** 27,36 ****
N_TOKENS += 2
- # Changes from 1.3:
- # Ignore now accepts \f as whitespace. Operator now includes '**'.
- # Ignore and Special now accept \n or \r\n at the end of a line.
- # Imagnumber is new. Expfloat is corrected to reject '0e4'.
- # Note: to quote a backslash in a regex, it must be doubled in a r'aw' string.
-
def group(*choices): return '(' + '|'.join(choices) + ')'
def any(*choices): return apply(group, choices) + '*'
--- 27,30 ----
***************
*** 103,109 ****
tabsize = 8
! class TokenError(Exception):
! pass
def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
--- 97,104 ----
tabsize = 8
+
+ class TokenError(Exception): pass
! class StopTokenizing(Exception): pass
def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
***************
*** 112,115 ****
--- 107,116 ----
def tokenize(readline, tokeneater=printtoken):
+ try:
+ tokenize_loop(readline, tokeneater)
+ except StopTokenizing:
+ pass
+
+ def tokenize_loop(readline, tokeneater):
lnum = parenlev = continued = 0
namechars, numchars = string.letters + '_', string.digits
***************
*** 179,184 ****
token, initial = line[start:end], line[start]
! if initial in numchars \
! or (initial == '.' and token != '.'): # ordinary number
tokeneater(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
--- 180,185 ----
token, initial = line[start:end], line[start]
! if initial in numchars or \
! (initial == '.' and token != '.'): # ordinary number
tokeneater(NUMBER, token, spos, epos, line)
elif initial in '\r\n':