[Python-checkins] CVS: python/dist/src/Lib tokenize.py,1.21,1.22

[Ka-Ping Yee]

Update of /cvsroot/python/python/dist/src/Lib
In directory usw-pr-cvs1:/tmp/cvs-serv22253

Modified Files:
	tokenize.py 
Log Message:
Provide a StopTokenizing exception for conveniently exiting the loop.


Index: tokenize.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/tokenize.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -C2 -r1.21 -r1.22
*** tokenize.py	2001/03/01 17:11:17	1.21
--- tokenize.py	2001/03/23 05:22:49	1.22
***************
*** 27,36 ****
  N_TOKENS += 2
  
- # Changes from 1.3:
- #     Ignore now accepts \f as whitespace.  Operator now includes '**'.
- #     Ignore and Special now accept \n or \r\n at the end of a line.
- #     Imagnumber is new.  Expfloat is corrected to reject '0e4'.
- # Note: to quote a backslash in a regex, it must be doubled in a r'aw' string.
- 
  def group(*choices): return '(' + '|'.join(choices) + ')'
  def any(*choices): return apply(group, choices) + '*'
--- 27,30 ----
***************
*** 103,109 ****
  
  tabsize = 8
  
! class TokenError(Exception):
!     pass
  
  def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
--- 97,104 ----
  
  tabsize = 8
+ 
+ class TokenError(Exception): pass
  
! class StopTokenizing(Exception): pass
  
  def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing
***************
*** 112,115 ****
--- 107,116 ----
  
  def tokenize(readline, tokeneater=printtoken):
+     try:
+         tokenize_loop(readline, tokeneater)
+     except StopTokenizing:
+         pass
+ 
+ def tokenize_loop(readline, tokeneater):
      lnum = parenlev = continued = 0
      namechars, numchars = string.letters + '_', string.digits
***************
*** 179,184 ****
                  token, initial = line[start:end], line[start]
  
!                 if initial in numchars \
!                     or (initial == '.' and token != '.'):  # ordinary number
                      tokeneater(NUMBER, token, spos, epos, line)
                  elif initial in '\r\n':
--- 180,185 ----
                  token, initial = line[start:end], line[start]
  
!                 if initial in numchars or \
!                    (initial == '.' and token != '.'):      # ordinary number
                      tokeneater(NUMBER, token, spos, epos, line)
                  elif initial in '\r\n':