[Python-checkins] CVS: python/dist/src/Lib inspect.py,1.17,1.18 tabnanny.py,1.14,1.15 tokenize.py,1.23,1.24
Tim Peters
tim_one@users.sourceforge.net
Fri, 29 Jun 2001 16:51:10 -0700
Update of /cvsroot/python/python/dist/src/Lib
In directory usw-pr-cvs1:/tmp/cvs-serv5587/python/dist/src/Lib
Modified Files:
inspect.py tabnanny.py tokenize.py
Log Message:
Turns out Neil didn't intend for *all* of his gen-branch work to get
committed.
tokenize.py: I like these changes, and have tested them extensively
without even realizing it, so I just updated the docstring and the docs.
tabnanny.py: Also liked this, but did a little code fiddling. I should
really rewrite this to *exploit* generators, but that's near the bottom
of my effort/benefit scale so doubt I'll get to it anytime soon (it
would be most useful as a non-trivial example of ideal use of generators;
but test_generators.py has already grown plenty of food-for-thought
examples).
inspect.py: I'm sure Ping intended for this to continue running even
under 1.5.2, so I reverted this to the last pre-gen-branch version. The
"bugfix" I checked in in-between was actually repairing a bug *introduced*
by the conversion to generators, so it's OK that the reverted version
doesn't reflect that checkin.
Index: inspect.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/inspect.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -r1.17 -r1.18
*** inspect.py 2001/06/18 22:08:13 1.17
--- inspect.py 2001/06/29 23:51:08 1.18
***************
*** 350,375 ****
else: return ''
! def getblock(lines):
! """Extract the block of code at the top of the given list of lines."""
! indent = 0
! started = 0
! last = 0
! tokens = tokenize.generate_tokens(ListReader(lines).readline)
! for (type, token, (srow, scol), (erow, ecol), line) in tokens:
! if not started:
! if type == tokenize.NAME:
! started = 1
elif type == tokenize.NEWLINE:
! last = srow
elif type == tokenize.INDENT:
! indent = indent + 1
elif type == tokenize.DEDENT:
! indent = indent - 1
! if indent == 0:
! return lines[:last]
! else:
! raise ValueError, "unable to find block"
def getsourcelines(object):
--- 350,379 ----
else: return ''
! class EndOfBlock(Exception): pass
! class BlockFinder:
! """Provide a tokeneater() method to detect the end of a code block."""
! def __init__(self):
! self.indent = 0
! self.started = 0
! self.last = 0
! def tokeneater(self, type, token, (srow, scol), (erow, ecol), line):
! if not self.started:
! if type == tokenize.NAME: self.started = 1
elif type == tokenize.NEWLINE:
! self.last = srow
elif type == tokenize.INDENT:
! self.indent = self.indent + 1
elif type == tokenize.DEDENT:
! self.indent = self.indent - 1
! if self.indent == 0: raise EndOfBlock, self.last
!
! def getblock(lines):
! """Extract the block of code at the top of the given list of lines."""
! try:
! tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
! except EndOfBlock, eob:
! return lines[:eob.args[0]]
def getsourcelines(object):
Index: tabnanny.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/tabnanny.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -C2 -r1.14 -r1.15
*** tabnanny.py 2001/06/18 22:08:13 1.14
--- tabnanny.py 2001/06/29 23:51:08 1.15
***************
*** 15,18 ****
--- 15,20 ----
import getopt
import tokenize
+ if not hasattr(tokenize, 'NL'):
+ raise ValueError("tokenize.NL doesn't exist -- tokenize module too old")
__all__ = ["check"]
***************
*** 244,256 ****
return prefix + " " + string.join(firsts, ', ')
! # Need Guido's enhancement
! assert hasattr(tokenize, 'NL'), "tokenize module too old"
!
! def process_tokens(tokens,
! INDENT=tokenize.INDENT,
! DEDENT=tokenize.DEDENT,
! NEWLINE=tokenize.NEWLINE,
! JUNK=(tokenize.COMMENT, tokenize.NL)):
!
indents = [Whitespace("")]
check_equal = 0
--- 246,254 ----
return prefix + " " + string.join(firsts, ', ')
! def process_tokens(tokens):
! INDENT = tokenize.INDENT
! DEDENT = tokenize.DEDENT
! NEWLINE = tokenize.NEWLINE
! JUNK = tokenize.COMMENT, tokenize.NL
indents = [Whitespace("")]
check_equal = 0
Index: tokenize.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/tokenize.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -r1.23 -r1.24
*** tokenize.py 2001/06/18 22:08:13 1.23
--- tokenize.py 2001/06/29 23:51:08 1.24
***************
*** 1,13 ****
"""Tokenization help for Python programs.
! This module exports a function called 'tokenize()' that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
! repeatedly to get the next line of input (or "" for EOF) and a "token-eater"
! function which is called once for each token found. The latter function is
! passed the token type, a string containing the token, the starting and
! ending (row, column) coordinates of the token, and the original line. It is
! designed to match the working of the Python tokenizer exactly, except that
! it produces COMMENT tokens for comments and gives type OP for all operators."""
__author__ = 'Ka-Ping Yee <ping@lfw.org>'
__credits__ = \
--- 1,26 ----
"""Tokenization help for Python programs.
! generate_tokens(readline) is a generator that breaks a stream of
text into Python tokens. It accepts a readline-like method which is called
! repeatedly to get the next line of input (or "" for EOF). It generates
! 5-tuples with these members:
+ the token type (see token.py)
+ the token (a string)
+ the starting (row, column) indices of the token (a 2-tuple of ints)
+ the ending (row, column) indices of the token (a 2-tuple of ints)
+ the original line (string)
+
+ It is designed to match the working of the Python tokenizer exactly, except
+ that it produces COMMENT tokens for comments and gives type OP for all
+ operators
+
+ Older entry points
+ tokenize_loop(readline, tokeneater)
+ tokenize(readline, tokeneater=printtoken)
+ are the same, except instead of generating tokens, tokeneater is a callback
+ function to which the 5 fields described above are passed as 5 arguments,
+ each time a new token is found."""
+
__author__ = 'Ka-Ping Yee <ping@lfw.org>'
__credits__ = \
***************
*** 112,116 ****
pass
! # backwards compatible interface, probably not used
def tokenize_loop(readline, tokeneater):
for token_info in generate_tokens(readline):
--- 125,129 ----
pass
! # backwards compatible interface
def tokenize_loop(readline, tokeneater):
for token_info in generate_tokens(readline):