[Python-checkins] CVS: python/dist/src/Lib dis.py,1.34,1.35 inspect.py,1.16,1.17 tabnanny.py,1.13,1.14 tokenize.py,1.22,1.23
Tim Peters
tim_one@users.sourceforge.net
Mon, 18 Jun 2001 15:08:15 -0700
- Previous message: [Python-checkins] CVS: python/dist/src/Include compile.h,2.29,2.30 frameobject.h,2.31,2.32 graminit.h,2.16,2.17 opcode.h,2.35,2.36 symtable.h,2.7,2.8
- Next message: [Python-checkins] CVS: python/dist/src/Objects frameobject.c,2.50,2.51
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Update of /cvsroot/python/python/dist/src/Lib
In directory usw-pr-cvs1:/tmp/cvs-serv4630/python/dist/src/Lib
Modified Files:
dis.py inspect.py tabnanny.py tokenize.py
Log Message:
Merging the gen-branch into the main line, at Guido's direction. Yay!
Bugfix candidate in inspect.py: it was referencing "self" outside of
a method.
Index: dis.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/dis.py,v
retrieving revision 1.34
retrieving revision 1.35
diff -C2 -r1.34 -r1.35
*** dis.py 2001/04/20 19:13:01 1.34
--- dis.py 2001/06/18 22:08:13 1.35
***************
*** 224,227 ****
--- 224,228 ----
def_op('IMPORT_STAR', 84)
def_op('EXEC_STMT', 85)
+ def_op('YIELD_STMT', 86)
def_op('POP_BLOCK', 87)
Index: inspect.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/inspect.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -r1.16 -r1.17
*** inspect.py 2001/04/13 14:04:02 1.16
--- inspect.py 2001/06/18 22:08:13 1.17
***************
*** 350,379 ****
else: return ''
! class EndOfBlock(Exception): pass
! class BlockFinder:
! """Provide a tokeneater() method to detect the end of a code block."""
! def __init__(self):
! self.indent = 0
! self.started = 0
! self.last = 0
! def tokeneater(self, type, token, (srow, scol), (erow, ecol), line):
! if not self.started:
! if type == tokenize.NAME: self.started = 1
elif type == tokenize.NEWLINE:
! self.last = srow
elif type == tokenize.INDENT:
! self.indent = self.indent + 1
elif type == tokenize.DEDENT:
! self.indent = self.indent - 1
! if self.indent == 0: raise EndOfBlock, self.last
!
! def getblock(lines):
! """Extract the block of code at the top of the given list of lines."""
! try:
! tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
! except EndOfBlock, eob:
! return lines[:eob.args[0]]
def getsourcelines(object):
--- 350,375 ----
else: return ''
! def getblock(lines):
! """Extract the block of code at the top of the given list of lines."""
! indent = 0
! started = 0
! last = 0
! tokens = tokenize.generate_tokens(ListReader(lines).readline)
! for (type, token, (srow, scol), (erow, ecol), line) in tokens:
! if not started:
! if type == tokenize.NAME:
! started = 1
elif type == tokenize.NEWLINE:
! last = srow
elif type == tokenize.INDENT:
! indent = indent + 1
elif type == tokenize.DEDENT:
! indent = indent - 1
! if indent == 0:
! return lines[:last]
! else:
! raise ValueError, "unable to find block"
def getsourcelines(object):
Index: tabnanny.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/tabnanny.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -r1.13 -r1.14
*** tabnanny.py 2001/04/08 00:38:42 1.13
--- tabnanny.py 2001/06/18 22:08:13 1.14
***************
*** 78,84 ****
print "checking", `file`, "..."
- reset_globals()
try:
! tokenize.tokenize(f.readline, tokeneater)
except tokenize.TokenError, msg:
--- 78,83 ----
print "checking", `file`, "..."
try:
! process_tokens(tokenize.generate_tokens(f.readline))
except tokenize.TokenError, msg:
***************
*** 245,270 ****
return prefix + " " + string.join(firsts, ', ')
! # The collection of globals, the reset_globals() function, and the
! # tokeneater() function, depend on which version of tokenize is
! # in use.
! if hasattr(tokenize, 'NL'):
! # take advantage of Guido's patch!
!
! indents = []
! check_equal = 0
!
! def reset_globals():
! global indents, check_equal
! check_equal = 0
! indents = [Whitespace("")]
!
! def tokeneater(type, token, start, end, line,
INDENT=tokenize.INDENT,
DEDENT=tokenize.DEDENT,
NEWLINE=tokenize.NEWLINE,
! JUNK=(tokenize.COMMENT, tokenize.NL) ):
! global indents, check_equal
if type == NEWLINE:
# a program statement, or ENDMARKER, will eventually follow,
--- 244,260 ----
return prefix + " " + string.join(firsts, ', ')
! # Need Guido's enhancement
! assert hasattr(tokenize, 'NL'), "tokenize module too old"
! def process_tokens(tokens,
INDENT=tokenize.INDENT,
DEDENT=tokenize.DEDENT,
NEWLINE=tokenize.NEWLINE,
! JUNK=(tokenize.COMMENT, tokenize.NL)):
+ indents = [Whitespace("")]
+ check_equal = 0
+
+ for (type, token, start, end, line) in tokens:
if type == NEWLINE:
# a program statement, or ENDMARKER, will eventually follow,
***************
*** 312,371 ****
raise NannyNag(start[0], msg, line)
- else:
- # unpatched version of tokenize
-
- nesting_level = 0
- indents = []
- check_equal = 0
-
- def reset_globals():
- global nesting_level, indents, check_equal
- nesting_level = check_equal = 0
- indents = [Whitespace("")]
-
- def tokeneater(type, token, start, end, line,
- INDENT=tokenize.INDENT,
- DEDENT=tokenize.DEDENT,
- NEWLINE=tokenize.NEWLINE,
- COMMENT=tokenize.COMMENT,
- OP=tokenize.OP):
- global nesting_level, indents, check_equal
-
- if type == INDENT:
- check_equal = 0
- thisguy = Whitespace(token)
- if not indents[-1].less(thisguy):
- witness = indents[-1].not_less_witness(thisguy)
- msg = "indent not greater e.g. " + format_witnesses(witness)
- raise NannyNag(start[0], msg, line)
- indents.append(thisguy)
-
- elif type == DEDENT:
- del indents[-1]
-
- elif type == NEWLINE:
- if nesting_level == 0:
- check_equal = 1
-
- elif type == COMMENT:
- pass
-
- elif check_equal:
- check_equal = 0
- thisguy = Whitespace(line)
- if not indents[-1].equal(thisguy):
- witness = indents[-1].not_equal_witness(thisguy)
- msg = "indent not equal e.g. " + format_witnesses(witness)
- raise NannyNag(start[0], msg, line)
-
- if type == OP and token in ('{', '[', '('):
- nesting_level = nesting_level + 1
-
- elif type == OP and token in ('}', ']', ')'):
- if nesting_level == 0:
- raise NannyNag(start[0],
- "unbalanced bracket '" + token + "'",
- line)
- nesting_level = nesting_level - 1
if __name__ == '__main__':
--- 302,305 ----
Index: tokenize.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/tokenize.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -r1.22 -r1.23
*** tokenize.py 2001/03/23 05:22:49 1.22
--- tokenize.py 2001/06/18 22:08:13 1.23
***************
*** 112,116 ****
--- 112,121 ----
pass
+ # backwards compatible interface, probably not used
def tokenize_loop(readline, tokeneater):
+ for token_info in generate_tokens(readline):
+ apply(tokeneater, token_info)
+
+ def generate_tokens(readline):
lnum = parenlev = continued = 0
namechars, numchars = string.letters + '_', string.digits
***************
*** 130,139 ****
if endmatch:
pos = end = endmatch.end(0)
! tokeneater(STRING, contstr + line[:end],
strstart, (lnum, end), contline + line)
contstr, needcont = '', 0
contline = None
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
! tokeneater(ERRORTOKEN, contstr + line,
strstart, (lnum, len(line)), contline)
contstr = ''
--- 135,144 ----
if endmatch:
pos = end = endmatch.end(0)
! yield (STRING, contstr + line[:end],
strstart, (lnum, end), contline + line)
contstr, needcont = '', 0
contline = None
elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
! yield (ERRORTOKEN, contstr + line,
strstart, (lnum, len(line)), contline)
contstr = ''
***************
*** 157,161 ****
if line[pos] in '#\r\n': # skip comments or blank lines
! tokeneater((NL, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)), line)
continue
--- 162,166 ----
if line[pos] in '#\r\n': # skip comments or blank lines
! yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
(lnum, pos), (lnum, len(line)), line)
continue
***************
*** 163,170 ****
if column > indents[-1]: # count indents or dedents
indents.append(column)
! tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
while column < indents[-1]:
indents = indents[:-1]
! tokeneater(DEDENT, '', (lnum, pos), (lnum, pos), line)
else: # continued statement
--- 168,175 ----
if column > indents[-1]: # count indents or dedents
indents.append(column)
! yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
while column < indents[-1]:
indents = indents[:-1]
! yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
else: # continued statement
***************
*** 182,191 ****
if initial in numchars or \
(initial == '.' and token != '.'): # ordinary number
! tokeneater(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
! tokeneater(parenlev > 0 and NL or NEWLINE,
token, spos, epos, line)
elif initial == '#':
! tokeneater(COMMENT, token, spos, epos, line)
elif token in ("'''", '"""', # triple-quoted
"r'''", 'r"""', "R'''", 'R"""',
--- 187,196 ----
if initial in numchars or \
(initial == '.' and token != '.'): # ordinary number
! yield (NUMBER, token, spos, epos, line)
elif initial in '\r\n':
! yield (parenlev > 0 and NL or NEWLINE,
token, spos, epos, line)
elif initial == '#':
! yield (COMMENT, token, spos, epos, line)
elif token in ("'''", '"""', # triple-quoted
"r'''", 'r"""', "R'''", 'R"""',
***************
*** 198,202 ****
pos = endmatch.end(0)
token = line[start:pos]
! tokeneater(STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
--- 203,207 ----
pos = endmatch.end(0)
token = line[start:pos]
! yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
***************
*** 217,223 ****
break
else: # ordinary string
! tokeneater(STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
! tokeneater(NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
continued = 1
--- 222,228 ----
break
else: # ordinary string
! yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
! yield (NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
continued = 1
***************
*** 225,237 ****
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
! tokeneater(OP, token, spos, epos, line)
else:
! tokeneater(ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
for indent in indents[1:]: # pop remaining indent levels
! tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '')
! tokeneater(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
if __name__ == '__main__': # testing
--- 230,242 ----
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
! yield (OP, token, spos, epos, line)
else:
! yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
for indent in indents[1:]: # pop remaining indent levels
! yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
! yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
if __name__ == '__main__': # testing
- Previous message: [Python-checkins] CVS: python/dist/src/Include compile.h,2.29,2.30 frameobject.h,2.31,2.32 graminit.h,2.16,2.17 opcode.h,2.35,2.36 symtable.h,2.7,2.8
- Next message: [Python-checkins] CVS: python/dist/src/Objects frameobject.c,2.50,2.51
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]