[pypy-svn] r14115 - in pypy/dist/pypy/interpreter/pyparser: . data test test/samples
adim at codespeak.net
adim at codespeak.net
Sun Jul 3 11:36:18 CEST 2005
Author: adim
Date: Sun Jul 3 11:36:11 2005
New Revision: 14115
Added:
pypy/dist/pypy/interpreter/pyparser/
pypy/dist/pypy/interpreter/pyparser/automata.py
- copied unchanged from r14092, pypy/dist/pypy/module/recparser/automata.py
pypy/dist/pypy/interpreter/pyparser/data/
- copied from r14092, pypy/dist/pypy/module/recparser/data/
pypy/dist/pypy/interpreter/pyparser/data/Grammar2.3
- copied unchanged from r14114, pypy/dist/pypy/module/recparser/data/Grammar2.3
pypy/dist/pypy/interpreter/pyparser/data/Grammar2.4
- copied unchanged from r14114, pypy/dist/pypy/module/recparser/data/Grammar2.4
pypy/dist/pypy/interpreter/pyparser/ebnflexer.py
- copied unchanged from r14092, pypy/dist/pypy/module/recparser/ebnflexer.py
pypy/dist/pypy/interpreter/pyparser/ebnfparse.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/ebnfparse.py
pypy/dist/pypy/interpreter/pyparser/grammar.py
- copied unchanged from r14092, pypy/dist/pypy/module/recparser/grammar.py
pypy/dist/pypy/interpreter/pyparser/pythonlexer.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/pythonlexer.py
pypy/dist/pypy/interpreter/pyparser/pythonparse.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/pythonparse.py
pypy/dist/pypy/interpreter/pyparser/pythonutil.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/pythonutil.py
pypy/dist/pypy/interpreter/pyparser/pytokenize.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/pytokenize.py
pypy/dist/pypy/interpreter/pyparser/syntaxtree.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/syntaxtree.py
pypy/dist/pypy/interpreter/pyparser/test/
- copied from r14092, pypy/dist/pypy/module/recparser/test/
pypy/dist/pypy/interpreter/pyparser/test/samples/
- copied from r14114, pypy/dist/pypy/module/recparser/test/samples/
pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/test_lookahead.py
pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py
- copied, changed from r14114, pypy/dist/pypy/module/recparser/test/test_pytokenizer.py
pypy/dist/pypy/interpreter/pyparser/test/test_samples.py
- copied, changed from r14114, pypy/dist/pypy/module/recparser/test/test_samples.py
pypy/dist/pypy/interpreter/pyparser/test/unittest_samples.py
- copied, changed from r14114, pypy/dist/pypy/module/recparser/test/unittest_samples.py
pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py
- copied, changed from r14092, pypy/dist/pypy/module/recparser/tuplebuilder.py
Log:
moved recparser's core into interpreter/pyparser
Copied: pypy/dist/pypy/interpreter/pyparser/ebnfparse.py (from r14092, pypy/dist/pypy/module/recparser/ebnfparse.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/ebnfparse.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/ebnfparse.py Sun Jul 3 11:36:11 2005
@@ -53,15 +53,7 @@
return 0
def match_token(self, other):
- """convenience '==' implementation, this is *not* a *real* equality test
- a Token instance can be compared to:
- - another Token instance in which case all fields (name and value)
- must be equal
- - a tuple, such as those yielded by the Python lexer, in which case
- the comparison algorithm is similar to the one in match()
- XXX:
- 1/ refactor match and __eq__ ?
- 2/ make source.next and source.peek return a Token() instance
+ """special case of match token for tokens which are really keywords
"""
if not isinstance(other, Token):
raise RuntimeError("Unexpected token type %r" % other)
Copied: pypy/dist/pypy/interpreter/pyparser/pythonlexer.py (from r14092, pypy/dist/pypy/module/recparser/pythonlexer.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/pythonlexer.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonlexer.py Sun Jul 3 11:36:11 2005
@@ -119,7 +119,8 @@
if contstr: # continued string
if not line:
- raise TokenError("EOF in multi-line string", strstart, token_list)
+ raise TokenError("EOF in multi-line string", strstart,
+ token_list)
endmatch = endDFA.recognize(line)
if -1 != endmatch:
pos = end = endmatch
@@ -166,8 +167,10 @@
else:
tok = token_from_values(tokenmod.NL, line[pos:])
last_comment = ''
- # XXX Skip NL and COMMENT Tokens # token_list.append((tok, line))
- # token_list.append(((NL, COMMENT)[line[pos] == '#'], line[pos:],
+ # XXX Skip NL and COMMENT Tokens
+ # token_list.append((tok, line))
+ # token_list.append(((NL, COMMENT)[line[pos] == '#'],
+ # line[pos:],
# (lnum, pos), (lnum, len(line)), line))
continue
@@ -176,13 +179,13 @@
tok = token_from_values(tokenmod.INDENT, line[:pos])
token_list.append((tok, line))
last_comment = ''
- # token_list.append((INDENT, line[:pos], (lnum, 0), (lnum, pos), line))
+ # token_list.append((INDENT, line[:pos],(lnum, 0),(lnum,pos),line))
while column < indents[-1]:
indents = indents[:-1]
tok = token_from_values(tokenmod.DEDENT, '')
token_list.append((tok, line))
last_comment = ''
- # token_list.append((DEDENT, '', (lnum, pos), (lnum, pos), line))
+ # token_list.append((DEDENT, '', (lnum, pos),(lnum,pos),line))
else: # continued statement
if not line:
@@ -305,6 +308,7 @@
self.stack_pos = 0
def next(self):
+ """Returns the next parsed token"""
if self.stack_pos >= len(self.token_stack):
raise StopIteration("Remove me")
tok, line = self.token_stack[self.stack_pos]
@@ -313,12 +317,15 @@
return tok
def current_line(self):
+ """Returns the current line being parsed"""
return self._current_line
def context(self):
+ """Returns an opaque context object for later restore"""
return self.stack_pos
def restore(self, ctx):
+ """Restores a context"""
self.stack_pos = ctx
def peek(self):
Copied: pypy/dist/pypy/interpreter/pyparser/pythonparse.py (from r14092, pypy/dist/pypy/module/recparser/pythonparse.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/pythonparse.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonparse.py Sun Jul 3 11:36:11 2005
@@ -34,6 +34,7 @@
builder.source_encoding = src.encoding
# </HACK>
if not result:
+ raise SyntaxError("...")
return None
# raise SyntaxError("at %s" % src.debug() )
return builder
Copied: pypy/dist/pypy/interpreter/pyparser/pythonutil.py (from r14092, pypy/dist/pypy/module/recparser/pythonutil.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/pythonutil.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pythonutil.py Sun Jul 3 11:36:11 2005
@@ -1,7 +1,6 @@
-__all__ = ["python_parse", "pypy_parse","ast_single_input", "ast_file_input",
+__all__ = ["python_parse", "pypy_parse", "ast_single_input", "ast_file_input",
"ast_eval_input" ]
-import grammar
import pythonparse
from compiler.transformer import Transformer
from tuplebuilder import TupleBuilder
@@ -33,8 +32,9 @@
pyf.close()
builder = TupleBuilder(PYTHON_PARSER.rules, lineno=False)
# make the annotator life easier
- strings = [line+'\n' for line in text.split('\n')]
- pythonparse.parse_python_source(strings, PYTHON_PARSER, 'file_input', builder)
+ strings = [line + '\n' for line in text.split('\n')]
+ pythonparse.parse_python_source(strings, PYTHON_PARSER,
+ 'file_input', builder)
nested_tuples = builder.stack[-1]
if builder.source_encoding is not None:
return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
Copied: pypy/dist/pypy/interpreter/pyparser/pytokenize.py (from r14092, pypy/dist/pypy/module/recparser/pytokenize.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/pytokenize.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/pytokenize.py Sun Jul 3 11:36:11 2005
@@ -20,7 +20,6 @@
# ______________________________________________________________________
from __future__ import generators
-import string
import automata
# ______________________________________________________________________
Copied: pypy/dist/pypy/interpreter/pyparser/syntaxtree.py (from r14092, pypy/dist/pypy/module/recparser/syntaxtree.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/syntaxtree.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/syntaxtree.py Sun Jul 3 11:36:11 2005
@@ -1,3 +1,4 @@
+"""SyntaxTree class definition"""
import symbol
import token
@@ -65,7 +66,7 @@
SYMBOLS = {}
# copies the numerical mapping between symbol name and symbol value
# into SYMBOLS
-for k,v in symbol.sym_name.items():
+for k, v in symbol.sym_name.items():
SYMBOLS[v] = k
SYMBOLS['UNKNOWN'] = -1
@@ -78,6 +79,7 @@
self.lineno = source.current_line()
def dumptree(self, treenodes, indent):
+ """helper function used to dump the syntax tree"""
treenodes.append(self.name)
if len(self.nodes) > 1:
treenodes.append(" -> (\n")
@@ -92,6 +94,7 @@
self.nodes[0].dumptree(treenodes, indent+" ")
def dumpstr(self):
+ """turns the tree repr into a string"""
treenodes = []
self.dumptree(treenodes, "")
return "".join(treenodes)
@@ -113,9 +116,13 @@
raise RuntimeError("Unknonw Visitor for %r" % self.name)
def expand(self):
+ """expand the syntax node to its content,
+ do nothing here since we are a regular node and not
+ a TempSyntaxNode"""
return [ self ]
def totuple(self, lineno=False ):
+ """returns a tuple representation of the syntax tree"""
symvalue = SYMBOLS.get( self.name, (0, self.name) )
l = [ symvalue ]
l += [node.totuple(lineno) for node in self.nodes]
@@ -125,6 +132,7 @@
class TempSyntaxNode(SyntaxNode):
"""A temporary syntax node to represent intermediate rules"""
def expand(self):
+ """expand the syntax node to its content"""
return self.nodes
class TokenNode(SyntaxNode):
@@ -134,8 +142,10 @@
self.value = value
def dumptree(self, treenodes, indent):
+ """helper function used to dump the syntax tree"""
if self.value:
- treenodes.append("%s='%s' (%d) " % (self.name, self.value, self.lineno))
+ treenodes.append("%s='%s' (%d) " % (self.name, self.value,
+ self.lineno))
else:
treenodes.append("'%s' (%d) " % (self.name, self.lineno))
@@ -146,6 +156,7 @@
return "<%s!>" % (self.name,)
def totuple(self, lineno=False):
+ """returns a tuple representation of the syntax tree"""
num = TOKEN_MAP.get(self.name, -1)
if num == -1:
print "Unknown", self.name, self.value
Copied: pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py (from r14092, pypy/dist/pypy/module/recparser/test_lookahead.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/test_lookahead.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_lookahead.py Sun Jul 3 11:36:11 2005
@@ -1,4 +1,4 @@
-from pypy.module.recparser.grammar import Alternative, Sequence, KleenStar, \
+from pypy.interpreter.pyparser.grammar import Alternative, Sequence, KleenStar, \
Token, EmptyToken, build_first_sets
class TestLookAheadBasics:
Copied: pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py (from r14114, pypy/dist/pypy/module/recparser/test/test_pytokenizer.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/test/test_pytokenizer.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_pytokenizer.py Sun Jul 3 11:36:11 2005
@@ -1,6 +1,6 @@
-from pypy.module.recparser.pythonlexer import Source, TokenError, \
+from pypy.interpreter.pyparser.pythonlexer import Source, TokenError, \
match_encoding_declaration
-from pypy.module.recparser.grammar import Token
+from pypy.interpreter.pyparser.grammar import Token
def parse_source(source):
"""returns list of parsed tokens"""
Copied: pypy/dist/pypy/interpreter/pyparser/test/test_samples.py (from r14114, pypy/dist/pypy/module/recparser/test/test_samples.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/test/test_samples.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/test_samples.py Sun Jul 3 11:36:11 2005
@@ -1,9 +1,9 @@
"""test module for CPython / PyPy nested tuples comparison"""
import os, os.path as osp
-from pypy.module.recparser.pythonutil import python_parse, pypy_parse
+from pypy.interpreter.pyparser.pythonutil import python_parse, pypy_parse
# from pypy.module.recparser.pythonparse import pypy_parse
from pprint import pprint
-from pypy.module.recparser import grammar
+from pypy.interpreter.pyparser import grammar
grammar.DEBUG = False
from symbol import sym_name
Copied: pypy/dist/pypy/interpreter/pyparser/test/unittest_samples.py (from r14114, pypy/dist/pypy/module/recparser/test/unittest_samples.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/test/unittest_samples.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/test/unittest_samples.py Sun Jul 3 11:36:11 2005
@@ -2,9 +2,9 @@
import os, os.path as osp
import sys
-from pypy.module.recparser.pythonutil import python_parse, pypy_parse
+from pypy.interpreter.pyparser.pythonutil import python_parse, pypy_parse
from pprint import pprint
-from pypy.module.recparser import grammar
+from pypy.interpreter.pyparser import grammar
grammar.DEBUG = False
from symbol import sym_name
Copied: pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py (from r14092, pypy/dist/pypy/module/recparser/tuplebuilder.py)
==============================================================================
--- pypy/dist/pypy/module/recparser/tuplebuilder.py (original)
+++ pypy/dist/pypy/interpreter/pyparser/tuplebuilder.py Sun Jul 3 11:36:11 2005
@@ -2,24 +2,6 @@
from grammar import BaseGrammarBuilder
from syntaxtree import TOKEN_MAP, SYMBOLS # , NT_OFFSET
-##
-## def _expand_nodes(nodes):
-## expanded = []
-## for n in nodes:
-## if n[0] == -2:
-## # expanded.extend(expand_nodes(n[1:]))
-## expanded.extend(n[1:])
-## else:
-## expanded.append(n)
-## return tuple(expanded)
-##
-## def expand_nodes(nodes):
-## r = _expand_nodes(nodes)
-## for n in nodes:
-## assert type(n[0]) == int
-## return r
-##
-
class StackElement:
"""wraps TupleBuilder's tuples"""
More information about the Pypy-commit
mailing list