[pypy-svn] r36451 - in pypy/branch/ast-experiments/pypy: interpreter/stablecompiler module/recparser module/symbol

adim at codespeak.net adim at codespeak.net
Thu Jan 11 10:37:03 CET 2007


Author: adim
Date: Thu Jan 11 10:36:52 2007
New Revision: 36451

Modified:
   pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py
   pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py
   pypy/branch/ast-experiments/pypy/module/symbol/__init__.py
Log:
try to avoid using global PYTHON_PARSER

Modified: pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py	(original)
+++ pypy/branch/ast-experiments/pypy/interpreter/stablecompiler/transformer.py	Thu Jan 11 10:36:52 2007
@@ -131,14 +131,14 @@
             if hasattr(self, name):
                 self._dispatch[value] = getattr(self, name)
             
-        self._dispatch[token.NEWLINE] = self.com_NEWLINE
-        self._atom_dispatch = {token.LPAR: self.atom_lpar,
-                               token.LSQB: self.atom_lsqb,
-                               token.LBRACE: self.atom_lbrace,
-                               token.BACKQUOTE: self.atom_backquote,
-                               token.NUMBER: self.atom_number,
-                               token.STRING: self.atom_string,
-                               token.NAME: self.atom_name,
+        self._dispatch[stable_parser.tokens['NEWLINE']] = self.com_NEWLINE
+        self._atom_dispatch = {stable_parser.tokens['LPAR']: self.atom_lpar,
+                               stable_parser.tokens['LSQB']: self.atom_lsqb,
+                               stable_parser.tokens['LBRACE']: self.atom_lbrace,
+                               stable_parser.tokens['BACKQUOTE']: self.atom_backquote,
+                               stable_parser.tokens['NUMBER']: self.atom_number,
+                               stable_parser.tokens['STRING']: self.atom_string,
+                               stable_parser.tokens['NAME']: self.atom_name,
                                }
         self.encoding = None
 
@@ -207,7 +207,7 @@
     def single_input(self, node):
         # NEWLINE | simple_stmt | compound_stmt NEWLINE
         n = node[0][0]
-        if n != token.NEWLINE:
+        if n != stable_parser.tokens['NEWLINE']:
             stmt = self.com_stmt(node[0])
         else:
             stmt = Pass()
@@ -217,7 +217,7 @@
         doc = self.get_docstring(nodelist, symbol.file_input)
         stmts = []
         for node in nodelist:
-            if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
+            if node[0] != stable_parser.tokens['ENDMARKER'] and node[0] != stable_parser.tokens['NEWLINE']:
                 self.com_append_stmt(stmts, node)
 
         if doc is not None:
@@ -238,8 +238,8 @@
         item = self.atom_name(nodelist)
         i = 1
         while i < listlen:
-            assert nodelist[i][0] == token.DOT
-            assert nodelist[i + 1][0] == token.NAME
+            assert nodelist[i][0] == stable_parser.tokens['DOT']
+            assert nodelist[i + 1][0] == stable_parser.tokens['NAME']
             item = Getattr(item, nodelist[i + 1][1])
             i += 2
 
@@ -248,14 +248,14 @@
     def decorator(self, nodelist):
         # '@' dotted_name [ '(' [arglist] ')' ]
         assert len(nodelist) in (3, 5, 6)
-        assert nodelist[0][0] == token.AT
-        assert nodelist[-1][0] == token.NEWLINE
+        assert nodelist[0][0] == stable_parser.tokens['AT']
+        assert nodelist[-1][0] == stable_parser.tokens['NEWLINE']
 
         assert nodelist[1][0] == symbol.dotted_name
         funcname = self.decorator_name(nodelist[1][1:])
 
         if len(nodelist) > 3:
-            assert nodelist[2][0] == token.LPAR
+            assert nodelist[2][0] == stable_parser.tokens['LPAR']
             expr = self.com_call_function(funcname, nodelist[3])
         else:
             expr = funcname
@@ -328,7 +328,7 @@
         # classdef: 'class' NAME ['(' testlist ')'] ':' suite
         name = nodelist[1][1]
         doc = self.get_docstring(nodelist[-1])
-        if nodelist[2][0] == token.COLON:
+        if nodelist[2][0] == stable_parser.tokens['COLON']:
             bases = []
         else:
             bases = self.com_bases(nodelist[3])
@@ -397,7 +397,7 @@
         exprNode = self.lookup_node(en)(en[1:])
         if len(nodelist) == 1:
             return Discard(exprNode, lineno=exprNode.lineno)
-        if nodelist[1][0] == token.EQUAL:
+        if nodelist[1][0] == stable_parser.tokens['EQUAL']:
             nodesl = []
             for i in range(0, len(nodelist) - 2, 2):
                 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
@@ -414,9 +414,9 @@
         if len(nodelist) == 1:
             start = 1
             dest = None
-        elif nodelist[1][0] == token.RIGHTSHIFT:
+        elif nodelist[1][0] == stable_parser.tokens['RIGHTSHIFT']:
             assert len(nodelist) == 3 \
-                   or nodelist[3][0] == token.COMMA
+                   or nodelist[3][0] == stable_parser.tokens['COMMA']
             dest = self.com_node(nodelist[2])
             start = 4
         else:
@@ -424,7 +424,7 @@
             start = 1
         for i in range(start, len(nodelist), 2):
             items.append(self.com_node(nodelist[i]))
-        if nodelist[-1][0] == token.COMMA:
+        if nodelist[-1][0] == stable_parser.tokens['COMMA']:
             return Print(items, dest, lineno=nodelist[0][2])
         return Printnl(items, dest, lineno=nodelist[0][2])
 
@@ -482,15 +482,15 @@
         assert nodelist[1][0] == symbol.dotted_name
         assert nodelist[2][1] == 'import'
         fromname = self.com_dotted_name(nodelist[1])
-        if nodelist[3][0] == token.STAR:
+        if nodelist[3][0] == stable_parser.tokens['STAR']:
             return From(fromname, [('*', None)],
                         lineno=nodelist[0][2])
         else:
-            if nodelist[3][0] == token.LPAR:
+            if nodelist[3][0] == stable_parser.tokens['LPAR']:
                 node = nodelist[4]
             else:
                 node = nodelist[3]
-                if node[-1][0] == token.COMMA:
+                if node[-1][0] == stable_parser.tokens['COMMA']:
                     self.syntaxerror("trailing comma not allowed without surrounding parentheses", node)
             return From(fromname, self.com_import_as_names(node),
                         lineno=nodelist[0][2])
@@ -662,7 +662,7 @@
             # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
             #          | 'in' | 'not' 'in' | 'is' | 'is' 'not'
             n = nl[1]
-            if n[0] == token.NAME:
+            if n[0] == stable_parser.tokens['NAME']:
                 type = n[1]
                 if len(nl) == 3:
                     if type == 'not':
@@ -699,9 +699,9 @@
         node = self.com_node(nodelist[0])
         for i in range(2, len(nodelist), 2):
             right = self.com_node(nodelist[i])
-            if nodelist[i-1][0] == token.LEFTSHIFT:
+            if nodelist[i-1][0] == stable_parser.tokens['LEFTSHIFT']:
                 node = LeftShift([node, right], lineno=nodelist[1][2])
-            elif nodelist[i-1][0] == token.RIGHTSHIFT:
+            elif nodelist[i-1][0] == stable_parser.tokens['RIGHTSHIFT']:
                 node = RightShift([node, right], lineno=nodelist[1][2])
             else:
                 raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
@@ -711,9 +711,9 @@
         node = self.com_node(nodelist[0])
         for i in range(2, len(nodelist), 2):
             right = self.com_node(nodelist[i])
-            if nodelist[i-1][0] == token.PLUS:
+            if nodelist[i-1][0] == stable_parser.tokens['PLUS']:
                 node = Add([node, right], lineno=nodelist[1][2])
-            elif nodelist[i-1][0] == token.MINUS:
+            elif nodelist[i-1][0] == stable_parser.tokens['MINUS']:
                 node = Sub([node, right], lineno=nodelist[1][2])
             else:
                 raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
@@ -724,13 +724,13 @@
         for i in range(2, len(nodelist), 2):
             right = self.com_node(nodelist[i])
             t = nodelist[i-1][0]
-            if t == token.STAR:
+            if t == stable_parser.tokens['STAR']:
                 node = Mul([node, right])
-            elif t == token.SLASH:
+            elif t == stable_parser.tokens['SLASH']:
                 node = Div([node, right])
-            elif t == token.PERCENT:
+            elif t == stable_parser.tokens['PERCENT']:
                 node = Mod([node, right])
-            elif t == token.DOUBLESLASH:
+            elif t == stable_parser.tokens['DOUBLESLASH']:
                 node = FloorDiv([node, right])
             else:
                 raise ValueError, "unexpected token: %s" % t
@@ -742,11 +742,11 @@
         t = elt[0]
         node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
         # need to handle (unary op)constant here...
-        if t == token.PLUS:
+        if t == stable_parser.tokens['PLUS']:
             return UnaryAdd(node, lineno=elt[2])
-        elif t == token.MINUS:
+        elif t == stable_parser.tokens['MINUS']:
             return UnarySub(node, lineno=elt[2])
-        elif t == token.TILDE:
+        elif t == stable_parser.tokens['TILDE']:
             node = Invert(node, lineno=elt[2])
         return node
 
@@ -755,7 +755,7 @@
         node = self.com_node(nodelist[0])
         for i in range(1, len(nodelist)):
             elt = nodelist[i]
-            if elt[0] == token.DOUBLESTAR:
+            if elt[0] == stable_parser.tokens['DOUBLESTAR']:
                 return Power([node, self.com_node(nodelist[i+1])],
                              lineno=elt[2])
 
@@ -769,17 +769,17 @@
         return n
 
     def atom_lpar(self, nodelist):
-        if nodelist[1][0] == token.RPAR:
+        if nodelist[1][0] == stable_parser.tokens['RPAR']:
             return Tuple(())
         return self.com_node(nodelist[1])
 
     def atom_lsqb(self, nodelist):
-        if nodelist[1][0] == token.RSQB:
+        if nodelist[1][0] == stable_parser.tokens['RSQB']:
             return List([], lineno=nodelist[0][2])
         return self.com_list_constructor(nodelist[1], nodelist[0][2])
 
     def atom_lbrace(self, nodelist):
-        if nodelist[1][0] == token.RBRACE:
+        if nodelist[1][0] == stable_parser.tokens['RBRACE']:
             return Dict(())
         return self.com_dictmaker(nodelist[1])
 
@@ -854,10 +854,10 @@
         i = 0
         while i < len(nodelist):
             node = nodelist[i]
-            if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
-                if node[0] == token.STAR:
+            if node[0] == stable_parser.tokens['STAR'] or node[0] == stable_parser.tokens['DOUBLESTAR']:
+                if node[0] == stable_parser.tokens['STAR']:
                     node = nodelist[i+1]
-                    if node[0] == token.NAME:
+                    if node[0] == stable_parser.tokens['NAME']:
                         name = node[1]
                         if name in names:
                             self.syntaxerror("duplicate argument '%s' in function definition" %
@@ -869,7 +869,7 @@
                 if i < len(nodelist):
                     # should be DOUBLESTAR
                     t = nodelist[i][0]
-                    if t == token.DOUBLESTAR:
+                    if t == stable_parser.tokens['DOUBLESTAR']:
                         node = nodelist[i+1]
                     else:
                         raise ValueError, "unexpected token: %s" % t
@@ -895,7 +895,7 @@
                     self.syntaxerror("non-default argument follows default argument",node)
                 break
             
-            if nodelist[i][0] == token.EQUAL:
+            if nodelist[i][0] == stable_parser.tokens['EQUAL']:
                 defaults.append(self.com_node(nodelist[i + 1]))
                 i = i + 2
             elif len(defaults):
@@ -909,7 +909,7 @@
 
     def com_fpdef(self, node):
         # fpdef: NAME | '(' fplist ')'
-        if node[1][0] == token.LPAR:
+        if node[1][0] == stable_parser.tokens['LPAR']:
             return self.com_fplist(node[2])
         return node[1][1]
 
@@ -937,7 +937,7 @@
         if len(node) == 1:
             return dot, None
         assert node[1][1] == 'as'
-        assert node[2][0] == token.NAME
+        assert node[2][0] == stable_parser.tokens['NAME']
         return dot, node[2][1]
 
     def com_dotted_as_names(self, node):
@@ -951,11 +951,11 @@
     def com_import_as_name(self, node):
         assert node[0] == symbol.import_as_name
         node = node[1:]
-        assert node[0][0] == token.NAME
+        assert node[0][0] == stable_parser.tokens['NAME']
         if len(node) == 1:
             return node[0][1], None
         assert node[1][1] == 'as', node
-        assert node[2][0] == token.NAME
+        assert node[2][0] == stable_parser.tokens['NAME']
         return node[0][1], node[2][1]
 
     def com_import_as_names(self, node):
@@ -998,7 +998,7 @@
                     expr1 = expr2 = None
                 clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))
 
-            if node[0] == token.NAME:
+            if node[0] == stable_parser.tokens['NAME']:
                 elseNode = self.com_node(nodelist[i+2])
         return TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
                          lineno=nodelist[0][2])
@@ -1042,7 +1042,7 @@
                     primary = self.com_node(node[1])
                     for i in range(2, len(node)-1):
                         ch = node[i]
-                        if ch[0] == token.DOUBLESTAR:
+                        if ch[0] == stable_parser.tokens['DOUBLESTAR']:
                             self.syntaxerror( "can't assign to operator", node)
                         primary = self.com_apply_trailer(primary, ch)
                     return self.com_assign_trailer(primary, node[-1],
@@ -1050,16 +1050,16 @@
                 node = node[1]
             elif t == symbol.atom:
                 t = node[1][0]
-                if t == token.LPAR:
+                if t == stable_parser.tokens['LPAR']:
                     node = node[2]
-                    if node[0] == token.RPAR:
+                    if node[0] == stable_parser.tokens['RPAR']:
                         self.syntaxerror( "can't assign to ()", node)
-                elif t == token.LSQB:
+                elif t == stable_parser.tokens['LSQB']:
                     node = node[2]
-                    if node[0] == token.RSQB:
+                    if node[0] == stable_parser.tokens['RSQB']:
                         self.syntaxerror( "can't assign to []", node)
                     return self.com_assign_list(node, assigning)
-                elif t == token.NAME:
+                elif t == stable_parser.tokens['NAME']:
                     if node[1][1] == "__debug__":
                         self.syntaxerror( "can not assign to __debug__", node )
                     if node[1][1] == "None":
@@ -1085,7 +1085,7 @@
             if i + 1 < len(node):
                 if node[i + 1][0] == symbol.list_for:
                     self.syntaxerror( "can't assign to list comprehension", node)
-                assert node[i + 1][0] == token.COMMA, node[i + 1]
+                assert node[i + 1][0] == stable_parser.tokens['COMMA'], node[i + 1]
             assigns.append(self.com_assign(node[i], assigning))
         return AssList(assigns, lineno=extractLineNo(node))
 
@@ -1094,11 +1094,11 @@
 
     def com_assign_trailer(self, primary, node, assigning):
         t = node[1][0]
-        if t == token.DOT:
+        if t == stable_parser.tokens['DOT']:
             return self.com_assign_attr(primary, node[2], assigning)
-        if t == token.LSQB:
+        if t == stable_parser.tokens['LSQB']:
             return self.com_subscriptlist(primary, node[2], assigning)
-        if t == token.LPAR:
+        if t == stable_parser.tokens['LPAR']:
             if assigning==OP_DELETE:
                 self.syntaxerror( "can't delete function call", node)
             else:
@@ -1146,7 +1146,7 @@
                     assert len(nodelist[i:]) == 1
                     return self.com_list_comprehension(values[0],
                                                        nodelist[i])
-                elif nodelist[i][0] == token.COMMA:
+                elif nodelist[i][0] == stable_parser.tokens['COMMA']:
                     continue
                 values.append(self.com_node(nodelist[i]))
             return List(values, lineno=lineno)
@@ -1245,29 +1245,29 @@
 
     def com_apply_trailer(self, primaryNode, nodelist):
         t = nodelist[1][0]
-        if t == token.LPAR:
+        if t == stable_parser.tokens['LPAR']:
             return self.com_call_function(primaryNode, nodelist[2])
-        if t == token.DOT:
+        if t == stable_parser.tokens['DOT']:
             return self.com_select_member(primaryNode, nodelist[2])
-        if t == token.LSQB:
+        if t == stable_parser.tokens['LSQB']:
             return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)
 
         self.syntaxerror( 'unknown node type: %s' % t, nodelist[1])
 
     def com_select_member(self, primaryNode, nodelist):
-        if nodelist[0] != token.NAME:
+        if nodelist[0] != stable_parser.tokens['NAME']:
             self.syntaxerror( "member must be a name", nodelist[0])
         return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
 
     def com_call_function(self, primaryNode, nodelist):
-        if nodelist[0] == token.RPAR:
+        if nodelist[0] == stable_parser.tokens['RPAR']:
             return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
         args = []
         kw = 0
         len_nodelist = len(nodelist)
         for i in range(1, len_nodelist, 2):
             node = nodelist[i]
-            if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
+            if node[0] == stable_parser.tokens['STAR'] or node[0] == stable_parser.tokens['DOUBLESTAR']:
                 break
             kw, result = self.com_argument(node, kw)
 
@@ -1281,7 +1281,7 @@
         else:
             # No broken by star arg, so skip the last one we processed.
             i = i + 1
-        if i < len_nodelist and nodelist[i][0] == token.COMMA:
+        if i < len_nodelist and nodelist[i][0] == stable_parser.tokens['COMMA']:
             # need to accept an application that looks like "f(a, b,)"
             i = i + 1
         star_node = dstar_node = None
@@ -1289,11 +1289,11 @@
             tok = nodelist[i]
             ch = nodelist[i+1]
             i = i + 3
-            if tok[0]==token.STAR:
+            if tok[0]==stable_parser.tokens['STAR']:
                 if star_node is not None:
                     self.syntaxerror( 'already have the varargs indentifier', tok )
                 star_node = self.com_node(ch)
-            elif tok[0]==token.DOUBLESTAR:
+            elif tok[0]==stable_parser.tokens['DOUBLESTAR']:
                 if dstar_node is not None:
                     self.syntaxerror( 'already have the kwargs indentifier', tok )
                 dstar_node = self.com_node(ch)
@@ -1312,9 +1312,9 @@
             return 0, self.com_node(nodelist[1])
         result = self.com_node(nodelist[3])
         n = nodelist[1]
-        while len(n) == 2 and n[0] != token.NAME:
+        while len(n) == 2 and n[0] != stable_parser.tokens['NAME']:
             n = n[1]
-        if n[0] != token.NAME:
+        if n[0] != stable_parser.tokens['NAME']:
             self.syntaxerror( "keyword can't be an expression (%s)"%n[0], n)
         node = Keyword(n[1], result, lineno=n[2])
         return 1, node
@@ -1328,8 +1328,8 @@
         # backwards compat slice for '[i:j]'
         if len(nodelist) == 2:
             sub = nodelist[1]
-            if (sub[1][0] == token.COLON or \
-                            (len(sub) > 2 and sub[2][0] == token.COLON)) and \
+            if (sub[1][0] == stable_parser.tokens['COLON'] or \
+                            (len(sub) > 2 and sub[2][0] == stable_parser.tokens['COLON'])) and \
                             sub[-1][0] != symbol.sliceop:
                 return self.com_slice(primary, sub, assigning)
 
@@ -1343,9 +1343,9 @@
         # slice_item: expression | proper_slice | ellipsis
         ch = node[1]
         t = ch[0]
-        if t == token.DOT and node[2][0] == token.DOT:
+        if t == stable_parser.tokens['DOT'] and node[2][0] == stable_parser.tokens['DOT']:
             return Ellipsis()
-        if t == token.COLON or len(node) > 2:
+        if t == stable_parser.tokens['COLON'] or len(node) > 2:
             return self.com_sliceobj(node)
         return self.com_node(ch)
 
@@ -1361,7 +1361,7 @@
 
         items = []
 
-        if node[1][0] == token.COLON:
+        if node[1][0] == stable_parser.tokens['COLON']:
             items.append(Const(None))
             i = 2
         else:
@@ -1389,7 +1389,7 @@
         # short_slice:  [lower_bound] ":" [upper_bound]
         lower = upper = None
         if len(node) == 3:
-            if node[1][0] == token.COLON:
+            if node[1][0] == stable_parser.tokens['COLON']:
                 upper = self.com_node(node[2])
             else:
                 lower = self.com_node(node[1])
@@ -1416,7 +1416,7 @@
                     return self.get_docstring(sub)
             return None
         if n == symbol.atom:
-            if node[0][0] == token.STRING:
+            if node[0][0] == stable_parser.tokens['STRING']:
                 s = ''
                 for t in node:
                     s = s + eval(t[1])
@@ -1453,13 +1453,13 @@
 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
 #             | 'in' | 'not' 'in' | 'is' | 'is' 'not'
 _cmp_types = {
-    token.LESS : '<',
-    token.GREATER : '>',
-    token.EQEQUAL : '==',
-    token.EQUAL : '==',
-    token.LESSEQUAL : '<=',
-    token.GREATEREQUAL : '>=',
-    token.NOTEQUAL : '!=',
+    stable_parser.tokens['LESS'] : '<',
+    stable_parser.tokens['GREATER'] : '>',
+    stable_parser.tokens['EQEQUAL'] : '==',
+    stable_parser.tokens['EQUAL'] : '==',
+    stable_parser.tokens['LESSEQUAL'] : '<=',
+    stable_parser.tokens['GREATEREQUAL'] : '>=',
+    stable_parser.tokens['NOTEQUAL'] : '!=',
     }
 
 _assign_types = [
@@ -1478,20 +1478,20 @@
     symbol.factor,
     ]
 
-import types
-_names = {}
-for k, v in sym_name.items():
-    _names[k] = v
-for k, v in token.tok_name.items():
-    _names[k] = v
-
-def debug_tree(tree):
-    l = []
-    for elt in tree:
-        if type(elt) == types.IntType:
-            l.append(_names.get(elt, elt))
-        elif type(elt) == types.StringType:
-            l.append(elt)
-        else:
-            l.append(debug_tree(elt))
-    return l
+# import types
+# _names = {}
+# for k, v in sym_name.items():
+#     _names[k] = v
+# for k, v in token.tok_name.items():
+#     _names[k] = v
+# 
+# def debug_tree(tree):
+#     l = []
+#     for elt in tree:
+#         if type(elt) == types.IntType:
+#             l.append(_names.get(elt, elt))
+#         elif type(elt) == types.StringType:
+#             l.append(elt)
+#         else:
+#             l.append(debug_tree(elt))
+#     return l

Modified: pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py	(original)
+++ pypy/branch/ast-experiments/pypy/module/recparser/pyparser.py	Thu Jan 11 10:36:52 2007
@@ -8,11 +8,13 @@
 from pypy.interpreter.typedef import interp_attrproperty, GetSetProperty
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.pyparser.syntaxtree import TokenNode, SyntaxNode, AbstractSyntaxVisitor
-from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
+from pypy.interpreter.pyparser.pythonparse import make_pyparser
 from pypy.interpreter.pyparser.error import SyntaxError
 from pypy.interpreter.pyparser import grammar, symbol, pytoken
 from pypy.interpreter.argument import Arguments
 
+# backward compat (temp)
+PYTHON_PARSER = make_pyparser()
 
 __all__ = [ "ASTType", "STType", "suite", "expr" ]
 
@@ -43,14 +45,15 @@
 
     def visit_tokennode( self, node ):
         space = self.space
+        tokens = space.parser.tokens
         num = node.name
         lineno = node.lineno
         if node.value is not None:
             val = node.value
         else:
-            if num not in ( pytoken.NEWLINE, pytoken.INDENT,
-                            pytoken.DEDENT, pytoken.ENDMARKER ):
-                val = pytoken.tok_rpunct[num]
+            if num not in ( tokens['NEWLINE'], tokens['INDENT'],
+                            tokens['DEDENT'], tokens['ENDMARKER'] ):
+                val = space.parser.tok_values[num]
             else:
                 val = node.value or ''
         if self.line_info:
@@ -180,7 +183,7 @@
     items = space.unpackiterable( w_sequence )
     nodetype = space.int_w( items[0] )
     is_syntax = True
-    if nodetype>=0 and nodetype<pytoken.N_TOKENS:
+    if nodetype>=0 and nodetype < pytoken.N_TOKENS:
         is_syntax = False
     if is_syntax:
         nodes = []
@@ -202,7 +205,7 @@
 
 def source2ast(space, source):
     from pypy.interpreter.pyparser.pythonutil import AstBuilder, PYTHON_PARSER
-    builder = AstBuilder(space=space)
+    builder = AstBuilder(parser=PYTHON_PARSER, space=space)
     PYTHON_PARSER.parse_source(source, 'file_input', builder)
     ast_tree = builder.rule_stack[-1]
     return space.wrap(ast_tree)

Modified: pypy/branch/ast-experiments/pypy/module/symbol/__init__.py
==============================================================================
--- pypy/branch/ast-experiments/pypy/module/symbol/__init__.py	(original)
+++ pypy/branch/ast-experiments/pypy/module/symbol/__init__.py	Thu Jan 11 10:36:52 2007
@@ -20,10 +20,11 @@
 # Export the values from our custom symbol module.
 # Skip negative values (the corresponding symbols are not visible in
 # pure Python).
-from pypy.interpreter.pyparser.pythonparse import PYTHON_PARSER
+from pypy.interpreter.pyparser.pythonparse import make_pyparser
+parser = make_pyparser()
 
 sym_name = {}
-for name, val in PYTHON_PARSER.symbols.items():
+for name, val in parser.symbols.items():
     if val >= 0:
         Module.interpleveldefs[name] = 'space.wrap(%d)' % val
         sym_name[val] = name



More information about the Pypy-commit mailing list