parse tree has symbols not in the grammar?

[Chad Whitacre]

Hey all,

I've been playing around with the parser module, and based on the 
documentation I would expect all symbols in a parse tree to be part of 
the grammar.[1] For example, I find this line in the symbol module docs:

   Refer to the file Grammar/Grammar in the Python distribution for the
   definitions of the names in the context of the language grammar.[2]

However, the program below gives me a human-readable parse tree (also 
below) that contains symbols that don't seem to be in the grammar, e.g., 
small_stmt, expr_stmt, factor. Is my expectation wrong?

Thanks.



chad

----------

[1] http://python.org/doc/ref/grammar.txt
[2] http://python.org/doc/lib/module-symbol.html



############################
#   A PROGRAM              #
############################

#!/usr/bin/env python
"""print out a human-readable parse tree for a simple code block
"""

import parser, symbol, token
from pprint import pprint

def experiment(block):
     """experimenting, like the man said
     """

     ast = parser.suite(block)
     raw = ast.tolist()

     def interp(x):
         """given an int or a string, return a symbol, token, or other 
string
         """
         if type(x) is type(0):
             if x < 256:
                 return token.tok_name[x]
             else:
                 return symbol.sym_name[x]
         return x

     def process(raw):
         """recursively convert a raw parse tree into something readable
         """
         for node in raw:
             if type(node) is type([]):
                 process(node)
             else:
                 raw[raw.index(node)] = interp(node)

     process(raw)
     pprint(raw)


experiment("""\

x = 3
for y in range(10):
     y == x

""")



############################
#   ITS OUTPUT             #
############################

['file_input',
  ['stmt',
   ['simple_stmt',
    ['small_stmt',
     ['expr_stmt',
      ['testlist',
       ['test',
        ['and_test',
         ['not_test',
          ['comparison',
           ['expr',
            ['xor_expr',
             ['and_expr',
              ['shift_expr',
               ['arith_expr',
                ['term',
                 ['factor', ['power', ['atom', ['NAME', 'x']]]]]]]]]]]]]]],
      ['EQUAL', '='],
      ['testlist',
       ['test',
        ['and_test',
         ['not_test',
          ['comparison',
           ['expr',
            ['xor_expr',
             ['and_expr',
              ['shift_expr',
               ['arith_expr',
                ['term',
                 ['factor',
                  ['power', ['atom', ['NUMBER', '3']]]]]]]]]]]]]]]]],
    ['NEWLINE', '']]],
  ['stmt',
   ['compound_stmt',
    ['for_stmt',
     ['NAME', 'for'],
     ['exprlist',
      ['expr',
       ['xor_expr',
        ['and_expr',
         ['shift_expr',
          ['arith_expr',
           ['term', ['factor', ['power', ['atom', ['NAME', 'y']]]]]]]]]]],
     ['NAME', 'in'],
     ['testlist',
      ['test',
       ['and_test',
        ['not_test',
         ['comparison',
          ['expr',
           ['xor_expr',
            ['and_expr',
             ['shift_expr',
              ['arith_expr',
               ['term',
                ['factor',
                 ['power',
                  ['atom', ['NAME', 'range']],
                  ['trailer',
                   ['LPAR', '('],
                   ['arglist',
                    ['argument',
                     ['test',
                      ['and_test',
                       ['not_test',
                        ['comparison',
                         ['expr',
                          ['xor_expr',
                           ['and_expr',
                            ['shift_expr',
                             ['arith_expr',
                              ['term',
                               ['factor',
                                ['power',
                                 ['atom',
                                  ['NUMBER',
                                   '10']]]]]]]]]]]]]]]],
                   ['RPAR', ')']]]]]]]]]]]]]]],
     ['COLON', ':'],
     ['suite',
      ['NEWLINE', ''],
      ['INDENT', ''],
      ['stmt',
       ['simple_stmt',
        ['small_stmt',
         ['expr_stmt',
          ['testlist',
           ['test',
            ['and_test',
             ['not_test',
              ['comparison',
               ['expr',
                ['xor_expr',
                 ['and_expr',
                  ['shift_expr',
                   ['arith_expr',
                    ['term',
                     ['factor',
                      ['power', ['atom', ['NAME', 'y']]]]]]]]]],
               ['comp_op', ['EQEQUAL', '==']],
               ['expr',
                ['xor_expr',
                 ['and_expr',
                  ['shift_expr',
                   ['arith_expr',
                    ['term',
                     ['factor',
                      ['power', ['atom', ['NAME', 'x']]]]]]]]]]]]]]]]],
        ['NEWLINE', '']]],
      ['DEDENT', '']]]]],
  ['NEWLINE', ''],
  ['ENDMARKER', '']]