[Python-checkins] r86715 - in python/branches/dmalcolm-ast-optimization-branch: Include/Python-ast.h Include/compile.h Include/opcode.h Include/symtable.h Lib/__optimizer__.py Lib/opcode.py Lib/test/test_compile.py Lib/test/test_optimize.py Parser/Python.asdl Parser/asdl_c.py Python/Python-ast.c Python/bltinmodule.c Python/ceval.c Python/compile.c Python/import.c Python/opcode_targets.h Python/pythonrun.c Python/symtable.c
david.malcolm
python-checkins at python.org
Tue Nov 23 20:24:04 CET 2010
Author: david.malcolm
Date: Tue Nov 23 20:24:02 2010
New Revision: 86715
Log:
Commit of work-in-progress, as per py3k-ast-pyoptimize-2010-11-19-006.patch
Added:
python/branches/dmalcolm-ast-optimization-branch/Lib/__optimizer__.py
python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_optimize.py
Modified:
python/branches/dmalcolm-ast-optimization-branch/Include/Python-ast.h
python/branches/dmalcolm-ast-optimization-branch/Include/compile.h
python/branches/dmalcolm-ast-optimization-branch/Include/opcode.h
python/branches/dmalcolm-ast-optimization-branch/Include/symtable.h
python/branches/dmalcolm-ast-optimization-branch/Lib/opcode.py
python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_compile.py
python/branches/dmalcolm-ast-optimization-branch/Parser/Python.asdl
python/branches/dmalcolm-ast-optimization-branch/Parser/asdl_c.py
python/branches/dmalcolm-ast-optimization-branch/Python/Python-ast.c
python/branches/dmalcolm-ast-optimization-branch/Python/bltinmodule.c
python/branches/dmalcolm-ast-optimization-branch/Python/ceval.c
python/branches/dmalcolm-ast-optimization-branch/Python/compile.c
python/branches/dmalcolm-ast-optimization-branch/Python/import.c
python/branches/dmalcolm-ast-optimization-branch/Python/opcode_targets.h
python/branches/dmalcolm-ast-optimization-branch/Python/pythonrun.c
python/branches/dmalcolm-ast-optimization-branch/Python/symtable.c
Modified: python/branches/dmalcolm-ast-optimization-branch/Include/Python-ast.h
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Include/Python-ast.h (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Include/Python-ast.h Tue Nov 23 20:24:02 2010
@@ -2,6 +2,7 @@
#include "asdl.h"
+struct symtable;
typedef struct _mod *mod_ty;
typedef struct _stmt *stmt_ty;
@@ -186,8 +187,8 @@
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
Yield_kind=12, Compare_kind=13, Call_kind=14, Num_kind=15,
Str_kind=16, Bytes_kind=17, Ellipsis_kind=18,
- Attribute_kind=19, Subscript_kind=20, Starred_kind=21,
- Name_kind=22, List_kind=23, Tuple_kind=24};
+ Specialize_kind=19, Attribute_kind=20, Subscript_kind=21,
+ Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
struct _expr {
enum _expr_kind kind;
union {
@@ -279,6 +280,13 @@
} Bytes;
struct {
+ expr_ty name;
+ asdl_seq *specialized_body;
+ expr_ty specialized_result;
+ expr_ty generalized;
+ } Specialize;
+
+ struct {
expr_ty value;
identifier attr;
expr_context_ty ctx;
@@ -505,6 +513,10 @@
expr_ty _Py_Bytes(string s, int lineno, int col_offset, PyArena *arena);
#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
+#define Specialize(a0, a1, a2, a3, a4, a5, a6) _Py_Specialize(a0, a1, a2, a3, a4, a5, a6)
+expr_ty _Py_Specialize(expr_ty name, asdl_seq * specialized_body, expr_ty
+ specialized_result, expr_ty generalized, int lineno, int
+ col_offset, PyArena *arena);
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
lineno, int col_offset, PyArena *arena);
@@ -548,6 +560,6 @@
#define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
-PyObject* PyAST_mod2obj(mod_ty t);
+PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st);
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
int PyAST_Check(PyObject* obj);
Modified: python/branches/dmalcolm-ast-optimization-branch/Include/compile.h
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Include/compile.h (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Include/compile.h Tue Nov 23 20:24:02 2010
@@ -19,6 +19,12 @@
int ff_lineno; /* line number of last future statement */
} PyFutureFeatures;
+enum PyCompilationMode {
+ PyCompilationMode_Exec_Module = 0,
+ PyCompilationMode_Eval_Expression = 1,
+ PyCompilationMode_Single_Interactive = 2,
+};
+
#define FUTURE_NESTED_SCOPES "nested_scopes"
#define FUTURE_GENERATORS "generators"
#define FUTURE_DIVISION "division"
@@ -29,10 +35,14 @@
#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL"
struct _mod; /* Declare the existence of this type */
+PyAPI_FUNC(enum PyCompilationMode) PyAST_CompilationModeFromStartToken(int);
+
PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
- PyCompilerFlags *, PyArena *);
+ PyCompilerFlags *, PyArena *, enum PyCompilationMode);
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+PyAPI_FUNC(void) _PyOptimizer_Init(void);
+
#ifdef __cplusplus
}
Modified: python/branches/dmalcolm-ast-optimization-branch/Include/opcode.h
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Include/opcode.h (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Include/opcode.h Tue Nov 23 20:24:02 2010
@@ -7,117 +7,117 @@
/* Instruction opcodes for compiled code */
-#define STOP_CODE 0
-#define POP_TOP 1
-#define ROT_TWO 2
-#define ROT_THREE 3
-#define DUP_TOP 4
+#define STOP_CODE 0
+#define POP_TOP 1
+#define ROT_TWO 2
+#define ROT_THREE 3
+#define DUP_TOP 4
#define DUP_TOP_TWO 5
-#define NOP 9
+#define NOP 9
-#define UNARY_POSITIVE 10
-#define UNARY_NEGATIVE 11
-#define UNARY_NOT 12
+#define UNARY_POSITIVE 10
+#define UNARY_NEGATIVE 11
+#define UNARY_NOT 12
-#define UNARY_INVERT 15
+#define UNARY_INVERT 15
-#define BINARY_POWER 19
+#define BINARY_POWER 19
-#define BINARY_MULTIPLY 20
+#define BINARY_MULTIPLY 20
-#define BINARY_MODULO 22
-#define BINARY_ADD 23
-#define BINARY_SUBTRACT 24
-#define BINARY_SUBSCR 25
+#define BINARY_MODULO 22
+#define BINARY_ADD 23
+#define BINARY_SUBTRACT 24
+#define BINARY_SUBSCR 25
#define BINARY_FLOOR_DIVIDE 26
#define BINARY_TRUE_DIVIDE 27
#define INPLACE_FLOOR_DIVIDE 28
#define INPLACE_TRUE_DIVIDE 29
-#define STORE_MAP 54
-#define INPLACE_ADD 55
-#define INPLACE_SUBTRACT 56
-#define INPLACE_MULTIPLY 57
-
-#define INPLACE_MODULO 59
-#define STORE_SUBSCR 60
-#define DELETE_SUBSCR 61
-
-#define BINARY_LSHIFT 62
-#define BINARY_RSHIFT 63
-#define BINARY_AND 64
-#define BINARY_XOR 65
-#define BINARY_OR 66
-#define INPLACE_POWER 67
-#define GET_ITER 68
-#define STORE_LOCALS 69
-#define PRINT_EXPR 70
+#define STORE_MAP 54
+#define INPLACE_ADD 55
+#define INPLACE_SUBTRACT 56
+#define INPLACE_MULTIPLY 57
+
+#define INPLACE_MODULO 59
+#define STORE_SUBSCR 60
+#define DELETE_SUBSCR 61
+
+#define BINARY_LSHIFT 62
+#define BINARY_RSHIFT 63
+#define BINARY_AND 64
+#define BINARY_XOR 65
+#define BINARY_OR 66
+#define INPLACE_POWER 67
+#define GET_ITER 68
+#define STORE_LOCALS 69
+#define PRINT_EXPR 70
#define LOAD_BUILD_CLASS 71
-#define INPLACE_LSHIFT 75
-#define INPLACE_RSHIFT 76
-#define INPLACE_AND 77
-#define INPLACE_XOR 78
-#define INPLACE_OR 79
-#define BREAK_LOOP 80
+#define INPLACE_LSHIFT 75
+#define INPLACE_RSHIFT 76
+#define INPLACE_AND 77
+#define INPLACE_XOR 78
+#define INPLACE_OR 79
+#define BREAK_LOOP 80
#define WITH_CLEANUP 81
-#define RETURN_VALUE 83
-#define IMPORT_STAR 84
+#define RETURN_VALUE 83
+#define IMPORT_STAR 84
-#define YIELD_VALUE 86
-#define POP_BLOCK 87
-#define END_FINALLY 88
-#define POP_EXCEPT 89
-
-#define HAVE_ARGUMENT 90 /* Opcodes from here have an argument: */
-
-#define STORE_NAME 90 /* Index in name list */
-#define DELETE_NAME 91 /* "" */
-#define UNPACK_SEQUENCE 92 /* Number of sequence items */
-#define FOR_ITER 93
+#define YIELD_VALUE 86
+#define POP_BLOCK 87
+#define END_FINALLY 88
+#define POP_EXCEPT 89
+
+#define HAVE_ARGUMENT 90 /* Opcodes from here have an argument: */
+
+#define STORE_NAME 90 /* Index in name list */
+#define DELETE_NAME 91 /* "" */
+#define UNPACK_SEQUENCE 92 /* Number of sequence items */
+#define FOR_ITER 93
#define UNPACK_EX 94 /* Num items before variable part +
(Num items after variable part << 8) */
-#define STORE_ATTR 95 /* Index in name list */
-#define DELETE_ATTR 96 /* "" */
-#define STORE_GLOBAL 97 /* "" */
-#define DELETE_GLOBAL 98 /* "" */
-
-#define LOAD_CONST 100 /* Index in const list */
-#define LOAD_NAME 101 /* Index in name list */
-#define BUILD_TUPLE 102 /* Number of tuple items */
-#define BUILD_LIST 103 /* Number of list items */
-#define BUILD_SET 104 /* Number of set items */
-#define BUILD_MAP 105 /* Always zero for now */
-#define LOAD_ATTR 106 /* Index in name list */
-#define COMPARE_OP 107 /* Comparison operator */
-#define IMPORT_NAME 108 /* Index in name list */
-#define IMPORT_FROM 109 /* Index in name list */
-
-#define JUMP_FORWARD 110 /* Number of bytes to skip */
-#define JUMP_IF_FALSE_OR_POP 111 /* Target byte offset from beginning of code */
-#define JUMP_IF_TRUE_OR_POP 112 /* "" */
-#define JUMP_ABSOLUTE 113 /* "" */
-#define POP_JUMP_IF_FALSE 114 /* "" */
-#define POP_JUMP_IF_TRUE 115 /* "" */
-
-#define LOAD_GLOBAL 116 /* Index in name list */
-
-#define CONTINUE_LOOP 119 /* Start of loop (absolute) */
-#define SETUP_LOOP 120 /* Target address (relative) */
-#define SETUP_EXCEPT 121 /* "" */
-#define SETUP_FINALLY 122 /* "" */
-
-#define LOAD_FAST 124 /* Local variable number */
-#define STORE_FAST 125 /* Local variable number */
-#define DELETE_FAST 126 /* Local variable number */
+#define STORE_ATTR 95 /* Index in name list */
+#define DELETE_ATTR 96 /* "" */
+#define STORE_GLOBAL 97 /* "" */
+#define DELETE_GLOBAL 98 /* "" */
+
+#define LOAD_CONST 100 /* Index in const list */
+#define LOAD_NAME 101 /* Index in name list */
+#define BUILD_TUPLE 102 /* Number of tuple items */
+#define BUILD_LIST 103 /* Number of list items */
+#define BUILD_SET 104 /* Number of set items */
+#define BUILD_MAP 105 /* Always zero for now */
+#define LOAD_ATTR 106 /* Index in name list */
+#define COMPARE_OP 107 /* Comparison operator */
+#define IMPORT_NAME 108 /* Index in name list */
+#define IMPORT_FROM 109 /* Index in name list */
+
+#define JUMP_FORWARD 110 /* Number of bytes to skip */
+#define JUMP_IF_FALSE_OR_POP 111 /* Target byte offset from beginning of code */
+#define JUMP_IF_TRUE_OR_POP 112 /* "" */
+#define JUMP_ABSOLUTE 113 /* "" */
+#define POP_JUMP_IF_FALSE 114 /* "" */
+#define POP_JUMP_IF_TRUE 115 /* "" */
+
+#define LOAD_GLOBAL 116 /* Index in name list */
+
+#define CONTINUE_LOOP 119 /* Start of loop (absolute) */
+#define SETUP_LOOP 120 /* Target address (relative) */
+#define SETUP_EXCEPT 121 /* "" */
+#define SETUP_FINALLY 122 /* "" */
+
+#define LOAD_FAST 124 /* Local variable number */
+#define STORE_FAST 125 /* Local variable number */
+#define DELETE_FAST 126 /* Local variable number */
-#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */
+#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */
/* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
-#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */
-#define MAKE_FUNCTION 132 /* #defaults + #kwdefaults<<8 + #annotations<<16 */
-#define BUILD_SLICE 133 /* Number of items */
+#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */
+#define MAKE_FUNCTION 132 /* #defaults + #kwdefaults<<8 + #annotations<<16 */
+#define BUILD_SLICE 133 /* Number of items */
#define MAKE_CLOSURE 134 /* same as MAKE_FUNCTION */
#define LOAD_CLOSURE 135 /* Load free variable from closure */
@@ -127,9 +127,9 @@
/* The next 3 opcodes must be contiguous and satisfy
(CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1 */
-#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_KW 141 /* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_VAR_KW 142 /* #args + (#kwargs<<8) */
+#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */
+#define CALL_FUNCTION_KW 141 /* #args + (#kwargs<<8) */
+#define CALL_FUNCTION_VAR_KW 142 /* #args + (#kwargs<<8) */
#define SETUP_WITH 143
@@ -140,6 +140,8 @@
#define SET_ADD 146
#define MAP_ADD 147
+#define JUMP_IF_SPECIALIZABLE 148 /* abs jump */
+
/* EXCEPT_HANDLER is a special, implicit block type which is created when
entering an except handler. It is not an opcode but we define it here
@@ -149,7 +151,7 @@
enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
- PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
+ PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
Modified: python/branches/dmalcolm-ast-optimization-branch/Include/symtable.h
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Include/symtable.h (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Include/symtable.h Tue Nov 23 20:24:02 2010
@@ -62,6 +62,7 @@
PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
PyFutureFeatures *);
PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
+PyAPI_FUNC(PySTEntryObject *) PySymtable_TryLookup(struct symtable *, void *);
PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
Added: python/branches/dmalcolm-ast-optimization-branch/Lib/__optimizer__.py
==============================================================================
--- (empty file)
+++ python/branches/dmalcolm-ast-optimization-branch/Lib/__optimizer__.py Tue Nov 23 20:24:02 2010
@@ -0,0 +1,747 @@
+import sys
+import ast
+from pprint import pprint
+#print("__optimizer__.py imported!")
+
+# Taken from symtable.h:
+DEF_GLOBAL = 1 # global stmt
+DEF_LOCAL = 2 # assignment in code block
+DEF_PARAM = 2<<1 # formal parameter
+DEF_NONLOCAL = 2<<2 # nonlocal stmt
+USE = 2<<3 # name is used
+DEF_FREE = 2<<4 # name used but not defined in nested block
+DEF_FREE_CLASS = 2<<5 # free variable from class's method
+DEF_IMPORT = 2<<6 # assignment occurred via import
+
+DEF_BOUND = (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
+
+# GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
+# table. GLOBAL is returned from PyST_GetScope() for either of them.
+# It is stored in ste_symbols at bits 12-15.
+
+SCOPE_OFFSET = 11
+SCOPE_MASK = (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL)
+
+LOCAL = 1
+GLOBAL_EXPLICIT = 2
+GLOBAL_IMPLICIT = 3
+FREE = 4
+CELL = 5
+
+
+# Suppress various local-manipulation transforms in the presence of usage
+# of these builtins:
+BUILTINS_THAT_READ_LOCALS = {'locals', 'vars', 'dir'}
+
+def log(msg):
+ if 0:
+ print(msg)
+
+# Analogous to PyST_GetScope:
+def get_scope(ste, name):
+ v = ste.symbols.get(name, 0)
+ if v:
+ return (v >> SCOPE_OFFSET) & SCOPE_MASK
+ else:
+ return 0
+
+def add_local(ste, name):
+ # Add a new local var to an STE
+ assert name not in ste.varnames
+ assert name not in ste.symbols
+ ste.symbols[name] = (LOCAL << SCOPE_OFFSET)
+ ste.varnames.append(name)
+
+def to_dot(t):
+ def _node_to_dot(node, indent):
+ result = ''
+ prefix = ' ' * indent
+ if isinstance(node, ast.AST):
+ if hasattr(node, 'ste'):
+ result += prefix + 'subgraph cluster_%s {\n' % id(node)
+ result += prefix + ' label = "%s"\n;' % node.ste.name
+ result += prefix + ' node%i [label=<%s>];\n' % (id(node), node.__class__.__name__)
+ for name, field in ast.iter_fields(node):
+ if field is not None:
+ result += prefix + ' node%i -> node%i [label="%s"];\n' % (id(node), id(field), name)
+ result += _node_to_dot(field, indent + 2)
+ if hasattr(node, 'ste'):
+ result += prefix + '}\n'
+ elif isinstance(node, list):
+ result += prefix + 'node%i [label=<[]>];\n' % (id(node))
+ for i, item in enumerate(node):
+ result += prefix + 'node%i -> node%i [label="[%i]"];\n' % (id(node), id(item), i)
+ result += _node_to_dot(item, indent)
+ elif node is None:
+ pass
+ else:
+ result += prefix + 'node%i [label=<%s>];\n' % (id(node), repr(node))
+ return result
+
+ result = 'digraph {\n'
+ result += _node_to_dot(t, 1)
+ result += '}'
+ return result
+
+def dot_to_png(dot, filename):
+ from subprocess import Popen, PIPE
+ p = Popen(['/usr/bin/dot',
+ '-T', 'png',
+ '-o', filename],
+ stdin=PIPE)
+ p.communicate(dot.encode('utf-8'))
+ p.wait()
+
+def get_ste_for_path(path):
+ '''
+ Given a list of (node, field, index) triples, obtain a list of
+ symbol table entries
+ '''
+ result = []
+ for node, field, index in path:
+ if hasattr(node, 'ste'):
+ result.append(node.ste)
+ return result
+
+class PathTransformer:
+ """
+ Similar to an ast.NodeTransformer, but passes in a path when visiting a node
+
+ The path is passed in as a list of (node, field, index) triples
+ """
+ def visit(self, node, path=[]):
+ """Visit a node."""
+ method = 'visit_' + node.__class__.__name__
+ visitor = getattr(self, method, self.generic_visit)
+ return visitor(node, path)
+
+ def generic_visit(self, node, path):
+ for field, old_value in ast.iter_fields(node):
+ old_value = getattr(node, field, None)
+ if isinstance(old_value, list):
+ new_values = []
+ for idx, value in enumerate(old_value):
+ if isinstance(value, ast.AST):
+ value = self.visit(value, path + [(node, field, idx)])
+ if value is None:
+ continue
+ elif not isinstance(value, ast.AST):
+ new_values.extend(value)
+ continue
+ new_values.append(value)
+ old_value[:] = new_values
+ elif isinstance(old_value, ast.AST):
+ new_node = self.visit(old_value, path + [(node, field, None)])
+ if new_node is None:
+ delattr(node, field)
+ else:
+ setattr(node, field, new_node)
+ return node
+
+def make_load_name(name, locnode):
+ return ast.copy_location(ast.Name(id=name, ctx=ast.Load()), locnode)
+
+def make_assignment(name, expr, locnode):
+ name_node = ast.copy_location(ast.Name(id=name, ctx=ast.Store()), locnode)
+ return ast.copy_location(ast.Assign(targets=[name_node],
+ value=expr),
+ locnode)
+
+def ast_clone(node):
+ #log('ast_clone', node)
+ if isinstance(node, ast.AST):
+ clone = node.__class__()
+ clone = ast.copy_location(clone, node)
+ for name, value in ast.iter_fields(node):
+ if isinstance(value, ast.AST):
+ cvalue = ast_clone(value)
+ elif isinstance(value, list):
+ cvalue = [ast_clone(el) for el in value]
+ else:
+ cvalue = value
+ setattr(clone, name, cvalue)
+ return clone
+ elif isinstance(node, str):
+ return node
+ else:
+ raise ValueError("Don't know how to clone %r" % node)
+
+class InlineBodyFixups(ast.NodeTransformer):
+ """
+ Fix up the cloned body of a function, for inlining
+ """
+ def __init__(self, varprefix, ste):
+ self.varprefix = varprefix
+ self.ste = ste
+
+ def visit_Name(self, node):
+ # Replace local names with prefixed versions:
+ scope = get_scope(self.ste, node.id)
+ if scope == LOCAL:
+ node.id = self.varprefix + node.id
+ return node
+
+ def visit_Return(self, node):
+ self.generic_visit(node)
+ # replace (the final) return with "__returnval__ = expr":
+ return make_assignment(self.varprefix + "__returnval__", node.value, node)
+
+class FunctionInliner(PathTransformer):
+ def __init__(self, tree, defn):
+ self.tree = tree
+ self.defn = defn
+ assert hasattr(defn, 'ste')
+ self.num_callsites = 0
+ log('ste for body: %r' % defn.ste)
+
+ def visit_Call(self, node, path):
+ # Stop inlining beyond an arbitrary cutoff
+ # (bm_simplecall was exploding):
+ if self.num_callsites > 10:
+ return node
+
+ # Visit children:
+ self.generic_visit(node, path)
+
+ if isinstance(node.func, ast.Attribute):
+ # Don't try to inline method calls yet:
+ return node
+
+ if not isinstance(node.func, ast.Name):
+ # Don't try to inline where the function is a non-trivial
+ # expression e.g. "f()()"
+ return node
+
+ if node.func.id != self.defn.name:
+ return node
+
+ log('Considering call to: %r' % node.func.id)
+ log('Path: %r' % path)
+ log('ste for callsite: %r' % get_ste_for_path(path))
+
+ # We need to find the insertion point for statements:
+ # Walk up the ancestors until you find a statement-holding node:
+ for ancestor in path[::-1]:
+ #log('foo:', ancestor)
+ if isinstance(ancestor[0], (ast.Module, ast.Interactive,
+ ast.FunctionDef, ast.ClassDef,
+ ast.For, ast.With,
+ ast.TryExcept, ast.TryFinally,
+ ast.ExceptHandler)):
+ break
+
+ # Can we inline predicates?
+ if isinstance(ancestor[0], (ast.While)):
+ if ancestor[1] == 'test':
+ # No good place to insert code for an inlined "while"
+ # predicate; bail:
+ return node
+
+ if isinstance(ancestor[0], (ast.If)):
+ if ancestor[1] == 'test':
+ # We may be able to inline a predicate before the "if":
+ continue
+ else:
+ # Inline within the body or the orelse:
+ break
+
+ # Locate innermost scope at callsite:
+ ste = get_ste_for_path(path)[-1]
+
+ #print('Inlining call to: %r within %r' % (node.func.id, ste.name))
+ self.num_callsites += 1
+ log('ancestor: %r' % (ancestor, ))
+
+ assert ancestor[2] is not None
+
+
+ #log(ast.dump(self.defn))
+ varprefix = '__inline_%s_%x__' % (node.func.id, id(node))
+ #log('varprefix: %s' % varprefix)
+
+ # Generate a body of specialized statements that can replace the call:
+ specialized = []
+
+ # Create assignment statements of the form:
+ # __inline__x = expr for x
+ # for each parameter
+ # We will insert before the callsite
+ for formal, actual in zip(self.defn.args.args, node.args):
+ #log('formal: %s' % ast.dump(formal))
+ #log('actual: %s' % ast.dump(actual))
+ # FIXME: ste
+ add_local(ste, varprefix+formal.arg)
+
+ assign = make_assignment(varprefix+formal.arg, actual, node)
+ specialized.append(assign)
+
+ # FIXME: these seem to be being done using LOAD_NAME/STORE_NAME; why isn't it using _FAST?
+ # aha: because they're in module scope, not within a function.
+
+ # Introduce __returnval__; initialize it to None, equivalent to
+ # implicit "return None" if there are no "Return" nodes:
+ returnval = varprefix + "__returnval__"
+ add_local(ste, returnval)
+ # FIXME: this requires "None", how to do this in AST?
+ assign = make_assignment(returnval,
+ make_load_name('None', node), node)
+ # FIXME: this leads to LOAD_NAME None, when it should be LOAD_CONST, surely?
+ specialized.append(assign)
+
+ # Make inline body, generating various statements
+ # ending with:
+ # __inline____returnval = expr
+ inline_body = []
+ fixer = InlineBodyFixups(varprefix, self.defn.ste)
+ for stmt in self.defn.body:
+ assert isinstance(stmt, ast.AST)
+ inline_body.append(fixer.visit(ast_clone(stmt)))
+ #log('inline_body:', inline_body)
+ specialized += inline_body
+
+ #log('Parent: %s' % ast.dump(find_parent(self.tree, node)))
+
+ #seq = getattr(ancestor[0], ancestor[1])
+
+ # Splice the compound statements into place:
+ #seq = seq[:ancestor[2]] + compound + seq[ancestor[2]:] # FIXME
+ #setattr(ancestor[0], ancestor[1], seq)
+
+ #log(seq)
+
+ #log(ast.dump(ancestor[0]))
+
+ # FIXME: need some smarts about the value of the "Specialize":
+ # it's the final Expr within the body
+ specialized_result = ast.copy_location(ast.Name(id=returnval,
+ ctx=ast.Load()),
+ node)
+
+ return ast.copy_location(ast.Specialize(name=node.func,
+ generalized=node,
+ specialized_body=specialized,
+ specialized_result=specialized_result),
+ node)
+
+ # Replace the call with a load from __inline____returnval__
+ return ast.copy_location(ast.Name(id=returnval,
+ ctx=ast.Load()), node)
+
+class NotInlinable(Exception):
+ pass
+
+class CheckInlinableVisitor(PathTransformer):
+ def __init__(self, defn):
+ self.defn = defn
+ self.returns = []
+
+ # Various nodes aren't handlable:
+ def visit_FunctionDef(self, node, path):
+ if node != self.defn:
+ raise NotInlinable()
+ self.generic_visit(node, path)
+ return node
+ def visit_ClassDef(self, node, path):
+ raise NotInlinable()
+ def visit_Yield(self, node, path):
+ raise NotInlinable()
+ def visit_Import(self, node, path):
+ raise NotInlinable()
+ def visit_ImportFrom(self, node, path):
+ raise NotInlinable()
+
+ def visit_Return(self, node, path):
+ self.returns.append(path)
+ return node
+
+def fn_is_inlinable(defn, mod):
+ # Should we inline calls to the given FunctionDef ?
+ assert(isinstance(defn, ast.FunctionDef))
+
+ # Only inline "simple" calling conventions for now:
+ if len(defn.decorator_list) > 0:
+ return False
+
+ if (defn.args.vararg is not None or
+ defn.args.kwarg is not None or
+ defn.args.kwonlyargs != [] or
+ defn.args.defaults != [] or
+ defn.args.kw_defaults != []):
+ return False
+
+ # Don't try to inline generators and other awkward cases:
+ v = CheckInlinableVisitor(defn)
+ try:
+ v.visit(defn)
+ except NotInlinable:
+ return False
+
+ # TODO: restrict to just those functions with only a "return" at
+ # the end (or implicit "return None"), no "return" in awkward places
+ # (but could have other control flow)
+ # Specifically: no returns in places that have successor code
+ # for each return:
+ # FIXME: for now, only inline functions which have a single, final
+ # explicit "return" at the end, or no returns:
+ log('returns of %s: %r' % (defn.name, v.returns))
+ if len(v.returns)>1:
+ return False
+
+ # Single "return"? Then it must be at the top level
+ if len(v.returns) == 1:
+ rpath = v.returns[0]
+ # Must be at toplevel:
+ if len(rpath) != 1:
+ return False
+
+ # Must be at the end of that level
+ if rpath[0][2] != len(defn.body)-1:
+ return False
+
+
+
+ # Don't inline functions that use free or cell vars
+ # (just locals and globals):
+ assert hasattr(defn, 'ste')
+ ste = defn.ste
+ for varname in ste.varnames:
+ scope = get_scope(ste, varname)
+ #log('%r: %r' % (varname, scope))
+ if scope not in {LOCAL, GLOBAL_EXPLICIT, GLOBAL_IMPLICIT}:
+ return False
+
+ # Don't inline functions that use the "locals" or "vars" builtins:
+ if varname in BUILTINS_THAT_READ_LOCALS:
+ return False
+
+ # Don't inline functions with names that get rebound:
+ for node in ast.walk(mod):
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name):
+ if target.id == defn.name:
+ if isinstance(target.ctx, ast.Store):
+ return False
+
+ return True
+
+
+
+def _inline_function_calls(t):
+ # Locate top-level function defs:
+ inlinable_function_defs = {}
+ for s in t.body:
+ if isinstance(s, ast.FunctionDef):
+ if fn_is_inlinable(s, t):
+ inlinable_function_defs[s.name] = s
+
+ # We need to be able to look up functions by name to detect "monkeypatching"
+ # Keep a stored copy of each function as it's built, for later comparison
+ # A better way to do this would be to store the const function somewhere as it's compiled
+ # and then go in and fixup all of the JUMP_IF_SPECIALIZABLE entrypoints
+ # But for now, this will do for a prototype:
+ new_body = []
+ for s in t.body:
+ new_body.append(s)
+ if isinstance(s, ast.FunctionDef):
+ storedname = '__saved__' + s.name
+ if s.name in inlinable_function_defs:
+ assign = ast.copy_location(ast.Assign(targets=[ast.Name(id=storedname, ctx=ast.Store())],
+ value=ast.Name(id=s.name, ctx=ast.Load())),
+ s)
+ ast.fix_missing_locations(assign)
+ new_body.append(assign)
+ t.body = new_body
+
+ #print('inlinable_function_defs:%r' % inlinable_function_defs)
+
+ # Locate call sites:
+ for name in inlinable_function_defs:
+ log('inlining calls to %r' % name)
+ inliner = FunctionInliner(t, inlinable_function_defs[name])
+ inliner.visit(t)
+
+ return t
+
+
+def is_local_name(ste, expr):
+ assert isinstance(expr, ast.AST)
+ if isinstance(expr, ast.Name):
+ if get_scope(ste, expr.id) == LOCAL:
+ return True
+
+def is_write_to_local(ste, expr):
+ assert isinstance(expr, ast.AST)
+ if is_local_name(ste, expr):
+ if isinstance(expr.ctx, ast.Store):
+ return True
+
+def is_read_from_local(ste, expr):
+ assert isinstance(expr, ast.AST)
+ if is_local_name(ste, expr):
+ if isinstance(expr.ctx, ast.Load):
+ return True
+
+def is_constant(expr):
+ assert isinstance(expr, ast.AST)
+ if isinstance(expr, (ast.Num, ast.Str, ast.Bytes)):
+ return True
+
+
+# The following optimizations currently can only cope with functions with a
+# single basic-block, to avoid the need to build a CFG and do real data flow
+# analysis.
+
+class MoreThanOneBasicBlock(Exception):
+ pass
+
+class LocalAssignmentWalker(ast.NodeTransformer):
+ def __init__(self, funcdef):
+ self.funcdef = funcdef
+ self.local_values = {}
+
+ def log(self, msg):
+ if 0:
+ print(msg)
+
+ def _is_local(self, varname):
+ ste = self.funcdef.ste
+ return get_scope(ste, varname) == LOCAL
+
+ def _is_propagatable(self, expr):
+ # Is this expression propagatable to successive store operations?
+
+ # We can propagate reads of locals (until they are written to):
+ if is_read_from_local(self.funcdef.ste, expr):
+ return True
+
+ if is_constant(expr):
+ return True
+
+ return False
+
+ def visit_Assign(self, assign):
+ self.generic_visit(assign)
+ self.log(' got assign: %s' % ast.dump(assign))
+
+ # Keep track of assignments to locals that are directly of constants
+ # or of other other locals:
+ for target in assign.targets:
+ if is_write_to_local(self.funcdef.ste, target):
+ self.log(' write to %r <- %s' % (target.id, ast.dump(assign.value)))
+ if len(assign.targets) == 1:
+ target = assign.targets[0]
+ if self._is_propagatable(assign.value):
+ self.log(' recording value for %r: %s' % (target.id, ast.dump(assign.value)))
+ self.local_values[target.id] = assign.value
+ continue
+ self.log(' %r is no longer known' % target.id)
+ self.local_values[target.id] = None
+
+ # Propagate earlier copies to this assignment:
+ if len(assign.targets) == 1:
+ target = assign.targets[0]
+ if is_write_to_local(self.funcdef.ste, target):
+ if target.id in self.local_values:
+ value = self.local_values[target.id]
+ if value is not None:
+ pass
+ #self.log(' copy-propagation target')
+
+ return assign
+
+ def visit_Name(self, name):
+ self.log('visit_Name %r' % name)
+ self.generic_visit(name)
+ if is_read_from_local(self.funcdef.ste, name):
+ self.log(' got read from local: %s' % ast.dump(name))
+ if name.id in self.local_values:
+ value = self.local_values[name.id]
+ if value is not None:
+ self.log(' copy-propagating: %r <- %s' % (name.id, ast.dump(value)))
+ return value # clone this?
+ return name
+
+ # Nodes implying branching and looping:
+ def visit_For(self, node):
+ raise MoreThanOneBasicBlock()
+ def visit_While(self, node):
+ raise MoreThanOneBasicBlock()
+ def visit_If(self, node):
+ raise MoreThanOneBasicBlock()
+ def visit_With(self, node):
+ raise MoreThanOneBasicBlock()
+ def visit_TryExcept(self, node):
+ raise MoreThanOneBasicBlock()
+ def visit_TryFinally(self, node):
+ raise MoreThanOneBasicBlock()
+ def visit_IfExp(self, node):
+ raise MoreThanOneBasicBlock()
+
+
+class CopyPropagation(ast.NodeTransformer):
+
+ def visit_FunctionDef(self, funcdef):
+ log('CopyPropagation: got function def: %r' % funcdef.name)
+ self.generic_visit(funcdef)
+
+ try:
+ w = LocalAssignmentWalker(funcdef)
+ w.visit(funcdef)
+ except MoreThanOneBasicBlock:
+ pass
+
+ return funcdef
+
+
+def _copy_propagation(t):
+ # Very simple copy propagation, which (for simplicity), requires that we
+ # have a single basic block
+ v = CopyPropagation()
+ v.visit(t)
+ return t
+
+
+
+
+class ReferenceToLocalFinder(PathTransformer):
+ # Gather all reads/writes of locals within the given FunctionDef
+ def __init__(self, funcdef):
+ assert isinstance(funcdef, ast.FunctionDef)
+ self.funcdef = funcdef
+ # varnames of locals:
+ self.local_reads = set()
+ self.local_writes = set()
+ self.globals = set()
+
+ def log(self, msg):
+ if 0:
+ print(msg)
+
+ def visit_Name(self, node, path):
+ scope = get_scope(self.funcdef.ste, node.id)
+ if scope == LOCAL:
+ self.log(' found local: %r %r' % (ast.dump(node), path))
+ if isinstance(node.ctx, ast.Store):
+ self.local_writes.add(node.id)
+ elif isinstance(node.ctx, ast.Load):
+ self.local_reads.add(node.id)
+ else:
+ assert isinstance(node.ctx, ast.Del) # FIXME: what about other cases?
+ elif scope in {GLOBAL_EXPLICIT, GLOBAL_IMPLICIT}:
+ self.globals.add(node.id)
+ return node
+
+ def visit_AugAssign(self, augassign, path):
+ # VAR += EXPR references VAR, and we can't remove it since it could
+ # have arbitrary sideeffects
+ if isinstance(augassign.target, ast.Name):
+ target = augassign.target
+ scope = get_scope(self.funcdef.ste, target.id)
+ if scope == LOCAL:
+ self.log(' found local: %r %r' % (ast.dump(augassign), path))
+ # An augassign is both a read and a write:
+ self.local_writes.add(target.id)
+ self.local_reads.add(target.id)
+ self.generic_visit(augassign, path)
+ return augassign
+
+
+class RemoveAssignmentToUnusedLocals(ast.NodeTransformer):
+ # Replace all Assign([local], expr) with Expr(expr) for the given locals
+ def __init__(self, varnames):
+ self.varnames = varnames
+
+ def visit_Assign(self, node):
+ if len(node.targets) == 1:
+ if isinstance(node.targets[0], ast.Name):
+ if node.targets[0].id in self.varnames:
+ if isinstance(node.targets[0].ctx, ast.Store):
+ # Eliminate the assignment
+ return ast.copy_location(ast.Expr(node.value),
+ node.value)
+ # FIXME: also eliminate the variable from symtable
+ return node
+
+
+
+class RedundantLocalRemover(PathTransformer):
+
+ def log(self, msg):
+ if 0:
+ print(msg)
+
+ def visit_FunctionDef(self, funcdef, path):
+ self.log('got function def: %r' % funcdef.name)
+ v = ReferenceToLocalFinder(funcdef)
+ v.visit(funcdef, path)
+ self.generic_visit(funcdef, path)
+
+ # Don't ellide locals if this function references the builtins
+ # that access them:
+ if v.globals.intersection(BUILTINS_THAT_READ_LOCALS):
+ return funcdef
+
+ unused_writes = v.local_writes - v.local_reads
+ if unused_writes:
+ self.log(' globals: %s' % v.globals)
+ self.log(' loaded from: %s' % v.local_reads)
+ self.log(' stored to: %s:' % v.local_writes)
+ self.log(' unused: %s' % unused_writes)
+ v = RemoveAssignmentToUnusedLocals(unused_writes)
+ v.visit(funcdef)
+ return funcdef
+
+def _remove_redundant_locals(t):
+ v = RedundantLocalRemover()
+ v.visit(t)
+ return t
+
+
+
+
+# For now restrict ourselves to just a few places:
+def is_test_code(t, filename):
+ if filename.endswith('test_optimize.py'):
+ return True
+ for n in ast.walk(t):
+ if isinstance(n, ast.FunctionDef):
+ if n.name == 'function_to_be_inlined':
+ return True
+ return False
+
+#class OptimizationError(Exception):
+# def __init__(self
+
+from pprint import pprint
+def optimize_ast(t, filename, st_blocks):
+ if 0:
+ print("optimize_ast called: %s" % filename)
+ if is_test_code(t, filename):
+ dot_before = to_dot(t)
+ try:
+ # pprint(st_blocks)
+ # log(t)
+ # pprint(t)
+ # log(ast.dump(t))
+
+ #dot_to_png(to_dot(t), 'before.png')
+
+ if isinstance(t, ast.Module):
+ t = _inline_function_calls(t)
+ #cfg = CFG.from_ast(t)
+ #print(cfg.to_dot())
+ #dot_to_png(cfg.to_dot(), 'cfg.png')
+
+ t = _copy_propagation(t)
+
+ t = _remove_redundant_locals(t)
+
+ #dot_to_png(to_dot(t), 'after.png')
+
+ except:
+ print('Exception during optimization of %r' % filename)
+ # dot_to_png(dot_before, 'before.png')
+ raise
+ #print('finished optimizing')
+ return t
Modified: python/branches/dmalcolm-ast-optimization-branch/Lib/opcode.py
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Lib/opcode.py (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Lib/opcode.py Tue Nov 23 20:24:02 2010
@@ -174,6 +174,8 @@
def_op('SET_ADD', 146)
def_op('MAP_ADD', 147)
+jabs_op('JUMP_IF_SPECIALIZABLE', 148)
+
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144
Modified: python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_compile.py
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_compile.py (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_compile.py Tue Nov 23 20:24:02 2010
@@ -433,6 +433,13 @@
ast.body = [_ast.BoolOp()]
self.assertRaises(TypeError, compile, ast, '<ast>', 'exec')
+ # raise exception when expr is not an expression:
+ self.assertRaises(TypeError, compile,
+ _ast.Module(body=[_ast.Expr(_ast.Assign(lineno=0,
+ col_offset=0),
+ lineno=0, col_offset=0)],
+ lineno=0),
+ 'test', 'exec')
def test_main():
support.run_unittest(TestSpecifics)
Added: python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_optimize.py
==============================================================================
--- (empty file)
+++ python/branches/dmalcolm-ast-optimization-branch/Lib/test/test_optimize.py Tue Nov 23 20:24:02 2010
@@ -0,0 +1,426 @@
+import dis
+import re
+import sys
+from io import StringIO
+import unittest
+import ast
+import types
+
+def disassemble(func):
+ f = StringIO()
+ tmp = sys.stdout
+ sys.stdout = f
+ dis.dis(func)
+ sys.stdout = tmp
+ result = f.getvalue()
+ f.close()
+ return result
+
+def dis_single(line):
+ return disassemble(compile(line, '', 'single'))
+
+def compile_fn(src, fnname='f'):
+ """
+ Compile src to a Module, return an (ast.FunctionDef, ast.Module) pair
+ where the former is the named function
+ """
+ node = ast.parse(src)
+ #print(ast.dump(node))
+ for f in node.body:
+ if isinstance(f, ast.FunctionDef):
+ if f.name == fnname:
+ return f, node
+ raise ValueError('Could not find function: %r' % fnname)
+
+def get_code_for_fn(co, fnname):
+ for const in co.co_consts:
+ if isinstance(const, types.CodeType):
+ if const.co_name == fnname:
+ return const
+ raise ValueError('code for %r not found' % fnname)
+
+
+class TestFramework(unittest.TestCase):
+ # Ensure that invoking the optimizer is working:
+ def test_eval(self):
+ self.assertEqual(eval('42'), 42)
+
+class TestInlining(unittest.TestCase):
+ def assertIsInlinable(self, src, fnname='f'):
+ from __optimizer__ import fn_is_inlinable
+ fn, mod = compile_fn(src)
+ self.assert_(fn_is_inlinable(fn, mod),
+ msg='Unexpectedly not inlinable: %s\n%r' % (src, ast.dump(fn)))
+
+ def assertIsNotInlinable(self, src, fnname='f'):
+ from __optimizer__ import fn_is_inlinable
+ fn, mod = compile_fn(src)
+ self.assert_(not fn_is_inlinable(fn, mod),
+ msg='Unexpectedly inlinable: %s\n%r' % (src, ast.dump(fn)))
+
+ def compile_to_code(self, src, fnname):
+ # Compile the given source code, returning the code object for the given function name
+ co = compile(src, 'test_optimize.py', 'exec')
+ return get_code_for_fn(co, fnname)
+
+ def test_simple_inlining(self):
+ src = (
+'''
+def function_to_be_inlined(x, y, z):
+ # this currently needs to exist to enable the inliner
+ pass
+sep = '-'
+def f(x, y, z):
+ return (2 * x * y) + (sep * z)
+def callsite():
+ f('foo', 3, 16)
+''')
+ self.assertIsInlinable(src)
+ callsite = self.compile_to_code(src, 'callsite')
+ asm = disassemble(callsite)
+ self.assertIn('JUMP_IF_SPECIALIZABLE', asm)
+ self.assertIn("('foofoofoofoofoofoo')", asm)
+ self.assertIn("(sep)", asm)
+
+
+
+ def test_copy_propagation(self):
+ src = (
+'''
+def function_to_be_inlined(a, b, c, d):
+ return(a + b + c + d)
+def callsite():
+ print(function_to_be_inlined('fee', 'fi', 'fo', 'fum'))
+''')
+ callsite = self.compile_to_code(src, 'callsite')
+ asm = disassemble(callsite)
+ # print(asm)
+ self.assertIn('JUMP_IF_SPECIALIZABLE', asm)
+ self.assertIn("('feefifofum')", asm)
+
+ def test_keyword_args(self):
+ src = (
+'''
+def f(x, y, z=42):
+ return 42
+''')
+ self.assertIsNotInlinable(src)
+
+ def test_star_args(self):
+ src = ('''
+def f(x, y, *args):
+ return [x, y] + args
+''')
+ self.assertIsNotInlinable(src)
+
+
+ def test_kwargs(self):
+ src = (
+'''
+def f(x, y, **kwargs):
+ return kwargs
+''')
+ self.assertIsNotInlinable(src)
+
+ def test_decorators(self):
+ src = (
+'''
+ at login_required
+def f(x, y, z):
+ return 42
+''')
+ self.assertIsNotInlinable(src)
+
+ def test_imports(self):
+ src = (
+'''
+def f(x, y, z):
+ import os
+ return 42
+''')
+ self.assertIsNotInlinable(src)
+
+ def test_generators(self):
+ src = (
+'''
+def f(x, y, z):
+ yield 42
+''')
+ self.assertIsNotInlinable(src)
+
+ def test_complex_return_logic(self):
+ # Only simple use of "return" for now
+ src = (
+'''
+def f(x, y, z):
+ if x:
+ return 42
+''')
+ self.assertIsNotInlinable(src)
+
+
+ def test_free_and_cell_variables(self):
+ # No free/cell variables
+ src = (
+'''
+def f(x, y, z):
+ def g(p, q):
+ return x*p + y*q + z
+ return g
+''')
+ self.assertIsNotInlinable(src)
+
+ def test_reassignment_of_function_name(self):
+ src = (
+'''
+def f(x):
+ return 42
+f = len
+''')
+ self.assertIsNotInlinable(src)
+
+
+
+ def not_test_simple(self):
+ src = '''
+my_global = 34
+
+def function_to_be_inlined(x, y, z):
+ return (2 * x * y) + z + my_global
+print(function_to_be_inlined(3, 4, 5))
+'''
+ asm = disassemble(src)
+ print(asm)
+ # FIXME: verify that the inlined callsite actually works!
+
+ def test_double(self):
+ src = '''
+def function_to_be_inlined():
+ pass
+def f(x, y, z):
+ return (2 * x * y) + z
+def g(x, y, z):
+ return (f(x, y, 3 * z)
+ - f(x + 1, y * 5, z -2))
+def h(x, y):
+ return g(x, y, 0)
+'''
+ co = compile(src, 'test_optimize.py', 'exec')
+ g = get_code_for_fn(co, 'g')
+ h = get_code_for_fn(co, 'h')
+ g_asm = disassemble(g)
+ h_asm = disassemble(h)
+ #print(disassemble(co))
+ #print('\n\ng')
+ #print(g_asm)
+ self.assertIn('JUMP_IF_SPECIALIZABLE', g_asm)
+ #print('\n\nh')
+ #print(h_asm)
+ self.assertIn('JUMP_IF_SPECIALIZABLE', h_asm)
+
+ def test_ignore_implicit_return(self):
+ src = '''
+my_global = 34
+def function_to_be_inlined(x, y, z):
+ print(2 * x * y) + z + my_global
+def callsite():
+ function_to_be_inlined(3, 4, 5)
+'''
+ callsite = self.compile_to_code(src, 'callsite')
+ asm = disassemble(callsite)
+ #print(asm)
+ # FIXME: actually test it
+
+ def test_call_call(self):
+ # seen in copy.py:
+ src = '''
+def _copy_with_constructor(x):
+ return type(x)(x)
+'''
+ fn = self.compile_to_code(src, '_copy_with_constructor')
+ asm = disassemble(fn)
+
+ def test_global(self):
+ # Taken from tempfile.py:
+ src = '''
+tempdir = None
+
+def gettempdir():
+ """Accessor for tempfile.tempdir."""
+ global tempdir
+ if tempdir is None:
+ _once_lock.acquire()
+ try:
+ if tempdir is None:
+ tempdir = _get_default_tempdir()
+ finally:
+ _once_lock.release()
+ return tempdir
+
+def callsite():
+ return gettempdir()
+'''
+ fn = self.compile_to_code(src, 'callsite')
+ asm = disassemble(fn)
+ #print(asm)
+ self.assertIn('JUMP_IF_SPECIALIZABLE', asm)
+ self.assertIn('(tempdir)', asm)
+
+ def test_del_local(self):
+ # Adapted from tempfile.py:
+ src = (
+'''
+def f():
+ fd = _os.open(filename, _bin_openflags, 0o600)
+ fp = _io.open(fd, 'wb')
+ fp.write(b'blat')
+ fp.close()
+ _os.unlink(filename)
+ del fp, fd
+ return dir
+''')
+ fn = self.compile_to_code(src, 'f')
+ asm = disassemble(fn)
+
+ def test_inplace(self):
+ src = (
+'''
+def inplace(self):
+ a = foo()
+ a *= 42
+''')
+ fn = self.compile_to_code(src, 'inplace')
+ asm = disassemble(fn)
+ # FIXME: Ensure that the initial write to "a" isn't optimized away:
+ #print(asm)
+
+ def test_use_implicit_return(self):
+ pass # TODO
+
+ def test_predicates(self):
+ src = '''
+def function_to_be_inlined():
+ pass
+
+def is_crunchy(path):
+ return True
+
+def find_file(filename, std_dirs, paths):
+ for dir in std_dirs:
+ if is_crunchy(dir):
+ f = os.path.join(sysroot, dir[1:], filename)
+ return None
+'''
+ fn = self.compile_to_code(src, 'find_file')
+ asm = disassemble(fn)
+ self.assertIn('JUMP_IF_SPECIALIZABLE', asm)
+
+ def test_useless_args(self):
+ src = '''
+def function_to_be_inlined():
+ pass
+
+def foo(a, b, c):
+ bar(a, b)
+
+def bar(a, b):
+ baz(a)
+
+def baz(a):
+ pass
+
+foo(1, 2, 3)
+'''
+ asm = disassemble(src)
+ #print(asm)
+ # FIXME
+
+ def test_simple_recursion(self):
+ src = '''
+def function_to_be_inlined():
+ return function_to_be_inlined()
+'''
+ fn = self.compile_to_code(src, 'function_to_be_inlined')
+ asm = disassemble(fn)
+ #print(asm)
+
+ def test_mutual_recursion(self):
+ src = '''
+def function_to_be_inlined():
+ return function_to_be_inlined()
+
+def f():
+ return g()
+
+def g():
+ return f()
+'''
+ fn = self.compile_to_code(src, 'f')
+ asm = disassemble(fn)
+ #print(asm)
+
+
+
+def function_to_be_inlined():
+ return 'I am the original implementation'
+
+def new_version():
+ return 'I am the new version'
+
+def call_inlinable_function():
+ return function_to_be_inlined()
+
+class TestRebinding(unittest.TestCase):
+
+ def test_rebinding(self):
+ # "call_inlinable_function" should have an inlined copy
+ # of "function_to_be_inlined":
+ asm = disassemble(call_inlinable_function)
+ #print(asm)
+ # Should have logic for detecting if it can specialize:
+ self.assertIn('JUMP_IF_SPECIALIZABLE', asm)
+ self.assertIn('(__saved__function_to_be_inlined)', asm)
+ # Should have inlined constant value:
+ self.assertIn("('I am the original implementation')", asm)
+
+ # Try calling it:
+ self.assertEquals(call_inlinable_function(),
+ 'I am the original implementation')
+
+ # Now rebind the wrapped function.
+ # The easy way to do this:
+ # function_to_be_inlined = new_version
+ # doesn't work: the optimizer spots it, and turns off inlining
+ # for "function_to_be_inlined" throughout this module
+ # Similarly, using "globals()" directly is spotted.
+
+ __builtins__.globals()['function_to_be_inlined'] = new_version
+
+ # Verify that we get the correct result:
+ self.assertEquals(call_inlinable_function(),
+ 'I am the new version')
+ # FIXME; should also check bytecode instrumentation to verify that we
+ # took the generalized implementation
+
+
+
+
+def test_main(verbose=None):
+ import sys
+ from test import support
+ test_classes = (TestFramework, TestInlining, TestRebinding)
+ support.run_unittest(*test_classes)
+
+ # verify reference counting
+ if verbose and hasattr(sys, "gettotalrefcount"):
+ import gc
+ counts = [None] * 5
+ for i in range(len(counts)):
+ support.run_unittest(*test_classes)
+ gc.collect()
+ counts[i] = sys.gettotalrefcount()
+ print(counts)
+
+if __name__ == "__main__":
+ test_main(verbose=False)
+ pass
Modified: python/branches/dmalcolm-ast-optimization-branch/Parser/Python.asdl
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Parser/Python.asdl (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Parser/Python.asdl Tue Nov 23 20:24:02 2010
@@ -72,6 +72,12 @@
| Ellipsis
-- other literals? bools?
+ -- Generated by the optimizer:
+ | Specialize(expr name,
+ stmt* specialized_body,
+ expr specialized_result,
+ expr generalized) -- must be a Call
+
-- the following expression can appear in assignment context
| Attribute(expr value, identifier attr, expr_context ctx)
| Subscript(expr value, slice slice, expr_context ctx)
Modified: python/branches/dmalcolm-ast-optimization-branch/Parser/asdl_c.py
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Parser/asdl_c.py (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Parser/asdl_c.py Tue Nov 23 20:24:02 2010
@@ -550,7 +550,7 @@
def visitProduct(self, prod, name):
self.emit("static PyTypeObject *%s_type;" % name, 0)
- self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
+ self.emit("static PyObject* ast2obj_%s(void*, struct symtable *st);" % name, 0)
if prod.fields:
self.emit("static char *%s_fields[]={" % name,0)
for f in prod.fields:
@@ -572,7 +572,7 @@
tnames.append(str(t.name)+"_singleton")
tnames = ", *".join(tnames)
self.emit("static PyObject *%s;" % tnames, 0)
- self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
+ self.emit("static PyObject* ast2obj_%s(%s, struct symtable *st);" % (name, ptype), 0)
for t in sum.types:
self.visitConstructor(t, name)
@@ -748,7 +748,7 @@
/* Conversion AST -> Python */
-static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
+static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*, struct symtable*), struct symtable *st)
{
int i, n = asdl_seq_LEN(seq);
PyObject *result = PyList_New(n);
@@ -756,7 +756,7 @@
if (!result)
return NULL;
for (i = 0; i < n; i++) {
- value = func(asdl_seq_GET(seq, i));
+ value = func(asdl_seq_GET(seq, i), st);
if (!value) {
Py_DECREF(result);
return NULL;
@@ -766,7 +766,7 @@
return result;
}
-static PyObject* ast2obj_object(void *o)
+static PyObject* ast2obj_object(void *o, struct symtable *st)
{
if (!o)
o = Py_None;
@@ -776,7 +776,7 @@
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
-static PyObject* ast2obj_int(long b)
+static PyObject* ast2obj_int(long b, struct symtable *st)
{
return PyLong_FromLong(b);
}
@@ -804,7 +804,7 @@
PyObject *s = PyObject_Repr(obj);
if (s == NULL) return 1;
PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s",
- PyBytes_AS_STRING(s));
+ _PyUnicode_AsString(s));
Py_DECREF(s);
return 1;
}
@@ -957,7 +957,7 @@
def func_begin(self, name):
ctype = get_c_type(name)
self.emit("PyObject*", 0)
- self.emit("ast2obj_%s(void* _o)" % (name), 0)
+ self.emit("ast2obj_%s(void* _o, struct symtable *st)" % (name), 0)
self.emit("{", 0)
self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
self.emit("PyObject *result = NULL, *value = NULL;", 1)
@@ -987,7 +987,7 @@
self.visitConstructor(t, i + 1, name)
self.emit("}", 1)
for a in sum.attributes:
- self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
+ self.emit("value = ast2obj_%s(o->%s, st);" % (a.type, a.name), 1)
self.emit("if (!value) goto failed;", 1)
self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1)
self.emit('goto failed;', 2)
@@ -995,7 +995,7 @@
self.func_end()
def simpleSum(self, sum, name):
- self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0)
+ self.emit("PyObject* ast2obj_%s(%s_ty o, struct symtable *st)" % (name, name), 0)
self.emit("{", 0)
self.emit("switch(o) {", 1)
for t in sum.types:
@@ -1014,6 +1014,8 @@
self.func_begin(name)
self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1);
self.emit("if (!result) return NULL;", 1)
+ self.emit("if (!obj2ast_write_scope(o, result, st))", 1)
+ self.emit("goto failed;", 2)
for field in prod.fields:
self.visitField(field, name, 1, True)
self.func_end()
@@ -1022,6 +1024,8 @@
self.emit("case %s_kind:" % cons.name, 1)
self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2);
self.emit("if (!result) goto failed;", 2)
+ self.emit("if (!obj2ast_write_scope(o, result, st))", 2)
+ self.emit("goto failed;", 3)
for f in cons.fields:
self.visitField(f, cons.name, 2, False)
self.emit("break;", 2)
@@ -1063,23 +1067,37 @@
self.emit("if (!value) goto failed;", depth+1)
self.emit("for(i = 0; i < n; i++)", depth+1)
# This cannot fail, so no need for error handling
- self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
+ self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i), st));" % value,
depth+2, reflow=False)
self.emit("}", depth)
else:
- self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
+ self.emit("value = ast2obj_list(%s, ast2obj_%s, st);" % (value, field.type), depth)
else:
ctype = get_c_type(field.type)
- self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False)
+ self.emit("value = ast2obj_%s(%s, st);" % (field.type, value), depth, reflow=False)
class PartingShots(StaticVisitor):
CODE = """
-PyObject* PyAST_mod2obj(mod_ty t)
+int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st)
+{
+ PySTEntryObject * ste;
+ if (st) {
+ ste = PySymtable_TryLookup(st, ast_obj);
+ if (ste) {
+ /* This AST node has a symtable entry: */
+ if (PyObject_SetAttrString(py_obj, "ste", (PyObject*)ste) == -1)
+ return 0;
+ }
+ }
+ return 1;
+}
+
+PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st)
{
init_types();
- return ast2obj_mod(t);
+ return ast2obj_mod(t, st);
}
/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
@@ -1150,12 +1168,13 @@
f = open(p, "w")
f.write(auto_gen_msg)
f.write('#include "asdl.h"\n\n')
+ f.write("struct symtable;\n")
c = ChainOfVisitors(TypeDefVisitor(f),
StructVisitor(f),
PrototypeVisitor(f),
)
c.visit(mod)
- f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
+ f.write("PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st);\n")
f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
f.write("int PyAST_Check(PyObject* obj);\n")
f.close()
@@ -1167,8 +1186,10 @@
f.write(c_file_msg % parse_version(mod))
f.write('#include "Python.h"\n')
f.write('#include "%s-ast.h"\n' % mod.name)
+ f.write('#include "symtable.h"\n')
f.write('\n')
f.write("static PyTypeObject AST_type;\n")
+ f.write("int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st);\n");
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/Python-ast.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/Python-ast.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/Python-ast.c Tue Nov 23 20:24:02 2010
@@ -11,10 +11,12 @@
#include "Python.h"
#include "Python-ast.h"
+#include "symtable.h"
static PyTypeObject AST_type;
+int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st);
static PyTypeObject *mod_type;
-static PyObject* ast2obj_mod(void*);
+static PyObject* ast2obj_mod(void*, struct symtable *st);
static PyTypeObject *Module_type;
static char *Module_fields[]={
"body",
@@ -36,7 +38,7 @@
"lineno",
"col_offset",
};
-static PyObject* ast2obj_stmt(void*);
+static PyObject* ast2obj_stmt(void*, struct symtable *st);
static PyTypeObject *FunctionDef_type;
static char *FunctionDef_fields[]={
"name",
@@ -150,7 +152,7 @@
"lineno",
"col_offset",
};
-static PyObject* ast2obj_expr(void*);
+static PyObject* ast2obj_expr(void*, struct symtable *st);
static PyTypeObject *BoolOp_type;
static char *BoolOp_fields[]={
"op",
@@ -239,6 +241,13 @@
"s",
};
static PyTypeObject *Ellipsis_type;
+static PyTypeObject *Specialize_type;
+static char *Specialize_fields[]={
+ "name",
+ "specialized_body",
+ "specialized_result",
+ "generalized",
+};
static PyTypeObject *Attribute_type;
static char *Attribute_fields[]={
"value",
@@ -274,7 +283,7 @@
static PyTypeObject *expr_context_type;
static PyObject *Load_singleton, *Store_singleton, *Del_singleton,
*AugLoad_singleton, *AugStore_singleton, *Param_singleton;
-static PyObject* ast2obj_expr_context(expr_context_ty);
+static PyObject* ast2obj_expr_context(expr_context_ty, struct symtable *st);
static PyTypeObject *Load_type;
static PyTypeObject *Store_type;
static PyTypeObject *Del_type;
@@ -282,7 +291,7 @@
static PyTypeObject *AugStore_type;
static PyTypeObject *Param_type;
static PyTypeObject *slice_type;
-static PyObject* ast2obj_slice(void*);
+static PyObject* ast2obj_slice(void*, struct symtable *st);
static PyTypeObject *Slice_type;
static char *Slice_fields[]={
"lower",
@@ -299,7 +308,7 @@
};
static PyTypeObject *boolop_type;
static PyObject *And_singleton, *Or_singleton;
-static PyObject* ast2obj_boolop(boolop_ty);
+static PyObject* ast2obj_boolop(boolop_ty, struct symtable *st);
static PyTypeObject *And_type;
static PyTypeObject *Or_type;
static PyTypeObject *operator_type;
@@ -307,7 +316,7 @@
*Div_singleton, *Mod_singleton, *Pow_singleton, *LShift_singleton,
*RShift_singleton, *BitOr_singleton, *BitXor_singleton, *BitAnd_singleton,
*FloorDiv_singleton;
-static PyObject* ast2obj_operator(operator_ty);
+static PyObject* ast2obj_operator(operator_ty, struct symtable *st);
static PyTypeObject *Add_type;
static PyTypeObject *Sub_type;
static PyTypeObject *Mult_type;
@@ -323,7 +332,7 @@
static PyTypeObject *unaryop_type;
static PyObject *Invert_singleton, *Not_singleton, *UAdd_singleton,
*USub_singleton;
-static PyObject* ast2obj_unaryop(unaryop_ty);
+static PyObject* ast2obj_unaryop(unaryop_ty, struct symtable *st);
static PyTypeObject *Invert_type;
static PyTypeObject *Not_type;
static PyTypeObject *UAdd_type;
@@ -332,7 +341,7 @@
static PyObject *Eq_singleton, *NotEq_singleton, *Lt_singleton, *LtE_singleton,
*Gt_singleton, *GtE_singleton, *Is_singleton, *IsNot_singleton, *In_singleton,
*NotIn_singleton;
-static PyObject* ast2obj_cmpop(cmpop_ty);
+static PyObject* ast2obj_cmpop(cmpop_ty, struct symtable *st);
static PyTypeObject *Eq_type;
static PyTypeObject *NotEq_type;
static PyTypeObject *Lt_type;
@@ -344,7 +353,7 @@
static PyTypeObject *In_type;
static PyTypeObject *NotIn_type;
static PyTypeObject *comprehension_type;
-static PyObject* ast2obj_comprehension(void*);
+static PyObject* ast2obj_comprehension(void*, struct symtable *st);
static char *comprehension_fields[]={
"target",
"iter",
@@ -355,7 +364,7 @@
"lineno",
"col_offset",
};
-static PyObject* ast2obj_excepthandler(void*);
+static PyObject* ast2obj_excepthandler(void*, struct symtable *st);
static PyTypeObject *ExceptHandler_type;
static char *ExceptHandler_fields[]={
"type",
@@ -363,7 +372,7 @@
"body",
};
static PyTypeObject *arguments_type;
-static PyObject* ast2obj_arguments(void*);
+static PyObject* ast2obj_arguments(void*, struct symtable *st);
static char *arguments_fields[]={
"args",
"vararg",
@@ -375,19 +384,19 @@
"kw_defaults",
};
static PyTypeObject *arg_type;
-static PyObject* ast2obj_arg(void*);
+static PyObject* ast2obj_arg(void*, struct symtable *st);
static char *arg_fields[]={
"arg",
"annotation",
};
static PyTypeObject *keyword_type;
-static PyObject* ast2obj_keyword(void*);
+static PyObject* ast2obj_keyword(void*, struct symtable *st);
static char *keyword_fields[]={
"arg",
"value",
};
static PyTypeObject *alias_type;
-static PyObject* ast2obj_alias(void*);
+static PyObject* ast2obj_alias(void*, struct symtable *st);
static char *alias_fields[]={
"name",
"asname",
@@ -554,7 +563,7 @@
/* Conversion AST -> Python */
-static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
+static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*, struct symtable*), struct symtable *st)
{
int i, n = asdl_seq_LEN(seq);
PyObject *result = PyList_New(n);
@@ -562,7 +571,7 @@
if (!result)
return NULL;
for (i = 0; i < n; i++) {
- value = func(asdl_seq_GET(seq, i));
+ value = func(asdl_seq_GET(seq, i), st);
if (!value) {
Py_DECREF(result);
return NULL;
@@ -572,7 +581,7 @@
return result;
}
-static PyObject* ast2obj_object(void *o)
+static PyObject* ast2obj_object(void *o, struct symtable *st)
{
if (!o)
o = Py_None;
@@ -582,7 +591,7 @@
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
-static PyObject* ast2obj_int(long b)
+static PyObject* ast2obj_int(long b, struct symtable *st)
{
return PyLong_FromLong(b);
}
@@ -610,7 +619,7 @@
PyObject *s = PyObject_Repr(obj);
if (s == NULL) return 1;
PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s",
- PyBytes_AS_STRING(s));
+ _PyUnicode_AsString(s));
Py_DECREF(s);
return 1;
}
@@ -748,6 +757,9 @@
if (!Bytes_type) return 0;
Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
if (!Ellipsis_type) return 0;
+ Specialize_type = make_type("Specialize", expr_type, Specialize_fields,
+ 4);
+ if (!Specialize_type) return 0;
Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
if (!Attribute_type) return 0;
Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3);
@@ -1819,6 +1831,40 @@
}
expr_ty
+Specialize(expr_ty name, asdl_seq * specialized_body, expr_ty
+ specialized_result, expr_ty generalized, int lineno, int col_offset,
+ PyArena *arena)
+{
+ expr_ty p;
+ if (!name) {
+ PyErr_SetString(PyExc_ValueError,
+ "field name is required for Specialize");
+ return NULL;
+ }
+ if (!specialized_result) {
+ PyErr_SetString(PyExc_ValueError,
+ "field specialized_result is required for Specialize");
+ return NULL;
+ }
+ if (!generalized) {
+ PyErr_SetString(PyExc_ValueError,
+ "field generalized is required for Specialize");
+ return NULL;
+ }
+ p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+ if (!p)
+ return NULL;
+ p->kind = Specialize_kind;
+ p->v.Specialize.name = name;
+ p->v.Specialize.specialized_body = specialized_body;
+ p->v.Specialize.specialized_result = specialized_result;
+ p->v.Specialize.generalized = generalized;
+ p->lineno = lineno;
+ p->col_offset = col_offset;
+ return p;
+}
+
+expr_ty
Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena)
{
@@ -2137,7 +2183,7 @@
PyObject*
-ast2obj_mod(void* _o)
+ast2obj_mod(void* _o, struct symtable *st)
{
mod_ty o = (mod_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -2150,7 +2196,9 @@
case Module_kind:
result = PyType_GenericNew(Module_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Module.body, ast2obj_stmt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Module.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
@@ -2159,7 +2207,9 @@
case Interactive_kind:
result = PyType_GenericNew(Interactive_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
@@ -2168,7 +2218,9 @@
case Expression_kind:
result = PyType_GenericNew(Expression_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Expression.body);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Expression.body, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
@@ -2177,7 +2229,9 @@
case Suite_kind:
result = PyType_GenericNew(Suite_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Suite.body, ast2obj_stmt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Suite.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
@@ -2192,7 +2246,7 @@
}
PyObject*
-ast2obj_stmt(void* _o)
+ast2obj_stmt(void* _o, struct symtable *st)
{
stmt_ty o = (stmt_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -2205,29 +2259,31 @@
case FunctionDef_kind:
result = PyType_GenericNew(FunctionDef_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_identifier(o->v.FunctionDef.name);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->v.FunctionDef.name, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "name", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_arguments(o->v.FunctionDef.args);
+ value = ast2obj_arguments(o->v.FunctionDef.args, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "args", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.FunctionDef.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.FunctionDef.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.FunctionDef.decorator_list,
- ast2obj_expr);
+ ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "decorator_list", value) ==
-1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.FunctionDef.returns);
+ value = ast2obj_expr(o->v.FunctionDef.returns, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "returns", value) == -1)
goto failed;
@@ -2236,38 +2292,41 @@
case ClassDef_kind:
result = PyType_GenericNew(ClassDef_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_identifier(o->v.ClassDef.name);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->v.ClassDef.name, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "name", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.ClassDef.bases, ast2obj_expr);
+ value = ast2obj_list(o->v.ClassDef.bases, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "bases", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword);
+ value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword,
+ st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "keywords", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.ClassDef.starargs);
+ value = ast2obj_expr(o->v.ClassDef.starargs, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "starargs", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.ClassDef.kwargs);
+ value = ast2obj_expr(o->v.ClassDef.kwargs, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwargs", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.ClassDef.decorator_list,
- ast2obj_expr);
+ ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "decorator_list", value) ==
-1)
@@ -2277,7 +2336,9 @@
case Return_kind:
result = PyType_GenericNew(Return_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Return.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Return.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -2286,7 +2347,9 @@
case Delete_kind:
result = PyType_GenericNew(Delete_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Delete.targets, ast2obj_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Delete.targets, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "targets", value) == -1)
goto failed;
@@ -2295,12 +2358,14 @@
case Assign_kind:
result = PyType_GenericNew(Assign_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Assign.targets, ast2obj_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Assign.targets, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "targets", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Assign.value);
+ value = ast2obj_expr(o->v.Assign.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -2309,17 +2374,19 @@
case AugAssign_kind:
result = PyType_GenericNew(AugAssign_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.AugAssign.target);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.AugAssign.target, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "target", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_operator(o->v.AugAssign.op);
+ value = ast2obj_operator(o->v.AugAssign.op, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "op", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.AugAssign.value);
+ value = ast2obj_expr(o->v.AugAssign.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -2328,22 +2395,24 @@
case For_kind:
result = PyType_GenericNew(For_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.For.target);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.For.target, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "target", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.For.iter);
+ value = ast2obj_expr(o->v.For.iter, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "iter", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.For.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.For.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.For.orelse, ast2obj_stmt);
+ value = ast2obj_list(o->v.For.orelse, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "orelse", value) == -1)
goto failed;
@@ -2352,17 +2421,19 @@
case While_kind:
result = PyType_GenericNew(While_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.While.test);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.While.test, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "test", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.While.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.While.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.While.orelse, ast2obj_stmt);
+ value = ast2obj_list(o->v.While.orelse, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "orelse", value) == -1)
goto failed;
@@ -2371,17 +2442,19 @@
case If_kind:
result = PyType_GenericNew(If_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.If.test);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.If.test, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "test", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.If.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.If.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.If.orelse, ast2obj_stmt);
+ value = ast2obj_list(o->v.If.orelse, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "orelse", value) == -1)
goto failed;
@@ -2390,18 +2463,20 @@
case With_kind:
result = PyType_GenericNew(With_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.With.context_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.With.context_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "context_expr", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.With.optional_vars);
+ value = ast2obj_expr(o->v.With.optional_vars, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "optional_vars", value) ==
-1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.With.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.With.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
@@ -2410,12 +2485,14 @@
case Raise_kind:
result = PyType_GenericNew(Raise_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Raise.exc);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Raise.exc, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "exc", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Raise.cause);
+ value = ast2obj_expr(o->v.Raise.cause, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "cause", value) == -1)
goto failed;
@@ -2424,18 +2501,20 @@
case TryExcept_kind:
result = PyType_GenericNew(TryExcept_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.TryExcept.body, ast2obj_stmt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.TryExcept.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.TryExcept.handlers,
- ast2obj_excepthandler);
+ ast2obj_excepthandler, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "handlers", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.TryExcept.orelse, ast2obj_stmt);
+ value = ast2obj_list(o->v.TryExcept.orelse, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "orelse", value) == -1)
goto failed;
@@ -2444,12 +2523,15 @@
case TryFinally_kind:
result = PyType_GenericNew(TryFinally_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.TryFinally.body, ast2obj_stmt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.TryFinally.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.TryFinally.finalbody, ast2obj_stmt);
+ value = ast2obj_list(o->v.TryFinally.finalbody, ast2obj_stmt,
+ st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "finalbody", value) == -1)
goto failed;
@@ -2458,12 +2540,14 @@
case Assert_kind:
result = PyType_GenericNew(Assert_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Assert.test);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Assert.test, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "test", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Assert.msg);
+ value = ast2obj_expr(o->v.Assert.msg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "msg", value) == -1)
goto failed;
@@ -2472,7 +2556,9 @@
case Import_kind:
result = PyType_GenericNew(Import_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Import.names, ast2obj_alias);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Import.names, ast2obj_alias, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "names", value) == -1)
goto failed;
@@ -2481,17 +2567,19 @@
case ImportFrom_kind:
result = PyType_GenericNew(ImportFrom_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_identifier(o->v.ImportFrom.module);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->v.ImportFrom.module, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "module", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.ImportFrom.names, ast2obj_alias);
+ value = ast2obj_list(o->v.ImportFrom.names, ast2obj_alias, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "names", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_int(o->v.ImportFrom.level);
+ value = ast2obj_int(o->v.ImportFrom.level, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "level", value) == -1)
goto failed;
@@ -2500,7 +2588,9 @@
case Global_kind:
result = PyType_GenericNew(Global_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Global.names, ast2obj_identifier);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Global.names, ast2obj_identifier, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "names", value) == -1)
goto failed;
@@ -2509,7 +2599,10 @@
case Nonlocal_kind:
result = PyType_GenericNew(Nonlocal_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier,
+ st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "names", value) == -1)
goto failed;
@@ -2518,7 +2611,9 @@
case Expr_kind:
result = PyType_GenericNew(Expr_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Expr.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Expr.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -2527,22 +2622,28 @@
case Pass_kind:
result = PyType_GenericNew(Pass_type, NULL, NULL);
if (!result) goto failed;
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
break;
case Break_kind:
result = PyType_GenericNew(Break_type, NULL, NULL);
if (!result) goto failed;
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
break;
case Continue_kind:
result = PyType_GenericNew(Continue_type, NULL, NULL);
if (!result) goto failed;
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
break;
}
- value = ast2obj_int(o->lineno);
+ value = ast2obj_int(o->lineno, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "lineno", value) < 0)
goto failed;
Py_DECREF(value);
- value = ast2obj_int(o->col_offset);
+ value = ast2obj_int(o->col_offset, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "col_offset", value) < 0)
goto failed;
@@ -2555,7 +2656,7 @@
}
PyObject*
-ast2obj_expr(void* _o)
+ast2obj_expr(void* _o, struct symtable *st)
{
expr_ty o = (expr_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -2568,12 +2669,14 @@
case BoolOp_kind:
result = PyType_GenericNew(BoolOp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_boolop(o->v.BoolOp.op);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_boolop(o->v.BoolOp.op, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "op", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr);
+ value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "values", value) == -1)
goto failed;
@@ -2582,17 +2685,19 @@
case BinOp_kind:
result = PyType_GenericNew(BinOp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.BinOp.left);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.BinOp.left, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "left", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_operator(o->v.BinOp.op);
+ value = ast2obj_operator(o->v.BinOp.op, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "op", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.BinOp.right);
+ value = ast2obj_expr(o->v.BinOp.right, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "right", value) == -1)
goto failed;
@@ -2601,12 +2706,14 @@
case UnaryOp_kind:
result = PyType_GenericNew(UnaryOp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_unaryop(o->v.UnaryOp.op);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_unaryop(o->v.UnaryOp.op, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "op", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.UnaryOp.operand);
+ value = ast2obj_expr(o->v.UnaryOp.operand, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "operand", value) == -1)
goto failed;
@@ -2615,12 +2722,14 @@
case Lambda_kind:
result = PyType_GenericNew(Lambda_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_arguments(o->v.Lambda.args);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_arguments(o->v.Lambda.args, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "args", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Lambda.body);
+ value = ast2obj_expr(o->v.Lambda.body, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
@@ -2629,17 +2738,19 @@
case IfExp_kind:
result = PyType_GenericNew(IfExp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.IfExp.test);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.IfExp.test, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "test", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.IfExp.body);
+ value = ast2obj_expr(o->v.IfExp.body, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.IfExp.orelse);
+ value = ast2obj_expr(o->v.IfExp.orelse, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "orelse", value) == -1)
goto failed;
@@ -2648,12 +2759,14 @@
case Dict_kind:
result = PyType_GenericNew(Dict_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Dict.keys, ast2obj_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Dict.keys, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "keys", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.Dict.values, ast2obj_expr);
+ value = ast2obj_list(o->v.Dict.values, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "values", value) == -1)
goto failed;
@@ -2662,7 +2775,9 @@
case Set_kind:
result = PyType_GenericNew(Set_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Set.elts, ast2obj_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Set.elts, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elts", value) == -1)
goto failed;
@@ -2671,13 +2786,15 @@
case ListComp_kind:
result = PyType_GenericNew(ListComp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.ListComp.elt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.ListComp.elt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.ListComp.generators,
- ast2obj_comprehension);
+ ast2obj_comprehension, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
@@ -2686,13 +2803,15 @@
case SetComp_kind:
result = PyType_GenericNew(SetComp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.SetComp.elt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.SetComp.elt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.SetComp.generators,
- ast2obj_comprehension);
+ ast2obj_comprehension, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
@@ -2701,18 +2820,20 @@
case DictComp_kind:
result = PyType_GenericNew(DictComp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.DictComp.key);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.DictComp.key, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "key", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.DictComp.value);
+ value = ast2obj_expr(o->v.DictComp.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.DictComp.generators,
- ast2obj_comprehension);
+ ast2obj_comprehension, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
@@ -2721,13 +2842,15 @@
case GeneratorExp_kind:
result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.GeneratorExp.elt);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.GeneratorExp.elt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elt", value) == -1)
goto failed;
Py_DECREF(value);
value = ast2obj_list(o->v.GeneratorExp.generators,
- ast2obj_comprehension);
+ ast2obj_comprehension, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "generators", value) == -1)
goto failed;
@@ -2736,7 +2859,9 @@
case Yield_kind:
result = PyType_GenericNew(Yield_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Yield.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Yield.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -2745,7 +2870,9 @@
case Compare_kind:
result = PyType_GenericNew(Compare_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Compare.left);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Compare.left, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "left", value) == -1)
goto failed;
@@ -2755,13 +2882,14 @@
value = PyList_New(n);
if (!value) goto failed;
for(i = 0; i < n; i++)
- PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i)));
+ PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i), st));
}
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ops", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr);
+ value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr,
+ st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "comparators", value) == -1)
goto failed;
@@ -2770,27 +2898,29 @@
case Call_kind:
result = PyType_GenericNew(Call_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Call.func);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Call.func, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "func", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.Call.args, ast2obj_expr);
+ value = ast2obj_list(o->v.Call.args, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "args", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword);
+ value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "keywords", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Call.starargs);
+ value = ast2obj_expr(o->v.Call.starargs, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "starargs", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Call.kwargs);
+ value = ast2obj_expr(o->v.Call.kwargs, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwargs", value) == -1)
goto failed;
@@ -2799,7 +2929,9 @@
case Num_kind:
result = PyType_GenericNew(Num_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_object(o->v.Num.n);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_object(o->v.Num.n, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "n", value) == -1)
goto failed;
@@ -2808,7 +2940,9 @@
case Str_kind:
result = PyType_GenericNew(Str_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_string(o->v.Str.s);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_string(o->v.Str.s, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "s", value) == -1)
goto failed;
@@ -2817,7 +2951,9 @@
case Bytes_kind:
result = PyType_GenericNew(Bytes_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_string(o->v.Bytes.s);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_string(o->v.Bytes.s, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "s", value) == -1)
goto failed;
@@ -2826,21 +2962,54 @@
case Ellipsis_kind:
result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
if (!result) goto failed;
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ break;
+ case Specialize_kind:
+ result = PyType_GenericNew(Specialize_type, NULL, NULL);
+ if (!result) goto failed;
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Specialize.name, st);
+ if (!value) goto failed;
+ if (PyObject_SetAttrString(result, "name", value) == -1)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_list(o->v.Specialize.specialized_body,
+ ast2obj_stmt, st);
+ if (!value) goto failed;
+ if (PyObject_SetAttrString(result, "specialized_body", value)
+ == -1)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_expr(o->v.Specialize.specialized_result, st);
+ if (!value) goto failed;
+ if (PyObject_SetAttrString(result, "specialized_result", value)
+ == -1)
+ goto failed;
+ Py_DECREF(value);
+ value = ast2obj_expr(o->v.Specialize.generalized, st);
+ if (!value) goto failed;
+ if (PyObject_SetAttrString(result, "generalized", value) == -1)
+ goto failed;
+ Py_DECREF(value);
break;
case Attribute_kind:
result = PyType_GenericNew(Attribute_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Attribute.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Attribute.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_identifier(o->v.Attribute.attr);
+ value = ast2obj_identifier(o->v.Attribute.attr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "attr", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr_context(o->v.Attribute.ctx);
+ value = ast2obj_expr_context(o->v.Attribute.ctx, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ctx", value) == -1)
goto failed;
@@ -2849,17 +3018,19 @@
case Subscript_kind:
result = PyType_GenericNew(Subscript_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Subscript.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Subscript.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_slice(o->v.Subscript.slice);
+ value = ast2obj_slice(o->v.Subscript.slice, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "slice", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr_context(o->v.Subscript.ctx);
+ value = ast2obj_expr_context(o->v.Subscript.ctx, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ctx", value) == -1)
goto failed;
@@ -2868,12 +3039,14 @@
case Starred_kind:
result = PyType_GenericNew(Starred_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Starred.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Starred.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr_context(o->v.Starred.ctx);
+ value = ast2obj_expr_context(o->v.Starred.ctx, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ctx", value) == -1)
goto failed;
@@ -2882,12 +3055,14 @@
case Name_kind:
result = PyType_GenericNew(Name_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_identifier(o->v.Name.id);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->v.Name.id, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "id", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr_context(o->v.Name.ctx);
+ value = ast2obj_expr_context(o->v.Name.ctx, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ctx", value) == -1)
goto failed;
@@ -2896,12 +3071,14 @@
case List_kind:
result = PyType_GenericNew(List_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.List.elts, ast2obj_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.List.elts, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elts", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr_context(o->v.List.ctx);
+ value = ast2obj_expr_context(o->v.List.ctx, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ctx", value) == -1)
goto failed;
@@ -2910,24 +3087,26 @@
case Tuple_kind:
result = PyType_GenericNew(Tuple_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "elts", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr_context(o->v.Tuple.ctx);
+ value = ast2obj_expr_context(o->v.Tuple.ctx, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ctx", value) == -1)
goto failed;
Py_DECREF(value);
break;
}
- value = ast2obj_int(o->lineno);
+ value = ast2obj_int(o->lineno, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "lineno", value) < 0)
goto failed;
Py_DECREF(value);
- value = ast2obj_int(o->col_offset);
+ value = ast2obj_int(o->col_offset, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "col_offset", value) < 0)
goto failed;
@@ -2939,7 +3118,7 @@
return NULL;
}
-PyObject* ast2obj_expr_context(expr_context_ty o)
+PyObject* ast2obj_expr_context(expr_context_ty o, struct symtable *st)
{
switch(o) {
case Load:
@@ -2967,7 +3146,7 @@
}
}
PyObject*
-ast2obj_slice(void* _o)
+ast2obj_slice(void* _o, struct symtable *st)
{
slice_ty o = (slice_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -2980,17 +3159,19 @@
case Slice_kind:
result = PyType_GenericNew(Slice_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Slice.lower);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Slice.lower, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "lower", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Slice.upper);
+ value = ast2obj_expr(o->v.Slice.upper, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "upper", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->v.Slice.step);
+ value = ast2obj_expr(o->v.Slice.step, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "step", value) == -1)
goto failed;
@@ -2999,7 +3180,9 @@
case ExtSlice_kind:
result = PyType_GenericNew(ExtSlice_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "dims", value) == -1)
goto failed;
@@ -3008,7 +3191,9 @@
case Index_kind:
result = PyType_GenericNew(Index_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.Index.value);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.Index.value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -3022,7 +3207,7 @@
return NULL;
}
-PyObject* ast2obj_boolop(boolop_ty o)
+PyObject* ast2obj_boolop(boolop_ty o, struct symtable *st)
{
switch(o) {
case And:
@@ -3037,7 +3222,7 @@
return NULL;
}
}
-PyObject* ast2obj_operator(operator_ty o)
+PyObject* ast2obj_operator(operator_ty o, struct symtable *st)
{
switch(o) {
case Add:
@@ -3082,7 +3267,7 @@
return NULL;
}
}
-PyObject* ast2obj_unaryop(unaryop_ty o)
+PyObject* ast2obj_unaryop(unaryop_ty o, struct symtable *st)
{
switch(o) {
case Invert:
@@ -3103,7 +3288,7 @@
return NULL;
}
}
-PyObject* ast2obj_cmpop(cmpop_ty o)
+PyObject* ast2obj_cmpop(cmpop_ty o, struct symtable *st)
{
switch(o) {
case Eq:
@@ -3143,7 +3328,7 @@
}
}
PyObject*
-ast2obj_comprehension(void* _o)
+ast2obj_comprehension(void* _o, struct symtable *st)
{
comprehension_ty o = (comprehension_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -3154,17 +3339,19 @@
result = PyType_GenericNew(comprehension_type, NULL, NULL);
if (!result) return NULL;
- value = ast2obj_expr(o->target);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->target, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "target", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->iter);
+ value = ast2obj_expr(o->iter, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "iter", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->ifs, ast2obj_expr);
+ value = ast2obj_list(o->ifs, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "ifs", value) == -1)
goto failed;
@@ -3177,7 +3364,7 @@
}
PyObject*
-ast2obj_excepthandler(void* _o)
+ast2obj_excepthandler(void* _o, struct symtable *st)
{
excepthandler_ty o = (excepthandler_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -3190,29 +3377,31 @@
case ExceptHandler_kind:
result = PyType_GenericNew(ExceptHandler_type, NULL, NULL);
if (!result) goto failed;
- value = ast2obj_expr(o->v.ExceptHandler.type);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_expr(o->v.ExceptHandler.type, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "type", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_identifier(o->v.ExceptHandler.name);
+ value = ast2obj_identifier(o->v.ExceptHandler.name, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "name", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt);
+ value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "body", value) == -1)
goto failed;
Py_DECREF(value);
break;
}
- value = ast2obj_int(o->lineno);
+ value = ast2obj_int(o->lineno, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "lineno", value) < 0)
goto failed;
Py_DECREF(value);
- value = ast2obj_int(o->col_offset);
+ value = ast2obj_int(o->col_offset, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "col_offset", value) < 0)
goto failed;
@@ -3225,7 +3414,7 @@
}
PyObject*
-ast2obj_arguments(void* _o)
+ast2obj_arguments(void* _o, struct symtable *st)
{
arguments_ty o = (arguments_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -3236,42 +3425,44 @@
result = PyType_GenericNew(arguments_type, NULL, NULL);
if (!result) return NULL;
- value = ast2obj_list(o->args, ast2obj_arg);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_list(o->args, ast2obj_arg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "args", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_identifier(o->vararg);
+ value = ast2obj_identifier(o->vararg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "vararg", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->varargannotation);
+ value = ast2obj_expr(o->varargannotation, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "varargannotation", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->kwonlyargs, ast2obj_arg);
+ value = ast2obj_list(o->kwonlyargs, ast2obj_arg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwonlyargs", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_identifier(o->kwarg);
+ value = ast2obj_identifier(o->kwarg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwarg", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->kwargannotation);
+ value = ast2obj_expr(o->kwargannotation, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kwargannotation", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->defaults, ast2obj_expr);
+ value = ast2obj_list(o->defaults, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "defaults", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_list(o->kw_defaults, ast2obj_expr);
+ value = ast2obj_list(o->kw_defaults, ast2obj_expr, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "kw_defaults", value) == -1)
goto failed;
@@ -3284,7 +3475,7 @@
}
PyObject*
-ast2obj_arg(void* _o)
+ast2obj_arg(void* _o, struct symtable *st)
{
arg_ty o = (arg_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -3295,12 +3486,14 @@
result = PyType_GenericNew(arg_type, NULL, NULL);
if (!result) return NULL;
- value = ast2obj_identifier(o->arg);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->arg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "arg", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->annotation);
+ value = ast2obj_expr(o->annotation, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "annotation", value) == -1)
goto failed;
@@ -3313,7 +3506,7 @@
}
PyObject*
-ast2obj_keyword(void* _o)
+ast2obj_keyword(void* _o, struct symtable *st)
{
keyword_ty o = (keyword_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -3324,12 +3517,14 @@
result = PyType_GenericNew(keyword_type, NULL, NULL);
if (!result) return NULL;
- value = ast2obj_identifier(o->arg);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->arg, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "arg", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_expr(o->value);
+ value = ast2obj_expr(o->value, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "value", value) == -1)
goto failed;
@@ -3342,7 +3537,7 @@
}
PyObject*
-ast2obj_alias(void* _o)
+ast2obj_alias(void* _o, struct symtable *st)
{
alias_ty o = (alias_ty)_o;
PyObject *result = NULL, *value = NULL;
@@ -3353,12 +3548,14 @@
result = PyType_GenericNew(alias_type, NULL, NULL);
if (!result) return NULL;
- value = ast2obj_identifier(o->name);
+ if (!obj2ast_write_scope(o, result, st))
+ goto failed;
+ value = ast2obj_identifier(o->name, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "name", value) == -1)
goto failed;
Py_DECREF(value);
- value = ast2obj_identifier(o->asname);
+ value = ast2obj_identifier(o->asname, st);
if (!value) goto failed;
if (PyObject_SetAttrString(result, "asname", value) == -1)
goto failed;
@@ -5557,6 +5754,82 @@
if (*out == NULL) goto failed;
return 0;
}
+ isinstance = PyObject_IsInstance(obj, (PyObject*)Specialize_type);
+ if (isinstance == -1) {
+ return 1;
+ }
+ if (isinstance) {
+ expr_ty name;
+ asdl_seq* specialized_body;
+ expr_ty specialized_result;
+ expr_ty generalized;
+
+ if (PyObject_HasAttrString(obj, "name")) {
+ int res;
+ tmp = PyObject_GetAttrString(obj, "name");
+ if (tmp == NULL) goto failed;
+ res = obj2ast_expr(tmp, &name, arena);
+ if (res != 0) goto failed;
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from Specialize");
+ return 1;
+ }
+ if (PyObject_HasAttrString(obj, "specialized_body")) {
+ int res;
+ Py_ssize_t len;
+ Py_ssize_t i;
+ tmp = PyObject_GetAttrString(obj, "specialized_body");
+ if (tmp == NULL) goto failed;
+ if (!PyList_Check(tmp)) {
+ PyErr_Format(PyExc_TypeError, "Specialize field \"specialized_body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
+ goto failed;
+ }
+ len = PyList_GET_SIZE(tmp);
+ specialized_body = asdl_seq_new(len, arena);
+ if (specialized_body == NULL) goto failed;
+ for (i = 0; i < len; i++) {
+ stmt_ty value;
+ res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena);
+ if (res != 0) goto failed;
+ asdl_seq_SET(specialized_body, i, value);
+ }
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"specialized_body\" missing from Specialize");
+ return 1;
+ }
+ if (PyObject_HasAttrString(obj, "specialized_result")) {
+ int res;
+ tmp = PyObject_GetAttrString(obj, "specialized_result");
+ if (tmp == NULL) goto failed;
+ res = obj2ast_expr(tmp, &specialized_result, arena);
+ if (res != 0) goto failed;
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"specialized_result\" missing from Specialize");
+ return 1;
+ }
+ if (PyObject_HasAttrString(obj, "generalized")) {
+ int res;
+ tmp = PyObject_GetAttrString(obj, "generalized");
+ if (tmp == NULL) goto failed;
+ res = obj2ast_expr(tmp, &generalized, arena);
+ if (res != 0) goto failed;
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"generalized\" missing from Specialize");
+ return 1;
+ }
+ *out = Specialize(name, specialized_body, specialized_result,
+ generalized, lineno, col_offset, arena);
+ if (*out == NULL) goto failed;
+ return 0;
+ }
isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type);
if (isinstance == -1) {
return 1;
@@ -6834,6 +7107,8 @@
NULL;
if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0)
return NULL;
+ if (PyDict_SetItemString(d, "Specialize", (PyObject*)Specialize_type) <
+ 0) return NULL;
if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) <
0) return NULL;
if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) <
@@ -6944,10 +7219,24 @@
}
-PyObject* PyAST_mod2obj(mod_ty t)
+int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st)
+{
+ PySTEntryObject * ste;
+ if (st) {
+ ste = PySymtable_TryLookup(st, ast_obj);
+ if (ste) {
+ /* This AST node has a symtable entry: */
+ if (PyObject_SetAttrString(py_obj, "ste", (PyObject*)ste) == -1)
+ return 0;
+ }
+ }
+ return 1;
+}
+
+PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st)
{
init_types();
- return ast2obj_mod(t);
+ return ast2obj_mod(t, st);
}
/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/bltinmodule.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/bltinmodule.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/bltinmodule.c Tue Nov 23 20:24:02 2010
@@ -592,7 +592,7 @@
goto error;
}
result = (PyObject*)PyAST_Compile(mod, filename,
- &cf, arena);
+ &cf, arena, mode);
PyArena_Free(arena);
}
goto finally;
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/ceval.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/ceval.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/ceval.c Tue Nov 23 20:24:02 2010
@@ -731,6 +731,30 @@
return 0;
}
+/* Compare two functions, one dynamically looked up, the other the expected
+ version.
+
+ If they're equal, then we can use specialized bytecode that assumes that
+ they were.
+*/
+static int
+is_specializable(PyObject *dynamic, PyObject *expected)
+{
+#if 0
+ if (!PyFunction_Check(dynamic))
+ return 0;
+
+ if (!PyFunction_Check(expected))
+ return 0;
+#endif
+
+ /* If they're the same object, we have a match: */
+ if (dynamic == expected)
+ return 1;
+
+ return 0; // FIXME!
+}
+
/* Status code for main loop (reason for stack unwind) */
enum why_code {
WHY_NOT = 0x0001, /* No error */
@@ -2836,6 +2860,25 @@
if (x != NULL) DISPATCH();
break;
+ TARGET(JUMP_IF_SPECIALIZABLE)
+ v = POP(); /* ...expected implementation */
+ u = TOP(); /* ...dynamic lookup of implementation */
+
+ if (is_specializable(u, v)) {
+ /* Jump to specialized implementation, popping u */
+ JUMPTO(oparg);
+ Py_DECREF(v);
+ STACKADJ(-1);
+ Py_DECREF(u);
+ FAST_DISPATCH();
+ } else {
+ /* Generalized implementation; fall through to next opcode,
+ keeping u on TOS */
+ Py_DECREF(v);
+ DISPATCH();
+ }
+ break;
+
TARGET(EXTENDED_ARG)
opcode = NEXTOP();
oparg = oparg<<16 | NEXTARG();
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/compile.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/compile.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/compile.c Tue Nov 23 20:24:02 2010
@@ -33,6 +33,7 @@
#include "opcode.h"
int Py_OptimizeFlag = 0;
+static int ready_to_optimize = 0;
#define DEFAULT_BLOCK_SIZE 16
#define DEFAULT_BLOCKS 8
@@ -149,6 +150,8 @@
PyArena *c_arena; /* pointer to memory allocation arena */
};
+static mod_ty optimize_mod(struct compiler *, mod_ty, enum PyCompilationMode);
+
static int compiler_enter_scope(struct compiler *, identifier, void *, int);
static void compiler_free(struct compiler *);
static basicblock *compiler_new_block(struct compiler *);
@@ -180,6 +183,7 @@
static int expr_constant(expr_ty e);
static int compiler_with(struct compiler *, stmt_ty);
+static int compiler_specialize(struct compiler *, expr_ty);
static int compiler_call_helper(struct compiler *c, int n,
asdl_seq *args,
asdl_seq *keywords,
@@ -257,7 +261,7 @@
PyCodeObject *
PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
- PyArena *arena)
+ PyArena *arena, enum PyCompilationMode mode)
{
struct compiler c;
PyCodeObject *co = NULL;
@@ -294,6 +298,10 @@
goto finally;
}
+ mod = optimize_mod(&c, mod, mode);
+ if (!mod)
+ goto finally;
+
co = compiler_mod(&c, mod);
finally:
@@ -312,11 +320,125 @@
return NULL;
mod = PyAST_FromNode(n, NULL, filename, arena);
if (mod)
- co = PyAST_Compile(mod, filename, NULL, arena);
+ co = PyAST_Compile(mod, filename, NULL, arena, PyCompilationMode_Exec_Module);
PyArena_Free(arena);
return co;
}
+/*
+ * Overview of the AST optimizer:
+ * 1. Convert the AST to python objects
+ * 2. Import __optimizer__ and invoke __optimizer__.optimize_ast() on the
+ * python form of the tree
+ * 3. This will returns a potentially modified version of the tree (in
+ * python form)
+ * 4. Convert the python objects back to an AST
+ * 5. Return the resulting AST back, for use by the compiler.
+ *
+ * TODO: how does this interract with symbol tables?
+ * TODO: what about the GIL?
+ * TODO: which errors should be fatal, if any? how to report errors?
+ */
+static int within_optimizer = 0;
+static mod_ty optimize_mod(struct compiler *c, mod_ty m, enum PyCompilationMode mode)
+{
+ PyObject *optimizer_module = NULL;
+ PyObject *optimize_ast_str = NULL;
+ PyObject *optimize_ast_fn = NULL;
+ PyObject *py_ast_in = NULL;
+ PyObject *filename = NULL;
+ PyObject *py_ast_out = NULL;
+ mod_ty new_mod = NULL;
+ PyObject *exc = NULL;
+ struct symtable *new_st = NULL;
+
+ assert(c);
+ assert(m);
+
+ /* Avoid running optimizer until the end of Py_InitializeEx: */
+ if (!ready_to_optimize)
+ return m;
+
+ /* Avoid infinite recursion: don't try to optimize the optimizer (or
+ modules imported during the import of the optimizer): */
+ if (within_optimizer)
+ return m;
+ within_optimizer = 1;
+
+ /* printf("Optimizing: %s\n", c->c_filename); */
+
+ /* Import "__optimizer__.optimize_ast" -> optimize_ast_fn: */
+ optimizer_module = PyImport_ImportModule("__optimizer__");
+ if (!optimizer_module)
+ goto finally;
+
+ optimize_ast_str= PyUnicode_InternFromString("optimize_ast");
+ if (!optimize_ast_str)
+ goto finally;
+
+ optimize_ast_fn = PyObject_GetAttr(optimizer_module, optimize_ast_str);
+ if (!optimize_ast_fn) {
+//PyErr_SetObject(PyExc_ImportError, optimize_ast_str);
+ goto finally;
+ }
+
+ /* Convert the AST repr to Python objects: */
+ py_ast_in = PyAST_mod2obj(m, c->c_st);
+ if (!py_ast_in)
+ goto finally;
+
+ /* Invoke the "__optimizer__.optimize_ast(ast, filename)": */
+ filename = PyUnicode_DecodeFSDefault(c->c_filename); /* FIXME: is this the correct encoding? */
+ if (!filename)
+ goto finally;
+
+ assert(c->c_st);
+ py_ast_out = PyObject_CallFunctionObjArgs(optimize_ast_fn,
+ py_ast_in, filename, c->c_st->st_blocks,
+ NULL);
+ if (!py_ast_out)
+ goto finally;
+
+ /* 4. Convert the python objects back to an AST: */
+ new_mod = PyAST_obj2mod(py_ast_out, c->c_arena, mode);
+ if (!new_mod)
+ goto finally;
+
+ /* 5. Rebuild the symbol table, using the new AST: */
+ new_st = PySymtable_Build(new_mod, c->c_filename, c->c_future);
+ if (new_st == NULL) {
+ if (!PyErr_Occurred())
+ PyErr_SetString(PyExc_SystemError, "no symtable");
+
+ goto finally;
+ }
+
+ /* 6. Use the optimizer's version of the AST and the new symtable: */
+ m = new_mod;
+ PySymtable_Free(c->c_st);
+ c->c_st = new_st;
+
+finally:
+ exc = PyErr_Occurred();
+ //if (exc) {
+ // PyErr_PrintEx(0);
+#if 0
+ PyObject *tb = PyException_GetTraceback(exc);
+ if (exc && optimize_ast_fn)
+ PyTraceBack_Print(tb, optimize_ast_fn);
+#endif
+// }
+ Py_XDECREF(py_ast_out);
+ Py_XDECREF(filename);
+ Py_XDECREF(py_ast_in);
+ Py_XDECREF(optimize_ast_fn);
+ Py_XDECREF(optimize_ast_str);
+ Py_XDECREF(optimizer_module);
+ within_optimizer = 0;
+ return m;
+}
+
+
static void
compiler_free(struct compiler *c)
{
@@ -861,6 +983,10 @@
return -1;
case DELETE_DEREF:
return 0;
+
+ case JUMP_IF_SPECIALIZABLE:
+ return -2; /* -1 if jump not taken */
+
default:
fprintf(stderr, "opcode = %d\n", opcode);
Py_FatalError("opcode_stack_effect()");
@@ -3114,6 +3240,74 @@
return 1;
}
+
+/*
+ FIXME: need a PEP for this
+
+ This one's unusual, in that it's an expression, but it contains statements
+ and control flow.
+*/
+static int
+compiler_specialize(struct compiler *c, expr_ty e)
+{
+ basicblock *specialized, *generalized, *end;
+ expr_ty call;
+ identifier id;
+ PyObject *saved_name = NULL;
+
+ assert(e->kind == Specialize_kind);
+
+ specialized = compiler_new_block(c);
+ generalized = compiler_new_block(c);
+ end = compiler_new_block(c);
+ if (!specialized || !generalized || !end)
+ return 0;
+
+ /* Do the dynamic name lookup, it will be the new TOS */
+ VISIT(c, expr, e->v.Specialize.name);
+
+ /* Push the expected version of the name, it will be the new TOS */
+ /* For now, look for a global named "__saved__" + funcname */
+ if (e->v.Specialize.name->kind != Name_kind)
+ return 0;
+ id = e->v.Specialize.name->v.Name.id;
+ assert(id);
+ saved_name = PyUnicode_FromFormat("__saved__%U", id);
+ if (!saved_name)
+ return 0;
+ ADDOP_O(c, LOAD_GLOBAL, saved_name, names); /* takes ownership of the reference */
+
+ /* JUMP_IF_SPECIALIZABLE; pops the expected, and peeks the dynamic,
+ leaving dynamically-looked-up fn as TOS */
+ ADDOP_JABS(c, JUMP_IF_SPECIALIZABLE, specialized);
+
+ /* The generalized version */
+ compiler_use_next_block(c, generalized);
+
+ call = e->v.Specialize.generalized;
+ if (call->kind != Call_kind)
+ return 0;
+ /* Dynamic lookup will be on TOS, don't need to look it up again; go
+ ahead with args: */
+ if (!compiler_call_helper(c, 0,
+ call->v.Call.args,
+ call->v.Call.keywords,
+ call->v.Call.starargs,
+ call->v.Call.kwargs))
+ return 0;
+ ADDOP_JREL(c, JUMP_FORWARD, end);
+
+ /* The specialized version */
+ compiler_use_next_block(c, specialized);
+ VISIT_SEQ(c, stmt, e->v.Specialize.specialized_body);
+ VISIT(c, expr, e->v.Specialize.specialized_result);
+
+ /* Fall through to "end" */
+ compiler_use_next_block(c, end);
+
+ return 1;
+}
+
static int
compiler_visit_expr(struct compiler *c, expr_ty e)
{
@@ -3195,6 +3389,9 @@
case Ellipsis_kind:
ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts);
break;
+ case Specialize_kind:
+ return compiler_specialize(c, e);
+ break;
/* The following exprs can be assignment targets. */
case Attribute_kind:
if (e->v.Attribute.ctx != AugStore)
@@ -3676,7 +3873,9 @@
d_lineno = i->i_lineno - a->a_lineno;
assert(d_bytecode >= 0);
- assert(d_lineno >= 0);
+ /* FIXME: currently the lineno implementation assumes monotonically
+ increasing line nums, which isn't true with function inlining: */
+ /* assert(d_lineno >= 0); */
if(d_bytecode == 0 && d_lineno == 0)
return 1;
@@ -4082,3 +4281,12 @@
assemble_free(&a);
return co;
}
+
+void _PyOptimizer_Init(void)
+{
+ /* Don't try to optimize until early initialization is done.
+ This prevents us from e.g. trying to optimize the "encodings" module
+ before there's a "sys" module:
+ */
+ ready_to_optimize = 1;
+}
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/import.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/import.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/import.c Tue Nov 23 20:24:02 2010
@@ -1110,7 +1110,8 @@
Py_file_input, 0, 0, &flags,
NULL, arena);
if (mod) {
- co = PyAST_Compile(mod, pathname, NULL, arena);
+ co = PyAST_Compile(mod, pathname, NULL, arena,
+ PyCompilationMode_Exec_Module);
}
PyArena_Free(arena);
return co;
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/opcode_targets.h
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/opcode_targets.h (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/opcode_targets.h Tue Nov 23 20:24:02 2010
@@ -147,7 +147,7 @@
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
&&TARGET_MAP_ADD,
- &&_unknown_opcode,
+ &&TARGET_JUMP_IF_SPECIALIZABLE,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/pythonrun.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/pythonrun.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/pythonrun.c Tue Nov 23 20:24:02 2010
@@ -62,7 +62,7 @@
static int initstdio(void);
static void flush_io(void);
static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *,
- PyCompilerFlags *, PyArena *);
+ PyCompilerFlags *, PyArena *, enum PyCompilationMode);
static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
PyCompilerFlags *);
static void err_input(perrdetail *);
@@ -316,6 +316,10 @@
if (!Py_NoSiteFlag)
initsite(); /* Module site */
+
+ /* It should now be safe to run the optimizer when importing modules: */
+ _PyOptimizer_Init();
+
}
void
@@ -1148,7 +1152,8 @@
return -1;
}
d = PyModule_GetDict(m);
- v = run_mod(mod, filename, d, d, flags, arena);
+ v = run_mod(mod, filename, d, d, flags, arena,
+ PyCompilationMode_Single_Interactive);
PyArena_Free(arena);
flush_io();
if (v == NULL) {
@@ -1693,7 +1698,8 @@
mod = PyParser_ASTFromString(str, "<string>", start, flags, arena);
if (mod != NULL)
- ret = run_mod(mod, "<string>", globals, locals, flags, arena);
+ ret = run_mod(mod, "<string>", globals, locals, flags, arena,
+ PyAST_CompilationModeFromStartToken(start));
PyArena_Free(arena);
return ret;
}
@@ -1716,7 +1722,8 @@
PyArena_Free(arena);
return NULL;
}
- ret = run_mod(mod, filename, globals, locals, flags, arena);
+ ret = run_mod(mod, filename, globals, locals, flags, arena,
+ PyAST_CompilationModeFromStartToken(start));
PyArena_Free(arena);
return ret;
}
@@ -1752,11 +1759,11 @@
static PyObject *
run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals,
- PyCompilerFlags *flags, PyArena *arena)
+ PyCompilerFlags *flags, PyArena *arena, enum PyCompilationMode mode)
{
PyCodeObject *co;
PyObject *v;
- co = PyAST_Compile(mod, filename, flags, arena);
+ co = PyAST_Compile(mod, filename, flags, arena, mode);
if (co == NULL)
return NULL;
v = PyEval_EvalCode(co, globals, locals);
@@ -1796,6 +1803,25 @@
return v;
}
+enum PyCompilationMode
+PyAST_CompilationModeFromStartToken(int start)
+{
+ switch (start) {
+ case Py_single_input:
+ return PyCompilationMode_Single_Interactive;
+
+ case Py_file_input:
+ return PyCompilationMode_Exec_Module;
+
+ case Py_eval_input:
+ return PyCompilationMode_Eval_Expression;
+
+ }
+
+ Py_FatalError("Unknown start token");
+ assert(0);
+}
+
PyObject *
Py_CompileStringFlags(const char *str, const char *filename, int start,
PyCompilerFlags *flags)
@@ -1812,11 +1838,23 @@
return NULL;
}
if (flags && (flags->cf_flags & PyCF_ONLY_AST)) {
- PyObject *result = PyAST_mod2obj(mod);
+ struct symtable *st = NULL;
+ PyObject *result = NULL;
+
+ st = PySymtable_Build(mod, filename, NULL); /* c.c_future); FIXME */
+ if (st == NULL) {
+ if (!PyErr_Occurred())
+ PyErr_SetString(PyExc_SystemError, "no symtable");
+ return NULL;
+ }
+ result = PyAST_mod2obj(mod, st);
+ PySymtable_Free(st);
PyArena_Free(arena);
return result;
}
- co = PyAST_Compile(mod, filename, flags, arena);
+
+ co = PyAST_Compile(mod, filename, flags, arena,
+ PyAST_CompilationModeFromStartToken(start));
PyArena_Free(arena);
return (PyObject *)co;
}
Modified: python/branches/dmalcolm-ast-optimization-branch/Python/symtable.c
==============================================================================
--- python/branches/dmalcolm-ast-optimization-branch/Python/symtable.c (original)
+++ python/branches/dmalcolm-ast-optimization-branch/Python/symtable.c Tue Nov 23 20:24:02 2010
@@ -310,6 +310,26 @@
return (PySTEntryObject *)v;
}
+/* As per PySymtable_TryLookup, but doesn't set an exception if the key is not
+found (could set an exception for other reasons e.g. out of memory, though) */
+PySTEntryObject *
+PySymtable_TryLookup(struct symtable *st, void *key)
+{
+ PyObject *k, *v;
+
+ k = PyLong_FromVoidPtr(key);
+ if (k == NULL)
+ return NULL;
+ v = PyDict_GetItemWithError(st->st_blocks, k);
+ if (v) {
+ assert(PySTEntry_Check(v));
+ Py_INCREF(v);
+ }
+
+ Py_DECREF(k);
+ return (PySTEntryObject *)v;
+}
+
int
PyST_GetScope(PySTEntryObject *ste, PyObject *name)
{
@@ -1403,6 +1423,12 @@
case Ellipsis_kind:
/* Nothing to do here. */
break;
+ case Specialize_kind:
+ VISIT(st, expr, e->v.Specialize.name);
+ VISIT_SEQ(st, stmt, e->v.Specialize.specialized_body);
+ VISIT(st, expr, e->v.Specialize.specialized_result);
+ VISIT(st, expr, e->v.Specialize.generalized);
+ break;
/* The following exprs can be assignment targets. */
case Attribute_kind:
VISIT(st, expr, e->v.Attribute.value);
More information about the Python-checkins
mailing list