Index: Python/ceval.c =================================================================== --- Python/ceval.c (revision 86497) +++ Python/ceval.c (working copy) @@ -731,6 +731,30 @@ return 0; } +/* Compare two functions, one dynamically looked up, the other the expected + version. + + If they're equal, then we can use specialized bytecode that assumes that + they were. +*/ +static int +is_specializable(PyObject *dynamic, PyObject *expected) +{ +#if 0 + if (!PyFunction_Check(dynamic)) + return 0; + + if (!PyFunction_Check(expected)) + return 0; +#endif + + /* If they're the same object, we have a match: */ + if (dynamic == expected) + return 1; + + return 0; // FIXME! +} + /* Status code for main loop (reason for stack unwind) */ enum why_code { WHY_NOT = 0x0001, /* No error */ @@ -2836,6 +2860,25 @@ if (x != NULL) DISPATCH(); break; + TARGET(JUMP_IF_SPECIALIZABLE) + v = POP(); /* ...expected implementation */ + u = TOP(); /* ...dynamic lookup of implementation */ + + if (is_specializable(u, v)) { + /* Jump to specialized implementation, popping u */ + JUMPTO(oparg); + Py_DECREF(v); + STACKADJ(-1); + Py_DECREF(u); + FAST_DISPATCH(); + } else { + /* Generalized implementation; fall through to next opcode, + keeping u on TOS */ + Py_DECREF(v); + DISPATCH(); + } + break; + TARGET(EXTENDED_ARG) opcode = NEXTOP(); oparg = oparg<<16 | NEXTARG(); Index: Python/pythonrun.c =================================================================== --- Python/pythonrun.c (revision 86497) +++ Python/pythonrun.c (working copy) @@ -62,7 +62,7 @@ static int initstdio(void); static void flush_io(void); static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *, - PyCompilerFlags *, PyArena *); + PyCompilerFlags *, PyArena *, enum PyCompilationMode); static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); @@ -316,6 +316,10 @@ if (!Py_NoSiteFlag) initsite(); /* Module site */ + + /* It should now be safe to run the optimizer when importing modules: */ + _PyOptimizer_Init(); + } void @@ -1146,7 +1150,8 @@ return -1; } d = PyModule_GetDict(m); - v = run_mod(mod, filename, d, d, flags, arena); + v = run_mod(mod, filename, d, d, flags, arena, + PyCompilationMode_Single_Interactive); PyArena_Free(arena); flush_io(); if (v == NULL) { @@ -1691,7 +1696,8 @@ mod = PyParser_ASTFromString(str, "", start, flags, arena); if (mod != NULL) - ret = run_mod(mod, "", globals, locals, flags, arena); + ret = run_mod(mod, "", globals, locals, flags, arena, + PyAST_CompilationModeFromStartToken(start)); PyArena_Free(arena); return ret; } @@ -1714,7 +1720,8 @@ PyArena_Free(arena); return NULL; } - ret = run_mod(mod, filename, globals, locals, flags, arena); + ret = run_mod(mod, filename, globals, locals, flags, arena, + PyAST_CompilationModeFromStartToken(start)); PyArena_Free(arena); return ret; } @@ -1750,11 +1757,11 @@ static PyObject * run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, enum PyCompilationMode mode) { PyCodeObject *co; PyObject *v; - co = PyAST_Compile(mod, filename, flags, arena); + co = PyAST_Compile(mod, filename, flags, arena, mode); if (co == NULL) return NULL; v = PyEval_EvalCode(co, globals, locals); @@ -1794,6 +1801,25 @@ return v; } +enum PyCompilationMode +PyAST_CompilationModeFromStartToken(int start) +{ + switch (start) { + case Py_single_input: + return PyCompilationMode_Single_Interactive; + + case Py_file_input: + return PyCompilationMode_Exec_Module; + + case Py_eval_input: + return PyCompilationMode_Eval_Expression; + + } + + Py_FatalError("Unknown start token"); + assert(0); +} + PyObject * Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) @@ -1810,11 +1836,23 @@ return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { - PyObject *result = PyAST_mod2obj(mod); + struct symtable *st = NULL; + PyObject *result = NULL; + + st = PySymtable_Build(mod, filename, NULL); /* c.c_future); FIXME */ + if (st == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_SystemError, "no symtable"); + return NULL; + } + result = PyAST_mod2obj(mod, st); + PySymtable_Free(st); PyArena_Free(arena); return result; } - co = PyAST_Compile(mod, filename, flags, arena); + + co = PyAST_Compile(mod, filename, flags, arena, + PyAST_CompilationModeFromStartToken(start)); PyArena_Free(arena); return (PyObject *)co; } Index: Python/opcode_targets.h =================================================================== --- Python/opcode_targets.h (revision 86497) +++ Python/opcode_targets.h (working copy) @@ -147,6 +147,7 @@ &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, + &&TARGET_JUMP_IF_SPECIALIZABLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -253,6 +254,5 @@ &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&_unknown_opcode }; Index: Python/import.c =================================================================== --- Python/import.c (revision 86497) +++ Python/import.c (working copy) @@ -1110,7 +1110,8 @@ Py_file_input, 0, 0, &flags, NULL, arena); if (mod) { - co = PyAST_Compile(mod, pathname, NULL, arena); + co = PyAST_Compile(mod, pathname, NULL, arena, + PyCompilationMode_Exec_Module); } PyArena_Free(arena); return co; Index: Python/symtable.c =================================================================== --- Python/symtable.c (revision 86497) +++ Python/symtable.c (working copy) @@ -310,6 +310,26 @@ return (PySTEntryObject *)v; } +/* As per PySymtable_TryLookup, but doesn't set an exception if the key is not +found (could set an exception for other reasons e.g. out of memory, though) */ +PySTEntryObject * +PySymtable_TryLookup(struct symtable *st, void *key) +{ + PyObject *k, *v; + + k = PyLong_FromVoidPtr(key); + if (k == NULL) + return NULL; + v = PyDict_GetItemWithError(st->st_blocks, k); + if (v) { + assert(PySTEntry_Check(v)); + Py_INCREF(v); + } + + Py_DECREF(k); + return (PySTEntryObject *)v; +} + int PyST_GetScope(PySTEntryObject *ste, PyObject *name) { @@ -1403,6 +1423,12 @@ case Ellipsis_kind: /* Nothing to do here. */ break; + case Specialize_kind: + VISIT(st, expr, e->v.Specialize.name); + VISIT_SEQ(st, stmt, e->v.Specialize.specialized_body); + VISIT(st, expr, e->v.Specialize.specialized_result); + VISIT(st, expr, e->v.Specialize.generalized); + break; /* The following exprs can be assignment targets. */ case Attribute_kind: VISIT(st, expr, e->v.Attribute.value); Index: Python/compile.c =================================================================== --- Python/compile.c (revision 86497) +++ Python/compile.c (working copy) @@ -33,6 +33,7 @@ #include "opcode.h" int Py_OptimizeFlag = 0; +static int ready_to_optimize = 0; #define DEFAULT_BLOCK_SIZE 16 #define DEFAULT_BLOCKS 8 @@ -149,6 +150,8 @@ PyArena *c_arena; /* pointer to memory allocation arena */ }; +static mod_ty optimize_mod(struct compiler *, mod_ty, enum PyCompilationMode); + static int compiler_enter_scope(struct compiler *, identifier, void *, int); static void compiler_free(struct compiler *); static basicblock *compiler_new_block(struct compiler *); @@ -180,6 +183,7 @@ static int expr_constant(expr_ty e); static int compiler_with(struct compiler *, stmt_ty); +static int compiler_specialize(struct compiler *, expr_ty); static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords, @@ -257,7 +261,7 @@ PyCodeObject * PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, - PyArena *arena) + PyArena *arena, enum PyCompilationMode mode) { struct compiler c; PyCodeObject *co = NULL; @@ -294,6 +298,10 @@ goto finally; } + mod = optimize_mod(&c, mod, mode); + if (!mod) + goto finally; + co = compiler_mod(&c, mod); finally: @@ -312,11 +320,125 @@ return NULL; mod = PyAST_FromNode(n, NULL, filename, arena); if (mod) - co = PyAST_Compile(mod, filename, NULL, arena); + co = PyAST_Compile(mod, filename, NULL, arena, PyCompilationMode_Exec_Module); PyArena_Free(arena); return co; } +/* + * Overview of the AST optimizer: + * 1. Convert the AST to python objects + * 2. Import __optimizer__ and invoke __optimizer__.optimize_ast() on the + * python form of the tree + * 3. This will returns a potentially modified version of the tree (in + * python form) + * 4. Convert the python objects back to an AST + * 5. Return the resulting AST back, for use by the compiler. + * + * TODO: how does this interract with symbol tables? + * TODO: what about the GIL? + * TODO: which errors should be fatal, if any? how to report errors? + */ +static int within_optimizer = 0; +static mod_ty optimize_mod(struct compiler *c, mod_ty m, enum PyCompilationMode mode) +{ + PyObject *optimizer_module = NULL; + PyObject *optimize_ast_str = NULL; + PyObject *optimize_ast_fn = NULL; + PyObject *py_ast_in = NULL; + PyObject *filename = NULL; + PyObject *py_ast_out = NULL; + mod_ty new_mod = NULL; + PyObject *exc = NULL; + struct symtable *new_st = NULL; + + assert(c); + assert(m); + + /* Avoid running optimizer until the end of Py_InitializeEx: */ + if (!ready_to_optimize) + return m; + + /* Avoid infinite recursion: don't try to optimize the optimizer (or + modules imported during the import of the optimizer): */ + if (within_optimizer) + return m; + within_optimizer = 1; + + /* printf("Optimizing: %s\n", c->c_filename); */ + + /* Import "__optimizer__.optimize_ast" -> optimize_ast_fn: */ + optimizer_module = PyImport_ImportModule("__optimizer__"); + if (!optimizer_module) + goto finally; + + optimize_ast_str= PyUnicode_InternFromString("optimize_ast"); + if (!optimize_ast_str) + goto finally; + + optimize_ast_fn = PyObject_GetAttr(optimizer_module, optimize_ast_str); + if (!optimize_ast_fn) { +//PyErr_SetObject(PyExc_ImportError, optimize_ast_str); + goto finally; + } + + /* Convert the AST repr to Python objects: */ + py_ast_in = PyAST_mod2obj(m, c->c_st); + if (!py_ast_in) + goto finally; + + /* Invoke the "__optimizer__.optimize_ast(ast, filename)": */ + filename = PyUnicode_DecodeFSDefault(c->c_filename); /* FIXME: is this the correct encoding? */ + if (!filename) + goto finally; + + assert(c->c_st); + py_ast_out = PyObject_CallFunctionObjArgs(optimize_ast_fn, + py_ast_in, filename, c->c_st->st_blocks, + NULL); + if (!py_ast_out) + goto finally; + + /* 4. Convert the python objects back to an AST: */ + new_mod = PyAST_obj2mod(py_ast_out, c->c_arena, mode); + if (!new_mod) + goto finally; + + /* 5. Rebuild the symbol table, using the new AST: */ + new_st = PySymtable_Build(new_mod, c->c_filename, c->c_future); + if (new_st == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_SystemError, "no symtable"); + + goto finally; + } + + /* 6. Use the optimizer's version of the AST and the new symtable: */ + m = new_mod; + PySymtable_Free(c->c_st); + c->c_st = new_st; + +finally: + exc = PyErr_Occurred(); + //if (exc) { + // PyErr_PrintEx(0); +#if 0 + PyObject *tb = PyException_GetTraceback(exc); + if (exc && optimize_ast_fn) + PyTraceBack_Print(tb, optimize_ast_fn); +#endif +// } + Py_XDECREF(py_ast_out); + Py_XDECREF(filename); + Py_XDECREF(py_ast_in); + Py_XDECREF(optimize_ast_fn); + Py_XDECREF(optimize_ast_str); + Py_XDECREF(optimizer_module); + within_optimizer = 0; + return m; +} + + static void compiler_free(struct compiler *c) { @@ -861,6 +983,10 @@ return -1; case DELETE_DEREF: return 0; + + case JUMP_IF_SPECIALIZABLE: + return -2; /* -1 if jump not taken */ + default: fprintf(stderr, "opcode = %d\n", opcode); Py_FatalError("opcode_stack_effect()"); @@ -3114,7 +3240,75 @@ return 1; } + +/* + FIXME: need a PEP for this + + This one's unusual, in that it's an expression, but it contains statements + and control flow. +*/ static int +compiler_specialize(struct compiler *c, expr_ty e) +{ + basicblock *specialized, *generalized, *end; + expr_ty call; + identifier id; + PyObject *saved_name = NULL; + + assert(e->kind == Specialize_kind); + + specialized = compiler_new_block(c); + generalized = compiler_new_block(c); + end = compiler_new_block(c); + if (!specialized || !generalized || !end) + return 0; + + /* Do the dynamic name lookup, it will be the new TOS */ + VISIT(c, expr, e->v.Specialize.name); + + /* Push the expected version of the name, it will be the new TOS */ + /* For now, look for a global named "__saved__" + funcname */ + if (e->v.Specialize.name->kind != Name_kind) + return 0; + id = e->v.Specialize.name->v.Name.id; + assert(id); + saved_name = PyUnicode_FromFormat("__saved__%U", id); + if (!saved_name) + return 0; + ADDOP_O(c, LOAD_GLOBAL, saved_name, names); /* takes ownership of the reference */ + + /* JUMP_IF_SPECIALIZABLE; pops the expected, and peeks the dynamic, + leaving dynamically-looked-up fn as TOS */ + ADDOP_JABS(c, JUMP_IF_SPECIALIZABLE, specialized); + + /* The generalized version */ + compiler_use_next_block(c, generalized); + + call = e->v.Specialize.generalized; + if (call->kind != Call_kind) + return 0; + /* Dynamic lookup will be on TOS, don't need to look it up again; go + ahead with args: */ + if (!compiler_call_helper(c, 0, + call->v.Call.args, + call->v.Call.keywords, + call->v.Call.starargs, + call->v.Call.kwargs)) + return 0; + ADDOP_JREL(c, JUMP_FORWARD, end); + + /* The specialized version */ + compiler_use_next_block(c, specialized); + VISIT_SEQ(c, stmt, e->v.Specialize.specialized_body); + VISIT(c, expr, e->v.Specialize.specialized_result); + + /* Fall through to "end" */ + compiler_use_next_block(c, end); + + return 1; +} + +static int compiler_visit_expr(struct compiler *c, expr_ty e) { int i, n; @@ -3195,6 +3389,9 @@ case Ellipsis_kind: ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); break; + case Specialize_kind: + return compiler_specialize(c, e); + break; /* The following exprs can be assignment targets. */ case Attribute_kind: if (e->v.Attribute.ctx != AugStore) @@ -3676,7 +3873,9 @@ d_lineno = i->i_lineno - a->a_lineno; assert(d_bytecode >= 0); - assert(d_lineno >= 0); + /* FIXME: currently the lineno implementation assumes monotonically + increasing line nums, which isn't true with function inlining: */ + /* assert(d_lineno >= 0); */ if(d_bytecode == 0 && d_lineno == 0) return 1; @@ -4082,3 +4281,12 @@ assemble_free(&a); return co; } + +void _PyOptimizer_Init(void) +{ + /* Don't try to optimize until early initialization is done. + This prevents us from e.g. trying to optimize the "encodings" module + before there's a "sys" module: + */ + ready_to_optimize = 1; +} Index: Python/Python-ast.c =================================================================== --- Python/Python-ast.c (revision 86497) +++ Python/Python-ast.c (working copy) @@ -11,10 +11,12 @@ #include "Python.h" #include "Python-ast.h" +#include "symtable.h" static PyTypeObject AST_type; +int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st); static PyTypeObject *mod_type; -static PyObject* ast2obj_mod(void*); +static PyObject* ast2obj_mod(void*, struct symtable *st); static PyTypeObject *Module_type; static char *Module_fields[]={ "body", @@ -36,7 +38,7 @@ "lineno", "col_offset", }; -static PyObject* ast2obj_stmt(void*); +static PyObject* ast2obj_stmt(void*, struct symtable *st); static PyTypeObject *FunctionDef_type; static char *FunctionDef_fields[]={ "name", @@ -150,7 +152,7 @@ "lineno", "col_offset", }; -static PyObject* ast2obj_expr(void*); +static PyObject* ast2obj_expr(void*, struct symtable *st); static PyTypeObject *BoolOp_type; static char *BoolOp_fields[]={ "op", @@ -239,6 +241,13 @@ "s", }; static PyTypeObject *Ellipsis_type; +static PyTypeObject *Specialize_type; +static char *Specialize_fields[]={ + "name", + "specialized_body", + "specialized_result", + "generalized", +}; static PyTypeObject *Attribute_type; static char *Attribute_fields[]={ "value", @@ -274,7 +283,7 @@ static PyTypeObject *expr_context_type; static PyObject *Load_singleton, *Store_singleton, *Del_singleton, *AugLoad_singleton, *AugStore_singleton, *Param_singleton; -static PyObject* ast2obj_expr_context(expr_context_ty); +static PyObject* ast2obj_expr_context(expr_context_ty, struct symtable *st); static PyTypeObject *Load_type; static PyTypeObject *Store_type; static PyTypeObject *Del_type; @@ -282,7 +291,7 @@ static PyTypeObject *AugStore_type; static PyTypeObject *Param_type; static PyTypeObject *slice_type; -static PyObject* ast2obj_slice(void*); +static PyObject* ast2obj_slice(void*, struct symtable *st); static PyTypeObject *Slice_type; static char *Slice_fields[]={ "lower", @@ -299,7 +308,7 @@ }; static PyTypeObject *boolop_type; static PyObject *And_singleton, *Or_singleton; -static PyObject* ast2obj_boolop(boolop_ty); +static PyObject* ast2obj_boolop(boolop_ty, struct symtable *st); static PyTypeObject *And_type; static PyTypeObject *Or_type; static PyTypeObject *operator_type; @@ -307,7 +316,7 @@ *Div_singleton, *Mod_singleton, *Pow_singleton, *LShift_singleton, *RShift_singleton, *BitOr_singleton, *BitXor_singleton, *BitAnd_singleton, *FloorDiv_singleton; -static PyObject* ast2obj_operator(operator_ty); +static PyObject* ast2obj_operator(operator_ty, struct symtable *st); static PyTypeObject *Add_type; static PyTypeObject *Sub_type; static PyTypeObject *Mult_type; @@ -323,7 +332,7 @@ static PyTypeObject *unaryop_type; static PyObject *Invert_singleton, *Not_singleton, *UAdd_singleton, *USub_singleton; -static PyObject* ast2obj_unaryop(unaryop_ty); +static PyObject* ast2obj_unaryop(unaryop_ty, struct symtable *st); static PyTypeObject *Invert_type; static PyTypeObject *Not_type; static PyTypeObject *UAdd_type; @@ -332,7 +341,7 @@ static PyObject *Eq_singleton, *NotEq_singleton, *Lt_singleton, *LtE_singleton, *Gt_singleton, *GtE_singleton, *Is_singleton, *IsNot_singleton, *In_singleton, *NotIn_singleton; -static PyObject* ast2obj_cmpop(cmpop_ty); +static PyObject* ast2obj_cmpop(cmpop_ty, struct symtable *st); static PyTypeObject *Eq_type; static PyTypeObject *NotEq_type; static PyTypeObject *Lt_type; @@ -344,7 +353,7 @@ static PyTypeObject *In_type; static PyTypeObject *NotIn_type; static PyTypeObject *comprehension_type; -static PyObject* ast2obj_comprehension(void*); +static PyObject* ast2obj_comprehension(void*, struct symtable *st); static char *comprehension_fields[]={ "target", "iter", @@ -355,7 +364,7 @@ "lineno", "col_offset", }; -static PyObject* ast2obj_excepthandler(void*); +static PyObject* ast2obj_excepthandler(void*, struct symtable *st); static PyTypeObject *ExceptHandler_type; static char *ExceptHandler_fields[]={ "type", @@ -363,7 +372,7 @@ "body", }; static PyTypeObject *arguments_type; -static PyObject* ast2obj_arguments(void*); +static PyObject* ast2obj_arguments(void*, struct symtable *st); static char *arguments_fields[]={ "args", "vararg", @@ -375,19 +384,19 @@ "kw_defaults", }; static PyTypeObject *arg_type; -static PyObject* ast2obj_arg(void*); +static PyObject* ast2obj_arg(void*, struct symtable *st); static char *arg_fields[]={ "arg", "annotation", }; static PyTypeObject *keyword_type; -static PyObject* ast2obj_keyword(void*); +static PyObject* ast2obj_keyword(void*, struct symtable *st); static char *keyword_fields[]={ "arg", "value", }; static PyTypeObject *alias_type; -static PyObject* ast2obj_alias(void*); +static PyObject* ast2obj_alias(void*, struct symtable *st); static char *alias_fields[]={ "name", "asname", @@ -554,7 +563,7 @@ /* Conversion AST -> Python */ -static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) +static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*, struct symtable*), struct symtable *st) { int i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -562,7 +571,7 @@ if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(asdl_seq_GET(seq, i)); + value = func(asdl_seq_GET(seq, i), st); if (!value) { Py_DECREF(result); return NULL; @@ -572,7 +581,7 @@ return result; } -static PyObject* ast2obj_object(void *o) +static PyObject* ast2obj_object(void *o, struct symtable *st) { if (!o) o = Py_None; @@ -582,7 +591,7 @@ #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(long b) +static PyObject* ast2obj_int(long b, struct symtable *st) { return PyLong_FromLong(b); } @@ -610,7 +619,7 @@ PyObject *s = PyObject_Repr(obj); if (s == NULL) return 1; PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", - PyBytes_AS_STRING(s)); + _PyUnicode_AsString(s)); Py_DECREF(s); return 1; } @@ -748,6 +757,9 @@ if (!Bytes_type) return 0; Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0); if (!Ellipsis_type) return 0; + Specialize_type = make_type("Specialize", expr_type, Specialize_fields, + 4); + if (!Specialize_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3); @@ -1819,6 +1831,40 @@ } expr_ty +Specialize(expr_ty name, asdl_seq * specialized_body, expr_ty + specialized_result, expr_ty generalized, int lineno, int col_offset, + PyArena *arena) +{ + expr_ty p; + if (!name) { + PyErr_SetString(PyExc_ValueError, + "field name is required for Specialize"); + return NULL; + } + if (!specialized_result) { + PyErr_SetString(PyExc_ValueError, + "field specialized_result is required for Specialize"); + return NULL; + } + if (!generalized) { + PyErr_SetString(PyExc_ValueError, + "field generalized is required for Specialize"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Specialize_kind; + p->v.Specialize.name = name; + p->v.Specialize.specialized_body = specialized_body; + p->v.Specialize.specialized_result = specialized_result; + p->v.Specialize.generalized = generalized; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena) { @@ -2137,7 +2183,7 @@ PyObject* -ast2obj_mod(void* _o) +ast2obj_mod(void* _o, struct symtable *st) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -2150,7 +2196,9 @@ case Module_kind: result = PyType_GenericNew(Module_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Module.body, ast2obj_stmt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Module.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; @@ -2159,7 +2207,9 @@ case Interactive_kind: result = PyType_GenericNew(Interactive_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; @@ -2168,7 +2218,9 @@ case Expression_kind: result = PyType_GenericNew(Expression_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Expression.body); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Expression.body, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; @@ -2177,7 +2229,9 @@ case Suite_kind: result = PyType_GenericNew(Suite_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Suite.body, ast2obj_stmt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Suite.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; @@ -2192,7 +2246,7 @@ } PyObject* -ast2obj_stmt(void* _o) +ast2obj_stmt(void* _o, struct symtable *st) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -2205,29 +2259,31 @@ case FunctionDef_kind: result = PyType_GenericNew(FunctionDef_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(o->v.FunctionDef.name); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->v.FunctionDef.name, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "name", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_arguments(o->v.FunctionDef.args); + value = ast2obj_arguments(o->v.FunctionDef.args, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "args", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.FunctionDef.body, ast2obj_stmt); + value = ast2obj_list(o->v.FunctionDef.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.FunctionDef.decorator_list, - ast2obj_expr); + ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "decorator_list", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.FunctionDef.returns); + value = ast2obj_expr(o->v.FunctionDef.returns, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "returns", value) == -1) goto failed; @@ -2236,38 +2292,41 @@ case ClassDef_kind: result = PyType_GenericNew(ClassDef_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(o->v.ClassDef.name); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->v.ClassDef.name, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "name", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.ClassDef.bases, ast2obj_expr); + value = ast2obj_list(o->v.ClassDef.bases, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "bases", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword); + value = ast2obj_list(o->v.ClassDef.keywords, ast2obj_keyword, + st); if (!value) goto failed; if (PyObject_SetAttrString(result, "keywords", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.ClassDef.starargs); + value = ast2obj_expr(o->v.ClassDef.starargs, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "starargs", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.ClassDef.kwargs); + value = ast2obj_expr(o->v.ClassDef.kwargs, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "kwargs", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt); + value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ClassDef.decorator_list, - ast2obj_expr); + ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "decorator_list", value) == -1) @@ -2277,7 +2336,9 @@ case Return_kind: result = PyType_GenericNew(Return_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Return.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Return.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -2286,7 +2347,9 @@ case Delete_kind: result = PyType_GenericNew(Delete_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Delete.targets, ast2obj_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Delete.targets, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "targets", value) == -1) goto failed; @@ -2295,12 +2358,14 @@ case Assign_kind: result = PyType_GenericNew(Assign_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Assign.targets, ast2obj_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Assign.targets, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "targets", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Assign.value); + value = ast2obj_expr(o->v.Assign.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -2309,17 +2374,19 @@ case AugAssign_kind: result = PyType_GenericNew(AugAssign_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.AugAssign.target); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.AugAssign.target, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "target", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(o->v.AugAssign.op); + value = ast2obj_operator(o->v.AugAssign.op, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "op", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.AugAssign.value); + value = ast2obj_expr(o->v.AugAssign.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -2328,22 +2395,24 @@ case For_kind: result = PyType_GenericNew(For_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.For.target); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.For.target, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "target", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.For.iter); + value = ast2obj_expr(o->v.For.iter, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "iter", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.For.body, ast2obj_stmt); + value = ast2obj_list(o->v.For.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.For.orelse, ast2obj_stmt); + value = ast2obj_list(o->v.For.orelse, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "orelse", value) == -1) goto failed; @@ -2352,17 +2421,19 @@ case While_kind: result = PyType_GenericNew(While_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.While.test); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.While.test, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "test", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.While.body, ast2obj_stmt); + value = ast2obj_list(o->v.While.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.While.orelse, ast2obj_stmt); + value = ast2obj_list(o->v.While.orelse, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "orelse", value) == -1) goto failed; @@ -2371,17 +2442,19 @@ case If_kind: result = PyType_GenericNew(If_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.If.test); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.If.test, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "test", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.If.body, ast2obj_stmt); + value = ast2obj_list(o->v.If.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.If.orelse, ast2obj_stmt); + value = ast2obj_list(o->v.If.orelse, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "orelse", value) == -1) goto failed; @@ -2390,18 +2463,20 @@ case With_kind: result = PyType_GenericNew(With_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.With.context_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.With.context_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "context_expr", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.With.optional_vars); + value = ast2obj_expr(o->v.With.optional_vars, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "optional_vars", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.With.body, ast2obj_stmt); + value = ast2obj_list(o->v.With.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; @@ -2410,12 +2485,14 @@ case Raise_kind: result = PyType_GenericNew(Raise_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Raise.exc); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Raise.exc, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "exc", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Raise.cause); + value = ast2obj_expr(o->v.Raise.cause, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "cause", value) == -1) goto failed; @@ -2424,18 +2501,20 @@ case TryExcept_kind: result = PyType_GenericNew(TryExcept_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.TryExcept.body, ast2obj_stmt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.TryExcept.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.TryExcept.handlers, - ast2obj_excepthandler); + ast2obj_excepthandler, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "handlers", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.TryExcept.orelse, ast2obj_stmt); + value = ast2obj_list(o->v.TryExcept.orelse, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "orelse", value) == -1) goto failed; @@ -2444,12 +2523,15 @@ case TryFinally_kind: result = PyType_GenericNew(TryFinally_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.TryFinally.body, ast2obj_stmt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.TryFinally.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.TryFinally.finalbody, ast2obj_stmt); + value = ast2obj_list(o->v.TryFinally.finalbody, ast2obj_stmt, + st); if (!value) goto failed; if (PyObject_SetAttrString(result, "finalbody", value) == -1) goto failed; @@ -2458,12 +2540,14 @@ case Assert_kind: result = PyType_GenericNew(Assert_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Assert.test); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Assert.test, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "test", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Assert.msg); + value = ast2obj_expr(o->v.Assert.msg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "msg", value) == -1) goto failed; @@ -2472,7 +2556,9 @@ case Import_kind: result = PyType_GenericNew(Import_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Import.names, ast2obj_alias); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Import.names, ast2obj_alias, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "names", value) == -1) goto failed; @@ -2481,17 +2567,19 @@ case ImportFrom_kind: result = PyType_GenericNew(ImportFrom_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(o->v.ImportFrom.module); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->v.ImportFrom.module, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "module", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.ImportFrom.names, ast2obj_alias); + value = ast2obj_list(o->v.ImportFrom.names, ast2obj_alias, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "names", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_int(o->v.ImportFrom.level); + value = ast2obj_int(o->v.ImportFrom.level, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "level", value) == -1) goto failed; @@ -2500,7 +2588,9 @@ case Global_kind: result = PyType_GenericNew(Global_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Global.names, ast2obj_identifier); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Global.names, ast2obj_identifier, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "names", value) == -1) goto failed; @@ -2509,7 +2599,10 @@ case Nonlocal_kind: result = PyType_GenericNew(Nonlocal_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier, + st); if (!value) goto failed; if (PyObject_SetAttrString(result, "names", value) == -1) goto failed; @@ -2518,7 +2611,9 @@ case Expr_kind: result = PyType_GenericNew(Expr_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Expr.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Expr.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -2527,22 +2622,28 @@ case Pass_kind: result = PyType_GenericNew(Pass_type, NULL, NULL); if (!result) goto failed; + if (!obj2ast_write_scope(o, result, st)) + goto failed; break; case Break_kind: result = PyType_GenericNew(Break_type, NULL, NULL); if (!result) goto failed; + if (!obj2ast_write_scope(o, result, st)) + goto failed; break; case Continue_kind: result = PyType_GenericNew(Continue_type, NULL, NULL); if (!result) goto failed; + if (!obj2ast_write_scope(o, result, st)) + goto failed; break; } - value = ast2obj_int(o->lineno); + value = ast2obj_int(o->lineno, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "lineno", value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(o->col_offset); + value = ast2obj_int(o->col_offset, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "col_offset", value) < 0) goto failed; @@ -2555,7 +2656,7 @@ } PyObject* -ast2obj_expr(void* _o) +ast2obj_expr(void* _o, struct symtable *st) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -2568,12 +2669,14 @@ case BoolOp_kind: result = PyType_GenericNew(BoolOp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_boolop(o->v.BoolOp.op); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_boolop(o->v.BoolOp.op, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "op", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr); + value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "values", value) == -1) goto failed; @@ -2582,17 +2685,19 @@ case BinOp_kind: result = PyType_GenericNew(BinOp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.BinOp.left); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.BinOp.left, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "left", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_operator(o->v.BinOp.op); + value = ast2obj_operator(o->v.BinOp.op, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "op", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.BinOp.right); + value = ast2obj_expr(o->v.BinOp.right, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "right", value) == -1) goto failed; @@ -2601,12 +2706,14 @@ case UnaryOp_kind: result = PyType_GenericNew(UnaryOp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_unaryop(o->v.UnaryOp.op); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_unaryop(o->v.UnaryOp.op, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "op", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.UnaryOp.operand); + value = ast2obj_expr(o->v.UnaryOp.operand, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "operand", value) == -1) goto failed; @@ -2615,12 +2722,14 @@ case Lambda_kind: result = PyType_GenericNew(Lambda_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_arguments(o->v.Lambda.args); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_arguments(o->v.Lambda.args, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "args", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Lambda.body); + value = ast2obj_expr(o->v.Lambda.body, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; @@ -2629,17 +2738,19 @@ case IfExp_kind: result = PyType_GenericNew(IfExp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.IfExp.test); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.IfExp.test, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "test", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.IfExp.body); + value = ast2obj_expr(o->v.IfExp.body, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.IfExp.orelse); + value = ast2obj_expr(o->v.IfExp.orelse, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "orelse", value) == -1) goto failed; @@ -2648,12 +2759,14 @@ case Dict_kind: result = PyType_GenericNew(Dict_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Dict.keys, ast2obj_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Dict.keys, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "keys", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.Dict.values, ast2obj_expr); + value = ast2obj_list(o->v.Dict.values, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "values", value) == -1) goto failed; @@ -2662,7 +2775,9 @@ case Set_kind: result = PyType_GenericNew(Set_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Set.elts, ast2obj_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Set.elts, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "elts", value) == -1) goto failed; @@ -2671,13 +2786,15 @@ case ListComp_kind: result = PyType_GenericNew(ListComp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.ListComp.elt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.ListComp.elt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "elt", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.ListComp.generators, - ast2obj_comprehension); + ast2obj_comprehension, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "generators", value) == -1) goto failed; @@ -2686,13 +2803,15 @@ case SetComp_kind: result = PyType_GenericNew(SetComp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.SetComp.elt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.SetComp.elt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "elt", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.SetComp.generators, - ast2obj_comprehension); + ast2obj_comprehension, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "generators", value) == -1) goto failed; @@ -2701,18 +2820,20 @@ case DictComp_kind: result = PyType_GenericNew(DictComp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.DictComp.key); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.DictComp.key, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "key", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.DictComp.value); + value = ast2obj_expr(o->v.DictComp.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.DictComp.generators, - ast2obj_comprehension); + ast2obj_comprehension, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "generators", value) == -1) goto failed; @@ -2721,13 +2842,15 @@ case GeneratorExp_kind: result = PyType_GenericNew(GeneratorExp_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.GeneratorExp.elt); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.GeneratorExp.elt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "elt", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.GeneratorExp.generators, - ast2obj_comprehension); + ast2obj_comprehension, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "generators", value) == -1) goto failed; @@ -2736,7 +2859,9 @@ case Yield_kind: result = PyType_GenericNew(Yield_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Yield.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Yield.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -2745,7 +2870,9 @@ case Compare_kind: result = PyType_GenericNew(Compare_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Compare.left); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Compare.left, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "left", value) == -1) goto failed; @@ -2755,13 +2882,14 @@ value = PyList_New(n); if (!value) goto failed; for(i = 0; i < n; i++) - PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i))); + PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i), st)); } if (!value) goto failed; if (PyObject_SetAttrString(result, "ops", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr); + value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr, + st); if (!value) goto failed; if (PyObject_SetAttrString(result, "comparators", value) == -1) goto failed; @@ -2770,27 +2898,29 @@ case Call_kind: result = PyType_GenericNew(Call_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Call.func); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Call.func, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "func", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.Call.args, ast2obj_expr); + value = ast2obj_list(o->v.Call.args, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "args", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword); + value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "keywords", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Call.starargs); + value = ast2obj_expr(o->v.Call.starargs, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "starargs", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Call.kwargs); + value = ast2obj_expr(o->v.Call.kwargs, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "kwargs", value) == -1) goto failed; @@ -2799,7 +2929,9 @@ case Num_kind: result = PyType_GenericNew(Num_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_object(o->v.Num.n); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_object(o->v.Num.n, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "n", value) == -1) goto failed; @@ -2808,7 +2940,9 @@ case Str_kind: result = PyType_GenericNew(Str_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_string(o->v.Str.s); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_string(o->v.Str.s, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "s", value) == -1) goto failed; @@ -2817,7 +2951,9 @@ case Bytes_kind: result = PyType_GenericNew(Bytes_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_string(o->v.Bytes.s); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_string(o->v.Bytes.s, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "s", value) == -1) goto failed; @@ -2826,21 +2962,54 @@ case Ellipsis_kind: result = PyType_GenericNew(Ellipsis_type, NULL, NULL); if (!result) goto failed; + if (!obj2ast_write_scope(o, result, st)) + goto failed; break; + case Specialize_kind: + result = PyType_GenericNew(Specialize_type, NULL, NULL); + if (!result) goto failed; + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Specialize.name, st); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "name", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(o->v.Specialize.specialized_body, + ast2obj_stmt, st); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "specialized_body", value) + == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.Specialize.specialized_result, st); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "specialized_result", value) + == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.Specialize.generalized, st); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "generalized", value) == -1) + goto failed; + Py_DECREF(value); + break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Attribute.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Attribute.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(o->v.Attribute.attr); + value = ast2obj_identifier(o->v.Attribute.attr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "attr", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(o->v.Attribute.ctx); + value = ast2obj_expr_context(o->v.Attribute.ctx, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ctx", value) == -1) goto failed; @@ -2849,17 +3018,19 @@ case Subscript_kind: result = PyType_GenericNew(Subscript_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Subscript.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Subscript.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_slice(o->v.Subscript.slice); + value = ast2obj_slice(o->v.Subscript.slice, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "slice", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(o->v.Subscript.ctx); + value = ast2obj_expr_context(o->v.Subscript.ctx, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ctx", value) == -1) goto failed; @@ -2868,12 +3039,14 @@ case Starred_kind: result = PyType_GenericNew(Starred_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Starred.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Starred.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(o->v.Starred.ctx); + value = ast2obj_expr_context(o->v.Starred.ctx, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ctx", value) == -1) goto failed; @@ -2882,12 +3055,14 @@ case Name_kind: result = PyType_GenericNew(Name_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(o->v.Name.id); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->v.Name.id, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "id", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(o->v.Name.ctx); + value = ast2obj_expr_context(o->v.Name.ctx, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ctx", value) == -1) goto failed; @@ -2896,12 +3071,14 @@ case List_kind: result = PyType_GenericNew(List_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.List.elts, ast2obj_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.List.elts, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "elts", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(o->v.List.ctx); + value = ast2obj_expr_context(o->v.List.ctx, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ctx", value) == -1) goto failed; @@ -2910,24 +3087,26 @@ case Tuple_kind: result = PyType_GenericNew(Tuple_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "elts", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr_context(o->v.Tuple.ctx); + value = ast2obj_expr_context(o->v.Tuple.ctx, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ctx", value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(o->lineno); + value = ast2obj_int(o->lineno, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "lineno", value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(o->col_offset); + value = ast2obj_int(o->col_offset, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "col_offset", value) < 0) goto failed; @@ -2939,7 +3118,7 @@ return NULL; } -PyObject* ast2obj_expr_context(expr_context_ty o) +PyObject* ast2obj_expr_context(expr_context_ty o, struct symtable *st) { switch(o) { case Load: @@ -2967,7 +3146,7 @@ } } PyObject* -ast2obj_slice(void* _o) +ast2obj_slice(void* _o, struct symtable *st) { slice_ty o = (slice_ty)_o; PyObject *result = NULL, *value = NULL; @@ -2980,17 +3159,19 @@ case Slice_kind: result = PyType_GenericNew(Slice_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Slice.lower); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Slice.lower, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "lower", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Slice.upper); + value = ast2obj_expr(o->v.Slice.upper, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "upper", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->v.Slice.step); + value = ast2obj_expr(o->v.Slice.step, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "step", value) == -1) goto failed; @@ -2999,7 +3180,9 @@ case ExtSlice_kind: result = PyType_GenericNew(ExtSlice_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->v.ExtSlice.dims, ast2obj_slice, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "dims", value) == -1) goto failed; @@ -3008,7 +3191,9 @@ case Index_kind: result = PyType_GenericNew(Index_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.Index.value); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.Index.value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -3022,7 +3207,7 @@ return NULL; } -PyObject* ast2obj_boolop(boolop_ty o) +PyObject* ast2obj_boolop(boolop_ty o, struct symtable *st) { switch(o) { case And: @@ -3037,7 +3222,7 @@ return NULL; } } -PyObject* ast2obj_operator(operator_ty o) +PyObject* ast2obj_operator(operator_ty o, struct symtable *st) { switch(o) { case Add: @@ -3082,7 +3267,7 @@ return NULL; } } -PyObject* ast2obj_unaryop(unaryop_ty o) +PyObject* ast2obj_unaryop(unaryop_ty o, struct symtable *st) { switch(o) { case Invert: @@ -3103,7 +3288,7 @@ return NULL; } } -PyObject* ast2obj_cmpop(cmpop_ty o) +PyObject* ast2obj_cmpop(cmpop_ty o, struct symtable *st) { switch(o) { case Eq: @@ -3143,7 +3328,7 @@ } } PyObject* -ast2obj_comprehension(void* _o) +ast2obj_comprehension(void* _o, struct symtable *st) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3154,17 +3339,19 @@ result = PyType_GenericNew(comprehension_type, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(o->target); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->target, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "target", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->iter); + value = ast2obj_expr(o->iter, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "iter", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->ifs, ast2obj_expr); + value = ast2obj_list(o->ifs, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "ifs", value) == -1) goto failed; @@ -3177,7 +3364,7 @@ } PyObject* -ast2obj_excepthandler(void* _o) +ast2obj_excepthandler(void* _o, struct symtable *st) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3190,29 +3377,31 @@ case ExceptHandler_kind: result = PyType_GenericNew(ExceptHandler_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.ExceptHandler.type); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_expr(o->v.ExceptHandler.type, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "type", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(o->v.ExceptHandler.name); + value = ast2obj_identifier(o->v.ExceptHandler.name, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "name", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt); + value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); break; } - value = ast2obj_int(o->lineno); + value = ast2obj_int(o->lineno, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "lineno", value) < 0) goto failed; Py_DECREF(value); - value = ast2obj_int(o->col_offset); + value = ast2obj_int(o->col_offset, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "col_offset", value) < 0) goto failed; @@ -3225,7 +3414,7 @@ } PyObject* -ast2obj_arguments(void* _o) +ast2obj_arguments(void* _o, struct symtable *st) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3236,42 +3425,44 @@ result = PyType_GenericNew(arguments_type, NULL, NULL); if (!result) return NULL; - value = ast2obj_list(o->args, ast2obj_arg); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_list(o->args, ast2obj_arg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "args", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(o->vararg); + value = ast2obj_identifier(o->vararg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "vararg", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->varargannotation); + value = ast2obj_expr(o->varargannotation, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "varargannotation", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->kwonlyargs, ast2obj_arg); + value = ast2obj_list(o->kwonlyargs, ast2obj_arg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "kwonlyargs", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(o->kwarg); + value = ast2obj_identifier(o->kwarg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "kwarg", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->kwargannotation); + value = ast2obj_expr(o->kwargannotation, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "kwargannotation", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->defaults, ast2obj_expr); + value = ast2obj_list(o->defaults, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "defaults", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->kw_defaults, ast2obj_expr); + value = ast2obj_list(o->kw_defaults, ast2obj_expr, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "kw_defaults", value) == -1) goto failed; @@ -3284,7 +3475,7 @@ } PyObject* -ast2obj_arg(void* _o) +ast2obj_arg(void* _o, struct symtable *st) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3295,12 +3486,14 @@ result = PyType_GenericNew(arg_type, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(o->arg); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->arg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "arg", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->annotation); + value = ast2obj_expr(o->annotation, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "annotation", value) == -1) goto failed; @@ -3313,7 +3506,7 @@ } PyObject* -ast2obj_keyword(void* _o) +ast2obj_keyword(void* _o, struct symtable *st) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3324,12 +3517,14 @@ result = PyType_GenericNew(keyword_type, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(o->arg); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->arg, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "arg", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_expr(o->value); + value = ast2obj_expr(o->value, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "value", value) == -1) goto failed; @@ -3342,7 +3537,7 @@ } PyObject* -ast2obj_alias(void* _o) +ast2obj_alias(void* _o, struct symtable *st) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3353,12 +3548,14 @@ result = PyType_GenericNew(alias_type, NULL, NULL); if (!result) return NULL; - value = ast2obj_identifier(o->name); + if (!obj2ast_write_scope(o, result, st)) + goto failed; + value = ast2obj_identifier(o->name, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "name", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(o->asname); + value = ast2obj_identifier(o->asname, st); if (!value) goto failed; if (PyObject_SetAttrString(result, "asname", value) == -1) goto failed; @@ -3516,7 +3713,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -4714,7 +4911,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -5561,6 +5758,82 @@ if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)Specialize_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty name; + asdl_seq* specialized_body; + expr_ty specialized_result; + expr_ty generalized; + + if (PyObject_HasAttrString(obj, "name")) { + int res; + tmp = PyObject_GetAttrString(obj, "name"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &name, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from Specialize"); + return 1; + } + if (PyObject_HasAttrString(obj, "specialized_body")) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "specialized_body"); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "Specialize field \"specialized_body\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + specialized_body = asdl_seq_new(len, arena); + if (specialized_body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(specialized_body, i, value); + } + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"specialized_body\" missing from Specialize"); + return 1; + } + if (PyObject_HasAttrString(obj, "specialized_result")) { + int res; + tmp = PyObject_GetAttrString(obj, "specialized_result"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &specialized_result, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"specialized_result\" missing from Specialize"); + return 1; + } + if (PyObject_HasAttrString(obj, "generalized")) { + int res; + tmp = PyObject_GetAttrString(obj, "generalized"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &generalized, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"generalized\" missing from Specialize"); + return 1; + } + *out = Specialize(name, specialized_body, specialized_result, + generalized, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type); if (isinstance == -1) { return 1; @@ -5832,7 +6105,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -5895,7 +6168,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6020,7 +6293,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of slice, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of slice, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6051,7 +6324,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6162,7 +6435,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6209,7 +6482,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6304,7 +6577,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of cmpop, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of cmpop, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6475,7 +6748,7 @@ tmp = PyObject_Repr(obj); if (tmp == NULL) goto failed; - PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %.400s", PyBytes_AS_STRING(tmp)); + PyErr_Format(PyExc_TypeError, "expected some sort of excepthandler, but got %.400s", _PyUnicode_AsString(tmp)); failed: Py_XDECREF(tmp); return 1; @@ -6869,6 +7142,8 @@ NULL; if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) return NULL; + if (PyDict_SetItemString(d, "Specialize", (PyObject*)Specialize_type) < + 0) return NULL; if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0) return NULL; if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < @@ -6979,10 +7254,24 @@ } -PyObject* PyAST_mod2obj(mod_ty t) +int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st) { + PySTEntryObject * ste; + if (st) { + ste = PySymtable_TryLookup(st, ast_obj); + if (ste) { + /* This AST node has a symtable entry: */ + if (PyObject_SetAttrString(py_obj, "ste", (PyObject*)ste) == -1) + return 0; + } + } + return 1; +} + +PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st) +{ init_types(); - return ast2obj_mod(t); + return ast2obj_mod(t, st); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ Index: Python/bltinmodule.c =================================================================== --- Python/bltinmodule.c (revision 86497) +++ Python/bltinmodule.c (working copy) @@ -592,7 +592,7 @@ goto error; } result = (PyObject*)PyAST_Compile(mod, filename, - &cf, arena); + &cf, arena, mode); PyArena_Free(arena); } goto finally; Index: Include/opcode.h =================================================================== --- Include/opcode.h (revision 86497) +++ Include/opcode.h (working copy) @@ -140,7 +140,9 @@ #define SET_ADD 146 #define MAP_ADD 147 +#define JUMP_IF_SPECIALIZABLE 148 /* abs jump */ + /* EXCEPT_HANDLER is a special, implicit block type which is created when entering an except handler. It is not an opcode but we define it here as we want it to be available to both frameobject.c and ceval.c, while Index: Include/symtable.h =================================================================== --- Include/symtable.h (revision 86497) +++ Include/symtable.h (working copy) @@ -62,6 +62,7 @@ PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *, PyFutureFeatures *); PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); +PyAPI_FUNC(PySTEntryObject *) PySymtable_TryLookup(struct symtable *, void *); PyAPI_FUNC(void) PySymtable_Free(struct symtable *); Index: Include/compile.h =================================================================== --- Include/compile.h (revision 86497) +++ Include/compile.h (working copy) @@ -19,6 +19,12 @@ int ff_lineno; /* line number of last future statement */ } PyFutureFeatures; +enum PyCompilationMode { + PyCompilationMode_Exec_Module = 0, + PyCompilationMode_Eval_Expression = 1, + PyCompilationMode_Single_Interactive = 2, +}; + #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" @@ -29,11 +35,15 @@ #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" struct _mod; /* Declare the existence of this type */ +PyAPI_FUNC(enum PyCompilationMode) PyAST_CompilationModeFromStartToken(int); + PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *, - PyCompilerFlags *, PyArena *); + PyCompilerFlags *, PyArena *, enum PyCompilationMode); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); +PyAPI_FUNC(void) _PyOptimizer_Init(void); + #ifdef __cplusplus } #endif Index: Include/Python-ast.h =================================================================== --- Include/Python-ast.h (revision 86497) +++ Include/Python-ast.h (working copy) @@ -2,6 +2,7 @@ #include "asdl.h" +struct symtable; typedef struct _mod *mod_ty; typedef struct _stmt *stmt_ty; @@ -186,8 +187,8 @@ SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, Yield_kind=12, Compare_kind=13, Call_kind=14, Num_kind=15, Str_kind=16, Bytes_kind=17, Ellipsis_kind=18, - Attribute_kind=19, Subscript_kind=20, Starred_kind=21, - Name_kind=22, List_kind=23, Tuple_kind=24}; + Specialize_kind=19, Attribute_kind=20, Subscript_kind=21, + Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25}; struct _expr { enum _expr_kind kind; union { @@ -279,6 +280,13 @@ } Bytes; struct { + expr_ty name; + asdl_seq *specialized_body; + expr_ty specialized_result; + expr_ty generalized; + } Specialize; + + struct { expr_ty value; identifier attr; expr_context_ty ctx; @@ -505,6 +513,10 @@ expr_ty _Py_Bytes(string s, int lineno, int col_offset, PyArena *arena); #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); +#define Specialize(a0, a1, a2, a3, a4, a5, a6) _Py_Specialize(a0, a1, a2, a3, a4, a5, a6) +expr_ty _Py_Specialize(expr_ty name, asdl_seq * specialized_body, expr_ty + specialized_result, expr_ty generalized, int lineno, int + col_offset, PyArena *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena); @@ -548,6 +560,6 @@ #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); -PyObject* PyAST_mod2obj(mod_ty t); +PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st); mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); int PyAST_Check(PyObject* obj); Index: Parser/Python.asdl =================================================================== --- Parser/Python.asdl (revision 86497) +++ Parser/Python.asdl (working copy) @@ -72,6 +72,12 @@ | Ellipsis -- other literals? bools? + -- Generated by the optimizer: + | Specialize(expr name, + stmt* specialized_body, + expr specialized_result, + expr generalized) -- must be a Call + -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) | Subscript(expr value, slice slice, expr_context ctx) Index: Parser/asdl_c.py =================================================================== --- Parser/asdl_c.py (revision 86497) +++ Parser/asdl_c.py (working copy) @@ -375,7 +375,7 @@ # there's really nothing more we can do if this fails ... self.emit("if (tmp == NULL) goto failed;", 1) error = "expected some sort of %s, but got %%.400s" % name - format = "PyErr_Format(PyExc_TypeError, \"%s\", PyBytes_AS_STRING(tmp));" + format = "PyErr_Format(PyExc_TypeError, \"%s\", _PyUnicode_AsString(tmp));" self.emit(format % error, 1, reflow=False) self.emit("failed:", 0) self.emit("Py_XDECREF(tmp);", 1) @@ -551,7 +551,7 @@ def visitProduct(self, prod, name): self.emit("static PyTypeObject *%s_type;" % name, 0) - self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(void*, struct symtable *st);" % name, 0) if prod.fields: self.emit("static char *%s_fields[]={" % name,0) for f in prod.fields: @@ -573,7 +573,7 @@ tnames.append(str(t.name)+"_singleton") tnames = ", *".join(tnames) self.emit("static PyObject *%s;" % tnames, 0) - self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(%s, struct symtable *st);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -749,7 +749,7 @@ /* Conversion AST -> Python */ -static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) +static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*, struct symtable*), struct symtable *st) { int i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -757,7 +757,7 @@ if (!result) return NULL; for (i = 0; i < n; i++) { - value = func(asdl_seq_GET(seq, i)); + value = func(asdl_seq_GET(seq, i), st); if (!value) { Py_DECREF(result); return NULL; @@ -767,7 +767,7 @@ return result; } -static PyObject* ast2obj_object(void *o) +static PyObject* ast2obj_object(void *o, struct symtable *st) { if (!o) o = Py_None; @@ -777,7 +777,7 @@ #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(long b) +static PyObject* ast2obj_int(long b, struct symtable *st) { return PyLong_FromLong(b); } @@ -805,7 +805,7 @@ PyObject *s = PyObject_Repr(obj); if (s == NULL) return 1; PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", - PyBytes_AS_STRING(s)); + _PyUnicode_AsString(s)); Py_DECREF(s); return 1; } @@ -958,7 +958,7 @@ def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(void* _o)" % (name), 0) + self.emit("ast2obj_%s(void* _o, struct symtable *st)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -988,7 +988,7 @@ self.visitConstructor(t, i + 1, name) self.emit("}", 1) for a in sum.attributes: - self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) + self.emit("value = ast2obj_%s(o->%s, st);" % (a.type, a.name), 1) self.emit("if (!value) goto failed;", 1) self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1) self.emit('goto failed;', 2) @@ -996,7 +996,7 @@ self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(%s_ty o, struct symtable *st)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1015,6 +1015,8 @@ self.func_begin(name) self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1); self.emit("if (!result) return NULL;", 1) + self.emit("if (!obj2ast_write_scope(o, result, st))", 1) + self.emit("goto failed;", 2) for field in prod.fields: self.visitField(field, name, 1, True) self.func_end() @@ -1023,6 +1025,8 @@ self.emit("case %s_kind:" % cons.name, 1) self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2); self.emit("if (!result) goto failed;", 2) + self.emit("if (!obj2ast_write_scope(o, result, st))", 2) + self.emit("goto failed;", 3) for f in cons.fields: self.visitField(f, cons.name, 2, False) self.emit("break;", 2) @@ -1064,23 +1068,37 @@ self.emit("if (!value) goto failed;", depth+1) self.emit("for(i = 0; i < n; i++)", depth+1) # This cannot fail, so no need for error handling - self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value, + self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i), st));" % value, depth+2, reflow=False) self.emit("}", depth) else: - self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth) + self.emit("value = ast2obj_list(%s, ast2obj_%s, st);" % (value, field.type), depth) else: ctype = get_c_type(field.type) - self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False) + self.emit("value = ast2obj_%s(%s, st);" % (field.type, value), depth, reflow=False) class PartingShots(StaticVisitor): CODE = """ -PyObject* PyAST_mod2obj(mod_ty t) +int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st) { + PySTEntryObject * ste; + if (st) { + ste = PySymtable_TryLookup(st, ast_obj); + if (ste) { + /* This AST node has a symtable entry: */ + if (PyObject_SetAttrString(py_obj, "ste", (PyObject*)ste) == -1) + return 0; + } + } + return 1; +} + +PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st) +{ init_types(); - return ast2obj_mod(t); + return ast2obj_mod(t, st); } /* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ @@ -1151,12 +1169,13 @@ f = open(p, "w") f.write(auto_gen_msg) f.write('#include "asdl.h"\n\n') + f.write("struct symtable;\n") c = ChainOfVisitors(TypeDefVisitor(f), StructVisitor(f), PrototypeVisitor(f), ) c.visit(mod) - f.write("PyObject* PyAST_mod2obj(mod_ty t);\n") + f.write("PyObject* PyAST_mod2obj(mod_ty t, struct symtable *st);\n") f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") f.write("int PyAST_Check(PyObject* obj);\n") f.close() @@ -1168,8 +1187,10 @@ f.write(c_file_msg % parse_version(mod)) f.write('#include "Python.h"\n') f.write('#include "%s-ast.h"\n' % mod.name) + f.write('#include "symtable.h"\n') f.write('\n') f.write("static PyTypeObject AST_type;\n") + f.write("int obj2ast_write_scope(void *ast_obj, PyObject *py_obj, struct symtable *st);\n"); v = ChainOfVisitors( PyTypesDeclareVisitor(f), PyTypesVisitor(f), Index: Lib/opcode.py =================================================================== --- Lib/opcode.py (revision 86497) +++ Lib/opcode.py (working copy) @@ -174,6 +174,8 @@ def_op('SET_ADD', 146) def_op('MAP_ADD', 147) +jabs_op('JUMP_IF_SPECIALIZABLE', 148) + def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 Index: Lib/test/test_compile.py =================================================================== --- Lib/test/test_compile.py (revision 86497) +++ Lib/test/test_compile.py (working copy) @@ -433,6 +433,13 @@ ast.body = [_ast.BoolOp()] self.assertRaises(TypeError, compile, ast, '', 'exec') + # raise exception when expr is not an expression: + self.assertRaises(TypeError, compile, + _ast.Module(body=[_ast.Expr(_ast.Assign(lineno=0, + col_offset=0), + lineno=0, col_offset=0)], + lineno=0), + 'test', 'exec') def test_main(): support.run_unittest(TestSpecifics) Index: Lib/test/test_optimize.py =================================================================== --- Lib/test/test_optimize.py (revision 0) +++ Lib/test/test_optimize.py (revision 0) @@ -0,0 +1,426 @@ +import dis +import re +import sys +from io import StringIO +import unittest +import ast +import types + +def disassemble(func): + f = StringIO() + tmp = sys.stdout + sys.stdout = f + dis.dis(func) + sys.stdout = tmp + result = f.getvalue() + f.close() + return result + +def dis_single(line): + return disassemble(compile(line, '', 'single')) + +def compile_fn(src, fnname='f'): + """ + Compile src to a Module, return an (ast.FunctionDef, ast.Module) pair + where the former is the named function + """ + node = ast.parse(src) + #print(ast.dump(node)) + for f in node.body: + if isinstance(f, ast.FunctionDef): + if f.name == fnname: + return f, node + raise ValueError('Could not find function: %r' % fnname) + +def get_code_for_fn(co, fnname): + for const in co.co_consts: + if isinstance(const, types.CodeType): + if const.co_name == fnname: + return const + raise ValueError('code for %r not found' % fnname) + + +class TestFramework(unittest.TestCase): + # Ensure that invoking the optimizer is working: + def test_eval(self): + self.assertEqual(eval('42'), 42) + +class TestInlining(unittest.TestCase): + def assertIsInlinable(self, src, fnname='f'): + from __optimizer__ import fn_is_inlinable + fn, mod = compile_fn(src) + self.assert_(fn_is_inlinable(fn, mod), + msg='Unexpectedly not inlinable: %s\n%r' % (src, ast.dump(fn))) + + def assertIsNotInlinable(self, src, fnname='f'): + from __optimizer__ import fn_is_inlinable + fn, mod = compile_fn(src) + self.assert_(not fn_is_inlinable(fn, mod), + msg='Unexpectedly inlinable: %s\n%r' % (src, ast.dump(fn))) + + def compile_to_code(self, src, fnname): + # Compile the given source code, returning the code object for the given function name + co = compile(src, 'test_optimize.py', 'exec') + return get_code_for_fn(co, fnname) + + def test_simple_inlining(self): + src = ( +''' +def function_to_be_inlined(x, y, z): + # this currently needs to exist to enable the inliner + pass +sep = '-' +def f(x, y, z): + return (2 * x * y) + (sep * z) +def callsite(): + f('foo', 3, 16) +''') + self.assertIsInlinable(src) + callsite = self.compile_to_code(src, 'callsite') + asm = disassemble(callsite) + self.assertIn('JUMP_IF_SPECIALIZABLE', asm) + self.assertIn("('foofoofoofoofoofoo')", asm) + self.assertIn("(sep)", asm) + + + + def test_copy_propagation(self): + src = ( +''' +def function_to_be_inlined(a, b, c, d): + return(a + b + c + d) +def callsite(): + print(function_to_be_inlined('fee', 'fi', 'fo', 'fum')) +''') + callsite = self.compile_to_code(src, 'callsite') + asm = disassemble(callsite) + # print(asm) + self.assertIn('JUMP_IF_SPECIALIZABLE', asm) + self.assertIn("('feefifofum')", asm) + + def test_keyword_args(self): + src = ( +''' +def f(x, y, z=42): + return 42 +''') + self.assertIsNotInlinable(src) + + def test_star_args(self): + src = (''' +def f(x, y, *args): + return [x, y] + args +''') + self.assertIsNotInlinable(src) + + + def test_kwargs(self): + src = ( +''' +def f(x, y, **kwargs): + return kwargs +''') + self.assertIsNotInlinable(src) + + def test_decorators(self): + src = ( +''' +@login_required +def f(x, y, z): + return 42 +''') + self.assertIsNotInlinable(src) + + def test_imports(self): + src = ( +''' +def f(x, y, z): + import os + return 42 +''') + self.assertIsNotInlinable(src) + + def test_generators(self): + src = ( +''' +def f(x, y, z): + yield 42 +''') + self.assertIsNotInlinable(src) + + def test_complex_return_logic(self): + # Only simple use of "return" for now + src = ( +''' +def f(x, y, z): + if x: + return 42 +''') + self.assertIsNotInlinable(src) + + + def test_free_and_cell_variables(self): + # No free/cell variables + src = ( +''' +def f(x, y, z): + def g(p, q): + return x*p + y*q + z + return g +''') + self.assertIsNotInlinable(src) + + def test_reassignment_of_function_name(self): + src = ( +''' +def f(x): + return 42 +f = len +''') + self.assertIsNotInlinable(src) + + + + def not_test_simple(self): + src = ''' +my_global = 34 + +def function_to_be_inlined(x, y, z): + return (2 * x * y) + z + my_global +print(function_to_be_inlined(3, 4, 5)) +''' + asm = disassemble(src) + print(asm) + # FIXME: verify that the inlined callsite actually works! + + def test_double(self): + src = ''' +def function_to_be_inlined(): + pass +def f(x, y, z): + return (2 * x * y) + z +def g(x, y, z): + return (f(x, y, 3 * z) + - f(x + 1, y * 5, z -2)) +def h(x, y): + return g(x, y, 0) +''' + co = compile(src, 'test_optimize.py', 'exec') + g = get_code_for_fn(co, 'g') + h = get_code_for_fn(co, 'h') + g_asm = disassemble(g) + h_asm = disassemble(h) + #print(disassemble(co)) + #print('\n\ng') + #print(g_asm) + self.assertIn('JUMP_IF_SPECIALIZABLE', g_asm) + #print('\n\nh') + #print(h_asm) + self.assertIn('JUMP_IF_SPECIALIZABLE', h_asm) + + def test_ignore_implicit_return(self): + src = ''' +my_global = 34 +def function_to_be_inlined(x, y, z): + print(2 * x * y) + z + my_global +def callsite(): + function_to_be_inlined(3, 4, 5) +''' + callsite = self.compile_to_code(src, 'callsite') + asm = disassemble(callsite) + #print(asm) + # FIXME: actually test it + + def test_call_call(self): + # seen in copy.py: + src = ''' +def _copy_with_constructor(x): + return type(x)(x) +''' + fn = self.compile_to_code(src, '_copy_with_constructor') + asm = disassemble(fn) + + def test_global(self): + # Taken from tempfile.py: + src = ''' +tempdir = None + +def gettempdir(): + """Accessor for tempfile.tempdir.""" + global tempdir + if tempdir is None: + _once_lock.acquire() + try: + if tempdir is None: + tempdir = _get_default_tempdir() + finally: + _once_lock.release() + return tempdir + +def callsite(): + return gettempdir() +''' + fn = self.compile_to_code(src, 'callsite') + asm = disassemble(fn) + #print(asm) + self.assertIn('JUMP_IF_SPECIALIZABLE', asm) + self.assertIn('(tempdir)', asm) + + def test_del_local(self): + # Adapted from tempfile.py: + src = ( +''' +def f(): + fd = _os.open(filename, _bin_openflags, 0o600) + fp = _io.open(fd, 'wb') + fp.write(b'blat') + fp.close() + _os.unlink(filename) + del fp, fd + return dir +''') + fn = self.compile_to_code(src, 'f') + asm = disassemble(fn) + + def test_inplace(self): + src = ( +''' +def inplace(self): + a = foo() + a *= 42 +''') + fn = self.compile_to_code(src, 'inplace') + asm = disassemble(fn) + # FIXME: Ensure that the initial write to "a" isn't optimized away: + #print(asm) + + def test_use_implicit_return(self): + pass # TODO + + def test_predicates(self): + src = ''' +def function_to_be_inlined(): + pass + +def is_crunchy(path): + return True + +def find_file(filename, std_dirs, paths): + for dir in std_dirs: + if is_crunchy(dir): + f = os.path.join(sysroot, dir[1:], filename) + return None +''' + fn = self.compile_to_code(src, 'find_file') + asm = disassemble(fn) + self.assertIn('JUMP_IF_SPECIALIZABLE', asm) + + def test_useless_args(self): + src = ''' +def function_to_be_inlined(): + pass + +def foo(a, b, c): + bar(a, b) + +def bar(a, b): + baz(a) + +def baz(a): + pass + +foo(1, 2, 3) +''' + asm = disassemble(src) + #print(asm) + # FIXME + + def test_simple_recursion(self): + src = ''' +def function_to_be_inlined(): + return function_to_be_inlined() +''' + fn = self.compile_to_code(src, 'function_to_be_inlined') + asm = disassemble(fn) + #print(asm) + + def test_mutual_recursion(self): + src = ''' +def function_to_be_inlined(): + return function_to_be_inlined() + +def f(): + return g() + +def g(): + return f() +''' + fn = self.compile_to_code(src, 'f') + asm = disassemble(fn) + #print(asm) + + + +def function_to_be_inlined(): + return 'I am the original implementation' + +def new_version(): + return 'I am the new version' + +def call_inlinable_function(): + return function_to_be_inlined() + +class TestRebinding(unittest.TestCase): + + def test_rebinding(self): + # "call_inlinable_function" should have an inlined copy + # of "function_to_be_inlined": + asm = disassemble(call_inlinable_function) + #print(asm) + # Should have logic for detecting if it can specialize: + self.assertIn('JUMP_IF_SPECIALIZABLE', asm) + self.assertIn('(__saved__function_to_be_inlined)', asm) + # Should have inlined constant value: + self.assertIn("('I am the original implementation')", asm) + + # Try calling it: + self.assertEquals(call_inlinable_function(), + 'I am the original implementation') + + # Now rebind the wrapped function. + # The easy way to do this: + # function_to_be_inlined = new_version + # doesn't work: the optimizer spots it, and turns off inlining + # for "function_to_be_inlined" throughout this module + # Similarly, using "globals()" directly is spotted. + + __builtins__.globals()['function_to_be_inlined'] = new_version + + # Verify that we get the correct result: + self.assertEquals(call_inlinable_function(), + 'I am the new version') + # FIXME; should also check bytecode instrumentation to verify that we + # took the generalized implementation + + + + +def test_main(verbose=None): + import sys + from test import support + test_classes = (TestFramework, TestInlining, TestRebinding) + support.run_unittest(*test_classes) + + # verify reference counting + if verbose and hasattr(sys, "gettotalrefcount"): + import gc + counts = [None] * 5 + for i in range(len(counts)): + support.run_unittest(*test_classes) + gc.collect() + counts[i] = sys.gettotalrefcount() + print(counts) + +if __name__ == "__main__": + test_main(verbose=False) + pass Index: Lib/__optimizer__.py =================================================================== --- Lib/__optimizer__.py (revision 0) +++ Lib/__optimizer__.py (revision 0) @@ -0,0 +1,749 @@ +import sys +import ast +from pprint import pprint +#print("__optimizer__.py imported!") + +# Taken from symtable.h: +DEF_GLOBAL = 1 # global stmt +DEF_LOCAL = 2 # assignment in code block +DEF_PARAM = 2<<1 # formal parameter +DEF_NONLOCAL = 2<<2 # nonlocal stmt +USE = 2<<3 # name is used +DEF_FREE = 2<<4 # name used but not defined in nested block +DEF_FREE_CLASS = 2<<5 # free variable from class's method +DEF_IMPORT = 2<<6 # assignment occurred via import + +DEF_BOUND = (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) + +# GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol +# table. GLOBAL is returned from PyST_GetScope() for either of them. +# It is stored in ste_symbols at bits 12-15. + +SCOPE_OFFSET = 11 +SCOPE_MASK = (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL) + +LOCAL = 1 +GLOBAL_EXPLICIT = 2 +GLOBAL_IMPLICIT = 3 +FREE = 4 +CELL = 5 + + +# Suppress various local-manipulation transforms in the presence of usage +# of these builtins: +BUILTINS_THAT_READ_LOCALS = {'locals', 'vars', 'dir'} + +def log(msg): + if 0: + print(msg) + +# Analogous to PyST_GetScope: +def get_scope(ste, name): + v = ste.symbols.get(name, 0) + if v: + return (v >> SCOPE_OFFSET) & SCOPE_MASK + else: + return 0 + +def add_local(ste, name): + # Add a new local var to an STE + assert name not in ste.varnames + assert name not in ste.symbols + ste.symbols[name] = (LOCAL << SCOPE_OFFSET) + ste.varnames.append(name) + +def to_dot(t): + def _node_to_dot(node, indent): + result = '' + prefix = ' ' * indent + if isinstance(node, ast.AST): + if hasattr(node, 'ste'): + result += prefix + 'subgraph cluster_%s {\n' % id(node) + result += prefix + ' label = "%s"\n;' % node.ste.name + result += prefix + ' node%i [label=<%s>];\n' % (id(node), node.__class__.__name__) + for name, field in ast.iter_fields(node): + if field is not None: + result += prefix + ' node%i -> node%i [label="%s"];\n' % (id(node), id(field), name) + result += _node_to_dot(field, indent + 2) + if hasattr(node, 'ste'): + result += prefix + '}\n' + elif isinstance(node, list): + result += prefix + 'node%i [label=<[]>];\n' % (id(node)) + for i, item in enumerate(node): + result += prefix + 'node%i -> node%i [label="[%i]"];\n' % (id(node), id(item), i) + result += _node_to_dot(item, indent) + elif node is None: + pass + else: + result += prefix + 'node%i [label=<%s>];\n' % (id(node), repr(node)) + return result + + result = 'digraph {\n' + result += _node_to_dot(t, 1) + result += '}' + return result + +def dot_to_png(dot, filename): + from subprocess import Popen, PIPE + p = Popen(['/usr/bin/dot', + '-T', 'png', + '-o', filename], + stdin=PIPE) + p.communicate(dot.encode('utf-8')) + p.wait() + +def get_ste_for_path(path): + ''' + Given a list of (node, field, index) triples, obtain a list of + symbol table entries + ''' + result = [] + for node, field, index in path: + if hasattr(node, 'ste'): + result.append(node.ste) + return result + +class PathTransformer: + """ + Similar to an ast.NodeTransformer, but passes in a path when visiting a node + + The path is passed in as a list of (node, field, index) triples + """ + def visit(self, node, path=[]): + """Visit a node.""" + method = 'visit_' + node.__class__.__name__ + visitor = getattr(self, method, self.generic_visit) + return visitor(node, path) + + def generic_visit(self, node, path): + for field, old_value in ast.iter_fields(node): + old_value = getattr(node, field, None) + if isinstance(old_value, list): + new_values = [] + for idx, value in enumerate(old_value): + if isinstance(value, ast.AST): + value = self.visit(value, path + [(node, field, idx)]) + if value is None: + continue + elif not isinstance(value, ast.AST): + new_values.extend(value) + continue + new_values.append(value) + old_value[:] = new_values + elif isinstance(old_value, ast.AST): + new_node = self.visit(old_value, path + [(node, field, None)]) + if new_node is None: + delattr(node, field) + else: + setattr(node, field, new_node) + return node + +def make_load_name(name, locnode): + return ast.copy_location(ast.Name(id=name, ctx=ast.Load()), locnode) + +def make_assignment(name, expr, locnode): + name_node = ast.copy_location(ast.Name(id=name, ctx=ast.Store()), locnode) + return ast.copy_location(ast.Assign(targets=[name_node], + value=expr), + locnode) + +def ast_clone(node): + #log('ast_clone', node) + if isinstance(node, ast.AST): + clone = node.__class__() + clone = ast.copy_location(clone, node) + for name, value in ast.iter_fields(node): + if isinstance(value, ast.AST): + cvalue = ast_clone(value) + elif isinstance(value, list): + cvalue = [ast_clone(el) for el in value] + else: + cvalue = value + setattr(clone, name, cvalue) + return clone + elif isinstance(node, str): + return node + else: + raise ValueError("Don't know how to clone %r" % node) + +class InlineBodyFixups(ast.NodeTransformer): + """ + Fix up the cloned body of a function, for inlining + """ + def __init__(self, varprefix, ste): + self.varprefix = varprefix + self.ste = ste + + def visit_Name(self, node): + # Replace local names with prefixed versions: + scope = get_scope(self.ste, node.id) + if scope == LOCAL: + node.id = self.varprefix + node.id + return node + + def visit_Return(self, node): + self.generic_visit(node) + # replace (the final) return with "__returnval__ = expr": + return make_assignment(self.varprefix + "__returnval__", node.value, node) + +class FunctionInliner(PathTransformer): + def __init__(self, tree, defn): + self.tree = tree + self.defn = defn + assert hasattr(defn, 'ste') + self.num_callsites = 0 + log('ste for body: %r' % defn.ste) + + def visit_Call(self, node, path): + # Stop inlining beyond an arbitrary cutoff + # (bm_simplecall was exploding): + if self.num_callsites > 10: + return node + + # Visit children: + self.generic_visit(node, path) + + if isinstance(node.func, ast.Attribute): + # Don't try to inline method calls yet: + return node + + if not isinstance(node.func, ast.Name): + # Don't try to inline where the function is a non-trivial + # expression e.g. "f()()" + return node + + if node.func.id != self.defn.name: + return node + + log('Considering call to: %r' % node.func.id) + log('Path: %r' % path) + log('ste for callsite: %r' % get_ste_for_path(path)) + + # We need to find the insertion point for statements: + # Walk up the ancestors until you find a statement-holding node: + for ancestor in path[::-1]: + #log('foo:', ancestor) + if isinstance(ancestor[0], (ast.Module, ast.Interactive, + ast.FunctionDef, ast.ClassDef, + ast.For, ast.With, + ast.TryExcept, ast.TryFinally, + ast.ExceptHandler)): + break + + # Can we inline predicates? + if isinstance(ancestor[0], (ast.While)): + if ancestor[1] == 'test': + # No good place to insert code for an inlined "while" + # predicate; bail: + return node + + if isinstance(ancestor[0], (ast.If)): + if ancestor[1] == 'test': + # We may be able to inline a predicate before the "if": + continue + else: + # Inline within the body or the orelse: + break + + # Locate innermost scope at callsite: + ste = get_ste_for_path(path)[-1] + + #print('Inlining call to: %r within %r' % (node.func.id, ste.name)) + self.num_callsites += 1 + log('ancestor: %r' % (ancestor, )) + + assert ancestor[2] is not None + + + #log(ast.dump(self.defn)) + varprefix = '__inline_%s_%x__' % (node.func.id, id(node)) + #log('varprefix: %s' % varprefix) + + # Generate a body of specialized statements that can replace the call: + specialized = [] + + # Create assignment statements of the form: + # __inline__x = expr for x + # for each parameter + # We will insert before the callsite + for formal, actual in zip(self.defn.args.args, node.args): + #log('formal: %s' % ast.dump(formal)) + #log('actual: %s' % ast.dump(actual)) + # FIXME: ste + add_local(ste, varprefix+formal.arg) + + assign = make_assignment(varprefix+formal.arg, actual, node) + specialized.append(assign) + + # FIXME: these seem to be being done using LOAD_NAME/STORE_NAME; why isn't it using _FAST? + # aha: because they're in module scope, not within a function. + + # Introduce __returnval__; initialize it to None, equivalent to + # implicit "return None" if there are no "Return" nodes: + returnval = varprefix + "__returnval__" + add_local(ste, returnval) + # FIXME: this requires "None", how to do this in AST? + assign = make_assignment(returnval, + make_load_name('None', node), node) + # FIXME: this leads to LOAD_NAME None, when it should be LOAD_CONST, surely? + specialized.append(assign) + + # Make inline body, generating various statements + # ending with: + # __inline____returnval = expr + inline_body = [] + fixer = InlineBodyFixups(varprefix, self.defn.ste) + for stmt in self.defn.body: + assert isinstance(stmt, ast.AST) + inline_body.append(fixer.visit(ast_clone(stmt))) + #log('inline_body:', inline_body) + specialized += inline_body + + #log('Parent: %s' % ast.dump(find_parent(self.tree, node))) + + #seq = getattr(ancestor[0], ancestor[1]) + + # Splice the compound statements into place: + #seq = seq[:ancestor[2]] + compound + seq[ancestor[2]:] # FIXME + #setattr(ancestor[0], ancestor[1], seq) + + #log(seq) + + #log(ast.dump(ancestor[0])) + + # FIXME: need some smarts about the value of the "Specialize": + # it's the final Expr within the body + specialized_result = ast.copy_location(ast.Name(id=returnval, + ctx=ast.Load()), + node) + + return ast.copy_location(ast.Specialize(name=node.func, + generalized=node, + specialized_body=specialized, + specialized_result=specialized_result), + node) + + # Replace the call with a load from __inline____returnval__ + return ast.copy_location(ast.Name(id=returnval, + ctx=ast.Load()), node) + +class NotInlinable(Exception): + pass + +class CheckInlinableVisitor(PathTransformer): + def __init__(self, defn): + self.defn = defn + self.returns = [] + + # Various nodes aren't handlable: + def visit_FunctionDef(self, node, path): + if node != self.defn: + raise NotInlinable() + self.generic_visit(node, path) + return node + def visit_ClassDef(self, node, path): + raise NotInlinable() + def visit_Yield(self, node, path): + raise NotInlinable() + def visit_Import(self, node, path): + raise NotInlinable() + def visit_ImportFrom(self, node, path): + raise NotInlinable() + + def visit_Return(self, node, path): + self.returns.append(path) + return node + +def fn_is_inlinable(defn, mod): + # Should we inline calls to the given FunctionDef ? + assert(isinstance(defn, ast.FunctionDef)) + + # Only inline "simple" calling conventions for now: + if len(defn.decorator_list) > 0: + return False + + if (defn.args.vararg is not None or + defn.args.kwarg is not None or + defn.args.kwonlyargs != [] or + defn.args.defaults != [] or + defn.args.kw_defaults != []): + return False + + # Don't try to inline generators and other awkward cases: + v = CheckInlinableVisitor(defn) + try: + v.visit(defn) + except NotInlinable: + return False + + # TODO: restrict to just those functions with only a "return" at + # the end (or implicit "return None"), no "return" in awkward places + # (but could have other control flow) + # Specifically: no returns in places that have successor code + # for each return: + # FIXME: for now, only inline functions which have a single, final + # explicit "return" at the end, or no returns: + log('returns of %s: %r' % (defn.name, v.returns)) + if len(v.returns)>1: + return False + + # Single "return"? Then it must be at the top level + if len(v.returns) == 1: + rpath = v.returns[0] + # Must be at toplevel: + if len(rpath) != 1: + return False + + # Must be at the end of that level + if rpath[0][2] != len(defn.body)-1: + return False + + + + # Don't inline functions that use free or cell vars + # (just locals and globals): + assert hasattr(defn, 'ste') + ste = defn.ste + for varname in ste.varnames: + scope = get_scope(ste, varname) + #log('%r: %r' % (varname, scope)) + if scope not in {LOCAL, GLOBAL_EXPLICIT, GLOBAL_IMPLICIT}: + return False + + # Don't inline functions that use the "locals" or "vars" builtins: + if varname in BUILTINS_THAT_READ_LOCALS: + return False + + # Don't inline functions with names that get rebound: + for node in ast.walk(mod): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + if target.id == defn.name: + if isinstance(target.ctx, ast.Store): + return False + + return True + + + +def _inline_function_calls(t): + # Locate top-level function defs: + inlinable_function_defs = {} + for s in t.body: + if isinstance(s, ast.FunctionDef): + if fn_is_inlinable(s, t): + inlinable_function_defs[s.name] = s + + # We need to be able to look up functions by name to detect "monkeypatching" + # Keep a stored copy of each function as it's built, for later comparison + # A better way to do this would be to store the const function somewhere as it's compiled + # and then go in and fixup all of the JUMP_IF_SPECIALIZABLE entrypoints + # But for now, this will do for a prototype: + new_body = [] + for s in t.body: + new_body.append(s) + if isinstance(s, ast.FunctionDef): + storedname = '__saved__' + s.name + if s.name in inlinable_function_defs: + assign = ast.copy_location(ast.Assign(targets=[ast.Name(id=storedname, ctx=ast.Store())], + value=ast.Name(id=s.name, ctx=ast.Load())), + s) + ast.fix_missing_locations(assign) + new_body.append(assign) + t.body = new_body + + #print('inlinable_function_defs:%r' % inlinable_function_defs) + + # Locate call sites: + for name in inlinable_function_defs: + log('inlining calls to %r' % name) + inliner = FunctionInliner(t, inlinable_function_defs[name]) + inliner.visit(t) + + return t + + +def is_local_name(ste, expr): + assert isinstance(expr, ast.AST) + if isinstance(expr, ast.Name): + if get_scope(ste, expr.id) == LOCAL: + return True + +def is_write_to_local(ste, expr): + assert isinstance(expr, ast.AST) + if is_local_name(ste, expr): + if isinstance(expr.ctx, ast.Store): + return True + +def is_read_from_local(ste, expr): + assert isinstance(expr, ast.AST) + if is_local_name(ste, expr): + if isinstance(expr.ctx, ast.Load): + return True + +def is_constant(expr): + assert isinstance(expr, ast.AST) + if isinstance(expr, (ast.Num, ast.Str, ast.Bytes)): + return True + + +# The following optimizations currently can only cope with functions with a +# single basic-block, to avoid the need to build a CFG and do real data flow +# analysis. + +class MoreThanOneBasicBlock(Exception): + pass + +class LocalAssignmentWalker(ast.NodeTransformer): + def __init__(self, funcdef): + self.funcdef = funcdef + self.local_values = {} + + def log(self, msg): + if 0: + print(msg) + + def _is_local(self, varname): + ste = self.funcdef.ste + return get_scope(ste, varname) == LOCAL + + def _is_propagatable(self, expr): + # Is this expression propagatable to successive store operations? + + # We can propagate reads of locals (until they are written to): + if is_read_from_local(self.funcdef.ste, expr): + return True + + if is_constant(expr): + return True + + return False + + def visit_Assign(self, assign): + self.generic_visit(assign) + self.log(' got assign: %s' % ast.dump(assign)) + + # Keep track of assignments to locals that are directly of constants + # or of other other locals: + for target in assign.targets: + if is_write_to_local(self.funcdef.ste, target): + self.log(' write to %r <- %s' % (target.id, ast.dump(assign.value))) + if len(assign.targets) == 1: + target = assign.targets[0] + if self._is_propagatable(assign.value): + self.log(' recording value for %r: %s' % (target.id, ast.dump(assign.value))) + self.local_values[target.id] = assign.value + continue + self.log(' %r is no longer known' % target.id) + self.local_values[target.id] = None + + # Propagate earlier copies to this assignment: + if len(assign.targets) == 1: + target = assign.targets[0] + if is_write_to_local(self.funcdef.ste, target): + if target.id in self.local_values: + value = self.local_values[target.id] + if value is not None: + pass + #self.log(' copy-propagation target') + + return assign + + def visit_Name(self, name): + self.log('visit_Name %r' % name) + self.generic_visit(name) + if is_read_from_local(self.funcdef.ste, name): + self.log(' got read from local: %s' % ast.dump(name)) + if name.id in self.local_values: + value = self.local_values[name.id] + if value is not None: + self.log(' copy-propagating: %r <- %s' % (name.id, ast.dump(value))) + return value # clone this? + return name + + # Nodes implying branching and looping: + def visit_For(self, node): + raise MoreThanOneBasicBlock() + def visit_While(self, node): + raise MoreThanOneBasicBlock() + def visit_If(self, node): + raise MoreThanOneBasicBlock() + def visit_With(self, node): + raise MoreThanOneBasicBlock() + def visit_TryExcept(self, node): + raise MoreThanOneBasicBlock() + def visit_TryFinally(self, node): + raise MoreThanOneBasicBlock() + def visit_IfExp(self, node): + raise MoreThanOneBasicBlock() + + +class CopyPropagation(ast.NodeTransformer): + + def visit_FunctionDef(self, funcdef): + log('CopyPropagation: got function def: %r' % funcdef.name) + self.generic_visit(funcdef) + + try: + w = LocalAssignmentWalker(funcdef) + w.visit(funcdef) + except MoreThanOneBasicBlock: + pass + + return funcdef + + +def _copy_propagation(t): + # Very simple copy propagation, which (for simplicity), requires that we + # have a single basic block + v = CopyPropagation() + v.visit(t) + return t + + + + +class ReferenceToLocalFinder(PathTransformer): + # Gather all reads/writes of locals within the given FunctionDef + def __init__(self, funcdef): + assert isinstance(funcdef, ast.FunctionDef) + self.funcdef = funcdef + # varnames of locals: + self.local_reads = set() + self.local_writes = set() + self.globals = set() + + def log(self, msg): + if 0: + print(msg) + + def visit_Name(self, node, path): + scope = get_scope(self.funcdef.ste, node.id) + if scope == LOCAL: + self.log(' found local: %r %r' % (ast.dump(node), path)) + if isinstance(node.ctx, ast.Store): + self.local_writes.add(node.id) + elif isinstance(node.ctx, ast.Load): + self.local_reads.add(node.id) + else: + assert isinstance(node.ctx, ast.Del) # FIXME: what about other cases? + elif scope in {GLOBAL_EXPLICIT, GLOBAL_IMPLICIT}: + self.globals.add(node.id) + return node + + def visit_AugAssign(self, augassign, path): + # VAR += EXPR references VAR, and we can't remove it since it could + # have arbitrary sideeffects + if isinstance(augassign.target, ast.Name): + target = augassign.target + scope = get_scope(self.funcdef.ste, target.id) + if scope == LOCAL: + self.log(' found local: %r %r' % (ast.dump(augassign), path)) + # An augassign is both a read and a write: + self.local_writes.add(target.id) + self.local_reads.add(target.id) + self.generic_visit(augassign, path) + return augassign + + +class RemoveAssignmentToUnusedLocals(ast.NodeTransformer): + # Replace all Assign([local], expr) with Expr(expr) for the given locals + def __init__(self, varnames): + self.varnames = varnames + + def visit_Assign(self, node): + if len(node.targets) == 1: + if isinstance(node.targets[0], ast.Name): + if node.targets[0].id in self.varnames: + if isinstance(node.targets[0].ctx, ast.Store): + # Eliminate the assignment + return ast.copy_location(ast.Expr(node.value), + node.value) + # FIXME: also eliminate the variable from symtable + return node + + + +class RedundantLocalRemover(PathTransformer): + + def log(self, msg): + if 0: + print(msg) + + def visit_FunctionDef(self, funcdef, path): + self.log('got function def: %r' % funcdef.name) + v = ReferenceToLocalFinder(funcdef) + v.visit(funcdef, path) + self.generic_visit(funcdef, path) + + # Don't ellide locals if this function references the builtins + # that access them: + if v.globals.intersection(BUILTINS_THAT_READ_LOCALS): + return funcdef + + unused_writes = v.local_writes - v.local_reads + if unused_writes: + self.log(' globals: %s' % v.globals) + self.log(' loaded from: %s' % v.local_reads) + self.log(' stored to: %s:' % v.local_writes) + self.log(' unused: %s' % unused_writes) + v = RemoveAssignmentToUnusedLocals(unused_writes) + v.visit(funcdef) + return funcdef + +def _remove_redundant_locals(t): + v = RedundantLocalRemover() + v.visit(t) + return t + + + + +# For now restrict ourselves to just a few places: +def is_test_code(t, filename): + if filename.endswith('test_optimize.py'): + return True + for n in ast.walk(t): + if isinstance(n, ast.FunctionDef): + if n.name == 'function_to_be_inlined': + return True + return False + +#class OptimizationError(Exception): +# def __init__(self + +from pprint import pprint +def optimize_ast(t, filename, st_blocks): + if 0: + print("optimize_ast called: %s" % filename) + if 1: # is_test_code(t, filename): + dot_before = to_dot(t) + try: + # pprint(st_blocks) + # log(t) + # pprint(t) + # log(ast.dump(t)) + + #dot_to_png(to_dot(t), 'before.png') + + if isinstance(t, ast.Module): + t = _inline_function_calls(t) + #cfg = CFG.from_ast(t) + #print(cfg.to_dot()) + #dot_to_png(cfg.to_dot(), 'cfg.png') + + t = _copy_propagation(t) + + t = _remove_redundant_locals(t) + + #dot_to_png(to_dot(t), 'after.png') + + except: + print('Exception during optimization of %r' % filename) + # dot_to_png(dot_before, 'before.png') + raise + #print('finished optimizing') + return t + +