# HG changeset patch # User Antoine Pitrou # Date 1230763825 -3600 diff -r c917cb926b0e -r 97d5e9feb366 Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -247,6 +247,16 @@ ASDLGEN= $(srcdir)/Parser/asdl_c.py ########################################################################## # Python + +OPCODETARGETS_H= \ + $(srcdir)/Python/opcode_targets.h + +OPCODETARGETGEN= \ + $(srcdir)/Python/makeopcodetargets.py + +OPCODETARGETGEN_FILES= \ + $(OPCODETARGETGEN) $(srcdir)/Lib/opcode.py + PYTHON_OBJS= \ Python/_warnings.o \ Python/Python-ast.o \ @@ -564,6 +574,11 @@ Objects/unicodeobject.o: $(srcdir)/Objec $(BYTESTR_DEPS) \ $(srcdir)/Objects/stringlib/formatter.h +$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES) + $(OPCODETARGETGEN) $(OPCODETARGETS_H) + +Python/ceval.o: $(OPCODETARGETS_H) + Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \ $(BYTESTR_DEPS) diff -r c917cb926b0e -r 97d5e9feb366 Python/ceval.c --- a/Python/ceval.c +++ b/Python/ceval.c @@ -581,6 +581,95 @@ PyEval_EvalFrameEx(PyFrameObject *f, int char *filename; #endif +/* Computed GOTOs, or + the-optimization-commonly-but-improperly-known-as-"threaded code" + using GCC's labels-as-values extension + (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html). + + The traditional bytecode evaluation loop uses a "switch" statement, which + decent compilers will optimize as a single indirect branch instruction + combined with a lookup table of jump addresses. However, since the + indirect jump instruction is shared by all opcodes, the CPU will have a + hard time making the right prediction for where to jump next (actually, + it will be always wrong except in the uncommon case of a sequence of + several identical opcodes). + + "Threaded code" in contrast, uses an explicit jump table and an explicit + indirect jump instruction at the end of each opcode. Since the jump + instruction is at a different address for each opcode, the CPU will make a + separate prediction for each of these instructions, which is equivalent to + predicting the second opcode of each opcode pair. These predictions have + a much better chance to turn out valid, especially in small bytecode loops. + + A mispredicted branch on a modern CPU flushes the whole pipeline and + can cost several CPU cycles (depending on the pipeline depth), + and potentially many more instructions (depending on the pipeline width). + A correctly predicted branch, however, is nearly free. + + At the time of this writing, the "threaded code" version is reportedly + 15-20% faster than the normal "switch" version. + + NOTE: care must be taken that the compiler doesn't try to "optimize" the + indirect jumps by sharing them between all opcodes. If needed, gcc has a + dedicated flag for that, "-fno-crossjumping". + +*/ + +#if defined(__GNUC__) \ + && !defined(DYNAMIC_EXECUTION_PROFILE) \ + && !defined(LLTRACE) +#define USE_COMPUTED_GOTOS +#endif + +#ifdef USE_COMPUTED_GOTOS +/* Import the static jump table */ +#include "opcode_targets.h" + +/* This macro is used when several opcodes defer to the same implementation + (e.g. SETUP_LOOP, SETUP_FINALLY) */ +#define TARGET_WITH_IMPL(op, impl) \ + TARGET_##op: \ + if (HAS_ARG(op)) \ + oparg = NEXTARG(); \ + case op: \ + goto impl; \ + +#define TARGET(op) \ + TARGET_##op: \ + if (HAS_ARG(op)) \ + oparg = NEXTARG(); \ + case op: + + +#define DISPATCH() \ + { \ + if (--_Py_Ticker >= 0) { \ + FAST_DISPATCH(); \ + } \ + continue; \ + } +#define FAST_DISPATCH() \ + { \ + if (!_Py_TracingPossible) { \ + register Py_ssize_t offset; \ + f->f_lasti = INSTR_OFFSET(); \ + offset = NEXTOP(); \ + opcode = offset; \ + goto *opcode_targets[offset]; \ + } \ + goto fast_next_opcode; \ + } + +#else +#define TARGET(op) \ + case op: +#define TARGET_WITH_IMPL(op, impl) \ + case op: +#define DISPATCH() continue +#define FAST_DISPATCH() goto fast_next_opcode +#endif + + /* Tuple access macros */ #ifndef Py_DEBUG @@ -658,16 +747,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int predictions turned-on and interpret the results as if some opcodes had been combined or turn-off predictions so that the opcode frequency counter updates for both opcodes. -*/ - -#ifdef DYNAMIC_EXECUTION_PROFILE + + Opcode prediction is disabled with threaded code, since the latter allows + the CPU to record separate branch prediction information for each + opcode. + +*/ + +#if defined(DYNAMIC_EXECUTION_PROFILE) || defined(USE_COMPUTED_GOTOS) #define PREDICT(op) if (0) goto PRED_##op +#define PREDICTED(op) PRED_##op: +#define PREDICTED_WITH_ARG(op) PRED_##op: #else #define PREDICT(op) if (*next_instr == op) goto PRED_##op -#endif - #define PREDICTED(op) PRED_##op: next_instr++ #define PREDICTED_WITH_ARG(op) PRED_##op: oparg = PEEKARG(); next_instr += 3 +#endif + /* Stack manipulation macros */ @@ -1029,56 +1125,56 @@ PyEval_EvalFrameEx(PyFrameObject *f, int /* case STOP_CODE: this is an error! */ - case NOP: - goto fast_next_opcode; - - case LOAD_FAST: + TARGET(NOP) + FAST_DISPATCH(); + + TARGET(LOAD_FAST) x = GETLOCAL(oparg); if (x != NULL) { Py_INCREF(x); PUSH(x); - goto fast_next_opcode; + FAST_DISPATCH(); } format_exc_check_arg(PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(co->co_varnames, oparg)); break; - case LOAD_CONST: + TARGET(LOAD_CONST) x = GETITEM(consts, oparg); Py_INCREF(x); PUSH(x); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED_WITH_ARG(STORE_FAST); - case STORE_FAST: + TARGET(STORE_FAST) v = POP(); SETLOCAL(oparg, v); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED(POP_TOP); - case POP_TOP: - v = POP(); - Py_DECREF(v); - goto fast_next_opcode; - - case ROT_TWO: + TARGET(POP_TOP) + v = POP(); + Py_DECREF(v); + FAST_DISPATCH(); + + TARGET(ROT_TWO) v = TOP(); w = SECOND(); SET_TOP(w); SET_SECOND(v); - goto fast_next_opcode; - - case ROT_THREE: + FAST_DISPATCH(); + + TARGET(ROT_THREE) v = TOP(); w = SECOND(); x = THIRD(); SET_TOP(w); SET_SECOND(x); SET_THIRD(v); - goto fast_next_opcode; - - case ROT_FOUR: + FAST_DISPATCH(); + + TARGET(ROT_FOUR) u = TOP(); v = SECOND(); w = THIRD(); @@ -1087,15 +1183,15 @@ PyEval_EvalFrameEx(PyFrameObject *f, int SET_SECOND(w); SET_THIRD(x); SET_FOURTH(u); - goto fast_next_opcode; - - case DUP_TOP: + FAST_DISPATCH(); + + TARGET(DUP_TOP) v = TOP(); Py_INCREF(v); PUSH(v); - goto fast_next_opcode; - - case DUP_TOPX: + FAST_DISPATCH(); + + TARGET(DUP_TOPX) if (oparg == 2) { x = TOP(); Py_INCREF(x); @@ -1104,7 +1200,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int STACKADJ(2); SET_TOP(x); SET_SECOND(w); - goto fast_next_opcode; + FAST_DISPATCH(); } else if (oparg == 3) { x = TOP(); Py_INCREF(x); @@ -1116,106 +1212,106 @@ PyEval_EvalFrameEx(PyFrameObject *f, int SET_TOP(x); SET_SECOND(w); SET_THIRD(v); - goto fast_next_opcode; + FAST_DISPATCH(); } Py_FatalError("invalid argument to DUP_TOPX" " (bytecode corruption?)"); /* Never returns, so don't bother to set why. */ break; - case UNARY_POSITIVE: + TARGET(UNARY_POSITIVE) v = TOP(); x = PyNumber_Positive(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; - break; - - case UNARY_NEGATIVE: + if (x != NULL) DISPATCH(); + break; + + TARGET(UNARY_NEGATIVE) v = TOP(); x = PyNumber_Negative(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; - break; - - case UNARY_NOT: + if (x != NULL) DISPATCH(); + break; + + TARGET(UNARY_NOT) v = TOP(); err = PyObject_IsTrue(v); Py_DECREF(v); if (err == 0) { Py_INCREF(Py_True); SET_TOP(Py_True); - continue; + DISPATCH(); } else if (err > 0) { Py_INCREF(Py_False); SET_TOP(Py_False); err = 0; - continue; + DISPATCH(); } STACKADJ(-1); break; - case UNARY_INVERT: + TARGET(UNARY_INVERT) v = TOP(); x = PyNumber_Invert(v); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_POWER: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_POWER) w = POP(); v = TOP(); x = PyNumber_Power(v, w, Py_None); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_MULTIPLY: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_Multiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_TRUE_DIVIDE: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_TRUE_DIVIDE) w = POP(); v = TOP(); x = PyNumber_TrueDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_FLOOR_DIVIDE: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_FLOOR_DIVIDE) w = POP(); v = TOP(); x = PyNumber_FloorDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_MODULO: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_MODULO) w = POP(); v = TOP(); x = PyNumber_Remainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_ADD: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_ADD) w = POP(); v = TOP(); if (PyUnicode_CheckExact(v) && @@ -1231,152 +1327,152 @@ PyEval_EvalFrameEx(PyFrameObject *f, int skip_decref_vx: Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_SUBTRACT: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_SUBTRACT) w = POP(); v = TOP(); x = PyNumber_Subtract(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_SUBSCR: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_SUBSCR) w = POP(); v = TOP(); x = PyObject_GetItem(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_LSHIFT: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_LSHIFT) w = POP(); v = TOP(); x = PyNumber_Lshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_RSHIFT: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_RSHIFT) w = POP(); v = TOP(); x = PyNumber_Rshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_AND: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_AND) w = POP(); v = TOP(); x = PyNumber_And(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_XOR: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_XOR) w = POP(); v = TOP(); x = PyNumber_Xor(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case BINARY_OR: + if (x != NULL) DISPATCH(); + break; + + TARGET(BINARY_OR) w = POP(); v = TOP(); x = PyNumber_Or(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case LIST_APPEND: + if (x != NULL) DISPATCH(); + break; + + TARGET(LIST_APPEND) w = POP(); v = stack_pointer[-oparg]; err = PyList_Append(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; - } - break; - - case SET_ADD: + DISPATCH(); + } + break; + + TARGET(SET_ADD) w = POP(); v = stack_pointer[-oparg]; err = PySet_Add(v, w); Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; - } - break; - - case INPLACE_POWER: + DISPATCH(); + } + break; + + TARGET(INPLACE_POWER) w = POP(); v = TOP(); x = PyNumber_InPlacePower(v, w, Py_None); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_MULTIPLY: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_MULTIPLY) w = POP(); v = TOP(); x = PyNumber_InPlaceMultiply(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_TRUE_DIVIDE: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_TRUE_DIVIDE) w = POP(); v = TOP(); x = PyNumber_InPlaceTrueDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_FLOOR_DIVIDE: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_FLOOR_DIVIDE) w = POP(); v = TOP(); x = PyNumber_InPlaceFloorDivide(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_MODULO: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_MODULO) w = POP(); v = TOP(); x = PyNumber_InPlaceRemainder(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_ADD: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_ADD) w = POP(); v = TOP(); if (PyUnicode_CheckExact(v) && @@ -1392,70 +1488,70 @@ PyEval_EvalFrameEx(PyFrameObject *f, int skip_decref_v: Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_SUBTRACT: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_SUBTRACT) w = POP(); v = TOP(); x = PyNumber_InPlaceSubtract(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_LSHIFT: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_LSHIFT) w = POP(); v = TOP(); x = PyNumber_InPlaceLshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_RSHIFT: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_RSHIFT) w = POP(); v = TOP(); x = PyNumber_InPlaceRshift(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_AND: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_AND) w = POP(); v = TOP(); x = PyNumber_InPlaceAnd(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_XOR: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_XOR) w = POP(); v = TOP(); x = PyNumber_InPlaceXor(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case INPLACE_OR: + if (x != NULL) DISPATCH(); + break; + + TARGET(INPLACE_OR) w = POP(); v = TOP(); x = PyNumber_InPlaceOr(v, w); Py_DECREF(v); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case STORE_SUBSCR: + if (x != NULL) DISPATCH(); + break; + + TARGET(STORE_SUBSCR) w = TOP(); v = SECOND(); u = THIRD(); @@ -1465,10 +1561,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_DECREF(u); Py_DECREF(v); Py_DECREF(w); - if (err == 0) continue; - break; - - case DELETE_SUBSCR: + if (err == 0) DISPATCH(); + break; + + TARGET(DELETE_SUBSCR) w = TOP(); v = SECOND(); STACKADJ(-2); @@ -1476,10 +1572,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int err = PyObject_DelItem(v, w); Py_DECREF(v); Py_DECREF(w); - if (err == 0) continue; - break; - - case PRINT_EXPR: + if (err == 0) DISPATCH(); + break; + + TARGET(PRINT_EXPR) v = POP(); w = PySys_GetObject("displayhook"); if (w == NULL) { @@ -1506,7 +1602,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int #ifdef CASE_TOO_BIG default: switch (opcode) { #endif - case RAISE_VARARGS: + TARGET(RAISE_VARARGS) v = w = NULL; switch (oparg) { case 2: @@ -1524,19 +1620,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int } break; - case STORE_LOCALS: + TARGET(STORE_LOCALS) x = POP(); v = f->f_locals; Py_XDECREF(v); f->f_locals = x; - continue; - - case RETURN_VALUE: + DISPATCH(); + + TARGET(RETURN_VALUE) retval = POP(); why = WHY_RETURN; goto fast_block_end; - case YIELD_VALUE: + TARGET(YIELD_VALUE) retval = POP(); f->f_stacktop = stack_pointer; why = WHY_YIELD; @@ -1546,7 +1642,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int SWAP_EXC_STATE(); goto fast_yield; - case POP_EXCEPT: + TARGET(POP_EXCEPT) { PyTryBlock *b = PyFrame_BlockPop(f); if (b->b_type != EXCEPT_HANDLER) { @@ -1557,17 +1653,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int } UNWIND_EXCEPT_HANDLER(b); } - continue; - - case POP_BLOCK: + DISPATCH(); + + TARGET(POP_BLOCK) { PyTryBlock *b = PyFrame_BlockPop(f); UNWIND_BLOCK(b); } - continue; + DISPATCH(); PREDICTED(END_FINALLY); - case END_FINALLY: + TARGET(END_FINALLY) v = POP(); if (PyLong_Check(v)) { why = (enum why_code) PyLong_AS_LONG(v); @@ -1607,7 +1703,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_DECREF(v); break; - case LOAD_BUILD_CLASS: + TARGET(LOAD_BUILD_CLASS) x = PyDict_GetItemString(f->f_builtins, "__build_class__"); if (x == NULL) { @@ -1619,7 +1715,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int PUSH(x); break; - case STORE_NAME: + TARGET(STORE_NAME) w = GETITEM(names, oparg); v = POP(); if ((x = f->f_locals) != NULL) { @@ -1628,14 +1724,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int else err = PyObject_SetItem(x, w, v); Py_DECREF(v); - if (err == 0) continue; + if (err == 0) DISPATCH(); break; } PyErr_Format(PyExc_SystemError, "no locals found when storing %R", w); break; - case DELETE_NAME: + TARGET(DELETE_NAME) w = GETITEM(names, oparg); if ((x = f->f_locals) != NULL) { if ((err = PyObject_DelItem(x, w)) != 0) @@ -1649,7 +1745,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int break; PREDICTED_WITH_ARG(UNPACK_SEQUENCE); - case UNPACK_SEQUENCE: + TARGET(UNPACK_SEQUENCE) v = POP(); if (PyTuple_CheckExact(v) && PyTuple_GET_SIZE(v) == oparg) { @@ -1661,7 +1757,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int PUSH(w); } Py_DECREF(v); - continue; + DISPATCH(); } else if (PyList_CheckExact(v) && PyList_GET_SIZE(v) == oparg) { PyObject **items = \ @@ -1681,7 +1777,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_DECREF(v); break; - case UNPACK_EX: + TARGET(UNPACK_EX) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); v = POP(); @@ -1696,7 +1792,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int break; } - case STORE_ATTR: + TARGET(STORE_ATTR) w = GETITEM(names, oparg); v = TOP(); u = SECOND(); @@ -1704,10 +1800,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int err = PyObject_SetAttr(v, w, u); /* v.w = u */ Py_DECREF(v); Py_DECREF(u); - if (err == 0) continue; - break; - - case DELETE_ATTR: + if (err == 0) DISPATCH(); + break; + + TARGET(DELETE_ATTR) w = GETITEM(names, oparg); v = POP(); err = PyObject_SetAttr(v, w, (PyObject *)NULL); @@ -1715,22 +1811,22 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_DECREF(v); break; - case STORE_GLOBAL: + TARGET(STORE_GLOBAL) w = GETITEM(names, oparg); v = POP(); err = PyDict_SetItem(f->f_globals, w, v); Py_DECREF(v); - if (err == 0) continue; - break; - - case DELETE_GLOBAL: + if (err == 0) DISPATCH(); + break; + + TARGET(DELETE_GLOBAL) w = GETITEM(names, oparg); if ((err = PyDict_DelItem(f->f_globals, w)) != 0) format_exc_check_arg( PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w); break; - case LOAD_NAME: + TARGET(LOAD_NAME) w = GETITEM(names, oparg); if ((v = f->f_locals) == NULL) { PyErr_Format(PyExc_SystemError, @@ -1765,9 +1861,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_INCREF(x); } PUSH(x); - continue; - - case LOAD_GLOBAL: + DISPATCH(); + + TARGET(LOAD_GLOBAL) w = GETITEM(names, oparg); if (PyUnicode_CheckExact(w)) { /* Inline the PyDict_GetItem() calls. @@ -1787,7 +1883,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int if (x != NULL) { Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); } d = (PyDictObject *)(f->f_builtins); e = d->ma_lookup(d, w, hash); @@ -1799,7 +1895,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int if (x != NULL) { Py_INCREF(x); PUSH(x); - continue; + DISPATCH(); } goto load_global_error; } @@ -1818,13 +1914,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int } Py_INCREF(x); PUSH(x); - continue; - - case DELETE_FAST: + DISPATCH(); + + TARGET(DELETE_FAST) x = GETLOCAL(oparg); if (x != NULL) { SETLOCAL(oparg, NULL); - continue; + DISPATCH(); } format_exc_check_arg( PyExc_UnboundLocalError, @@ -1833,19 +1929,19 @@ PyEval_EvalFrameEx(PyFrameObject *f, int ); break; - case LOAD_CLOSURE: + TARGET(LOAD_CLOSURE) x = freevars[oparg]; Py_INCREF(x); PUSH(x); - if (x != NULL) continue; - break; - - case LOAD_DEREF: + if (x != NULL) DISPATCH(); + break; + + TARGET(LOAD_DEREF) x = freevars[oparg]; w = PyCell_Get(x); if (w != NULL) { PUSH(w); - continue; + DISPATCH(); } err = -1; /* Don't stomp existing exception */ @@ -1866,14 +1962,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int } break; - case STORE_DEREF: + TARGET(STORE_DEREF) w = POP(); x = freevars[oparg]; PyCell_Set(x, w); Py_DECREF(w); - continue; - - case BUILD_TUPLE: + DISPATCH(); + + TARGET(BUILD_TUPLE) x = PyTuple_New(oparg); if (x != NULL) { for (; --oparg >= 0;) { @@ -1881,11 +1977,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int PyTuple_SET_ITEM(x, oparg, w); } PUSH(x); - continue; - } - break; - - case BUILD_LIST: + DISPATCH(); + } + break; + + TARGET(BUILD_LIST) x = PyList_New(oparg); if (x != NULL) { for (; --oparg >= 0;) { @@ -1893,11 +1989,11 @@ PyEval_EvalFrameEx(PyFrameObject *f, int PyList_SET_ITEM(x, oparg, w); } PUSH(x); - continue; - } - break; - - case BUILD_SET: + DISPATCH(); + } + break; + + TARGET(BUILD_SET) x = PySet_New(NULL); if (x != NULL) { for (; --oparg >= 0;) { @@ -1911,17 +2007,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int break; } PUSH(x); - continue; - } - break; - - case BUILD_MAP: + DISPATCH(); + } + break; + + TARGET(BUILD_MAP) x = _PyDict_NewPresized((Py_ssize_t)oparg); PUSH(x); - if (x != NULL) continue; - break; - - case STORE_MAP: + if (x != NULL) DISPATCH(); + break; + + TARGET(STORE_MAP) w = TOP(); /* key */ u = SECOND(); /* value */ v = THIRD(); /* dict */ @@ -1930,10 +2026,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int err = PyDict_SetItem(v, w, u); /* v[w] = u */ Py_DECREF(u); Py_DECREF(w); - if (err == 0) continue; - break; - - case MAP_ADD: + if (err == 0) DISPATCH(); + break; + + TARGET(MAP_ADD) w = TOP(); /* key */ u = SECOND(); /* value */ STACKADJ(-2); @@ -1944,20 +2040,20 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_DECREF(w); if (err == 0) { PREDICT(JUMP_ABSOLUTE); - continue; - } - break; - - case LOAD_ATTR: + DISPATCH(); + } + break; + + TARGET(LOAD_ATTR) w = GETITEM(names, oparg); v = TOP(); x = PyObject_GetAttr(v, w); Py_DECREF(v); SET_TOP(x); - if (x != NULL) continue; - break; - - case COMPARE_OP: + if (x != NULL) DISPATCH(); + break; + + TARGET(COMPARE_OP) w = POP(); v = TOP(); x = cmp_outcome(oparg, v, w); @@ -1967,9 +2063,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int if (x == NULL) break; PREDICT(JUMP_IF_FALSE); PREDICT(JUMP_IF_TRUE); - continue; - - case IMPORT_NAME: + DISPATCH(); + + TARGET(IMPORT_NAME) w = GETITEM(names, oparg); x = PyDict_GetItemString(f->f_builtins, "__import__"); if (x == NULL) { @@ -2010,10 +2106,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int READ_TIMESTAMP(intr1); Py_DECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case IMPORT_STAR: + if (x != NULL) DISPATCH(); + break; + + TARGET(IMPORT_STAR) v = POP(); PyFrame_FastToLocals(f); if ((x = f->f_locals) == NULL) { @@ -2026,33 +2122,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int READ_TIMESTAMP(intr1); PyFrame_LocalsToFast(f, 0); Py_DECREF(v); - if (err == 0) continue; - break; - - case IMPORT_FROM: + if (err == 0) DISPATCH(); + break; + + TARGET(IMPORT_FROM) w = GETITEM(names, oparg); v = TOP(); READ_TIMESTAMP(intr0); x = import_from(v, w); READ_TIMESTAMP(intr1); PUSH(x); - if (x != NULL) continue; - break; - - case JUMP_FORWARD: + if (x != NULL) DISPATCH(); + break; + + TARGET(JUMP_FORWARD) JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); PREDICTED_WITH_ARG(JUMP_IF_FALSE); - case JUMP_IF_FALSE: + TARGET(JUMP_IF_FALSE) w = TOP(); if (w == Py_True) { PREDICT(POP_TOP); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_False) { JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); if (err > 0) @@ -2061,18 +2157,18 @@ PyEval_EvalFrameEx(PyFrameObject *f, int JUMPBY(oparg); else break; - continue; + DISPATCH(); PREDICTED_WITH_ARG(JUMP_IF_TRUE); - case JUMP_IF_TRUE: + TARGET(JUMP_IF_TRUE) w = TOP(); if (w == Py_False) { PREDICT(POP_TOP); - goto fast_next_opcode; + FAST_DISPATCH(); } if (w == Py_True) { JUMPBY(oparg); - goto fast_next_opcode; + FAST_DISPATCH(); } err = PyObject_IsTrue(w); if (err > 0) { @@ -2083,10 +2179,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int ; else break; - continue; + DISPATCH(); PREDICTED_WITH_ARG(JUMP_ABSOLUTE); - case JUMP_ABSOLUTE: + TARGET(JUMP_ABSOLUTE) JUMPTO(oparg); #if FAST_LOOPS /* Enabling this path speeds-up all while and for-loops by bypassing @@ -2094,14 +2190,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int because it prevents detection of a control-break in tight loops like "while 1: pass". Compile with this option turned-on when you need the speed-up and do not need break checking inside tight loops (ones - that contain only instructions ending with goto fast_next_opcode). + that contain only instructions ending with FAST_DISPATCH). */ - goto fast_next_opcode; -#else - continue; -#endif - - case GET_ITER: + FAST_DISPATCH(); +#else + DISPATCH(); +#endif + + TARGET(GET_ITER) /* before: [obj]; after [getiter(obj)] */ v = TOP(); x = PyObject_GetIter(v); @@ -2109,13 +2205,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int if (x != NULL) { SET_TOP(x); PREDICT(FOR_ITER); - continue; + DISPATCH(); } STACKADJ(-1); break; PREDICTED_WITH_ARG(FOR_ITER); - case FOR_ITER: + TARGET(FOR_ITER) /* before: [iter]; after: [iter, iter()] *or* [] */ v = TOP(); x = (*v->ob_type->tp_iternext)(v); @@ -2123,7 +2219,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int PUSH(x); PREDICT(STORE_FAST); PREDICT(UNPACK_SEQUENCE); - continue; + DISPATCH(); } if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches( @@ -2135,13 +2231,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int x = v = POP(); Py_DECREF(v); JUMPBY(oparg); - continue; - - case BREAK_LOOP: + DISPATCH(); + + TARGET(BREAK_LOOP) why = WHY_BREAK; goto fast_block_end; - case CONTINUE_LOOP: + TARGET(CONTINUE_LOOP) retval = PyLong_FromLong(oparg); if (!retval) { x = NULL; @@ -2150,9 +2246,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int why = WHY_CONTINUE; goto fast_block_end; - case SETUP_LOOP: - case SETUP_EXCEPT: - case SETUP_FINALLY: + TARGET_WITH_IMPL(SETUP_LOOP, _setup_finally) + TARGET_WITH_IMPL(SETUP_EXCEPT, _setup_finally) + TARGET(SETUP_FINALLY) + _setup_finally: /* NOTE: If you add any new block-setup opcodes that are not try/except/finally handlers, you may need to update the PyGen_NeedsFinalizing() function. @@ -2160,9 +2257,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int PyFrame_BlockSetup(f, opcode, INSTR_OFFSET() + oparg, STACK_LEVEL()); - continue; - - case WITH_CLEANUP: + DISPATCH(); + + TARGET(WITH_CLEANUP) { /* At the top of the stack are 1-3 values indicating how/why we entered the finally clause: @@ -2226,7 +2323,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int break; } - case CALL_FUNCTION: + TARGET(CALL_FUNCTION) { PyObject **sp; PCALL(PCALL_ALL); @@ -2239,13 +2336,14 @@ PyEval_EvalFrameEx(PyFrameObject *f, int stack_pointer = sp; PUSH(x); if (x != NULL) - continue; - break; - } - - case CALL_FUNCTION_VAR: - case CALL_FUNCTION_KW: - case CALL_FUNCTION_VAR_KW: + DISPATCH(); + break; + } + + TARGET_WITH_IMPL(CALL_FUNCTION_VAR, _call_function_var_kw) + TARGET_WITH_IMPL(CALL_FUNCTION_KW, _call_function_var_kw) + TARGET(CALL_FUNCTION_VAR_KW) + _call_function_var_kw: { int na = oparg & 0xff; int nk = (oparg>>8) & 0xff; @@ -2285,12 +2383,13 @@ PyEval_EvalFrameEx(PyFrameObject *f, int } PUSH(x); if (x != NULL) - continue; + DISPATCH(); break; } - case MAKE_CLOSURE: - case MAKE_FUNCTION: + TARGET_WITH_IMPL(MAKE_CLOSURE, _make_function) + TARGET(MAKE_FUNCTION) + _make_function: { int posdefaults = oparg & 0xff; int kwdefaults = (oparg>>8) & 0xff; @@ -2383,7 +2482,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int break; } - case BUILD_SLICE: + TARGET(BUILD_SLICE) if (oparg == 3) w = POP(); else @@ -2395,14 +2494,17 @@ PyEval_EvalFrameEx(PyFrameObject *f, int Py_DECREF(v); Py_XDECREF(w); SET_TOP(x); - if (x != NULL) continue; - break; - - case EXTENDED_ARG: + if (x != NULL) DISPATCH(); + break; + + TARGET(EXTENDED_ARG) opcode = NEXTOP(); oparg = oparg<<16 | NEXTARG(); goto dispatch_opcode; +#ifdef USE_COMPUTED_GOTOS + _unknown_opcode: +#endif default: fprintf(stderr, "XXX lineno: %d, opcode: %d\n", diff -r c917cb926b0e -r 97d5e9feb366 Python/makeopcodetargets.py --- /dev/null +++ b/Python/makeopcodetargets.py @@ -0,0 +1,44 @@ +#! /usr/bin/env python +"""Generate C code for the jump table of the threaded code interpreter +(GCC only). +""" + +import imp +import os + + +def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib") + return imp.load_module(modname, *imp.find_module(modname, [modpath])) + +def write_contents(f): + """Write C code contents to the target file object. + """ + opcode = find_module("opcode") + targets = ['_unknown_opcode'] * 256 + for opname, op in opcode.opmap.items(): + if opname == "STOP_CODE": + # XXX opcode not implemented + continue + targets[op] = "TARGET_%s" % opname + f.write("static void *opcode_targets[256] = {\n") + f.write(",\n".join("\t&&%s" % s for s in targets)) + f.write("\n};\n") + + +if __name__ == "__main__": + import sys + assert len(sys.argv) < 3, "Too many arguments" + if len(sys.argv) == 2: + target = sys.argv[1] + else: + target = "Python/opcode_targets.c" + f = open(target, "w") + try: + write_contents(f) + finally: + f.close() + diff -r c917cb926b0e -r 97d5e9feb366 Python/opcode_targets.h --- /dev/null +++ b/Python/opcode_targets.h @@ -0,0 +1,258 @@ +static void *opcode_targets[256] = { + &&_unknown_opcode, + &&TARGET_POP_TOP, + &&TARGET_ROT_TWO, + &&TARGET_ROT_THREE, + &&TARGET_DUP_TOP, + &&TARGET_ROT_FOUR, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_NOP, + &&TARGET_UNARY_POSITIVE, + &&TARGET_UNARY_NEGATIVE, + &&TARGET_UNARY_NOT, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_UNARY_INVERT, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_BINARY_POWER, + &&TARGET_BINARY_MULTIPLY, + &&_unknown_opcode, + &&TARGET_BINARY_MODULO, + &&TARGET_BINARY_ADD, + &&TARGET_BINARY_SUBTRACT, + &&TARGET_BINARY_SUBSCR, + &&TARGET_BINARY_FLOOR_DIVIDE, + &&TARGET_BINARY_TRUE_DIVIDE, + &&TARGET_INPLACE_FLOOR_DIVIDE, + &&TARGET_INPLACE_TRUE_DIVIDE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_STORE_MAP, + &&TARGET_INPLACE_ADD, + &&TARGET_INPLACE_SUBTRACT, + &&TARGET_INPLACE_MULTIPLY, + &&_unknown_opcode, + &&TARGET_INPLACE_MODULO, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, + &&TARGET_BINARY_LSHIFT, + &&TARGET_BINARY_RSHIFT, + &&TARGET_BINARY_AND, + &&TARGET_BINARY_XOR, + &&TARGET_BINARY_OR, + &&TARGET_INPLACE_POWER, + &&TARGET_GET_ITER, + &&TARGET_STORE_LOCALS, + &&TARGET_PRINT_EXPR, + &&TARGET_LOAD_BUILD_CLASS, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_INPLACE_LSHIFT, + &&TARGET_INPLACE_RSHIFT, + &&TARGET_INPLACE_AND, + &&TARGET_INPLACE_XOR, + &&TARGET_INPLACE_OR, + &&TARGET_BREAK_LOOP, + &&TARGET_WITH_CLEANUP, + &&_unknown_opcode, + &&TARGET_RETURN_VALUE, + &&TARGET_IMPORT_STAR, + &&_unknown_opcode, + &&TARGET_YIELD_VALUE, + &&TARGET_POP_BLOCK, + &&TARGET_END_FINALLY, + &&TARGET_POP_EXCEPT, + &&TARGET_STORE_NAME, + &&TARGET_DELETE_NAME, + &&TARGET_UNPACK_SEQUENCE, + &&TARGET_FOR_ITER, + &&TARGET_UNPACK_EX, + &&TARGET_STORE_ATTR, + &&TARGET_DELETE_ATTR, + &&TARGET_STORE_GLOBAL, + &&TARGET_DELETE_GLOBAL, + &&TARGET_DUP_TOPX, + &&TARGET_LOAD_CONST, + &&TARGET_LOAD_NAME, + &&TARGET_BUILD_TUPLE, + &&TARGET_BUILD_LIST, + &&TARGET_BUILD_SET, + &&TARGET_BUILD_MAP, + &&TARGET_LOAD_ATTR, + &&TARGET_COMPARE_OP, + &&TARGET_IMPORT_NAME, + &&TARGET_IMPORT_FROM, + &&TARGET_JUMP_FORWARD, + &&TARGET_JUMP_IF_FALSE, + &&TARGET_JUMP_IF_TRUE, + &&TARGET_JUMP_ABSOLUTE, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_LOAD_GLOBAL, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_CONTINUE_LOOP, + &&TARGET_SETUP_LOOP, + &&TARGET_SETUP_EXCEPT, + &&TARGET_SETUP_FINALLY, + &&_unknown_opcode, + &&TARGET_LOAD_FAST, + &&TARGET_STORE_FAST, + &&TARGET_DELETE_FAST, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_RAISE_VARARGS, + &&TARGET_CALL_FUNCTION, + &&TARGET_MAKE_FUNCTION, + &&TARGET_BUILD_SLICE, + &&TARGET_MAKE_CLOSURE, + &&TARGET_LOAD_CLOSURE, + &&TARGET_LOAD_DEREF, + &&TARGET_STORE_DEREF, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_CALL_FUNCTION_VAR, + &&TARGET_CALL_FUNCTION_KW, + &&TARGET_CALL_FUNCTION_VAR_KW, + &&TARGET_EXTENDED_ARG, + &&_unknown_opcode, + &&TARGET_LIST_APPEND, + &&TARGET_SET_ADD, + &&TARGET_MAP_ADD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode +};