Index: Python/ceval.c =================================================================== --- Python/ceval.c (revision 70087) +++ Python/ceval.c (working copy) @@ -111,7 +111,7 @@ PyFrameObject *, int, PyObject *); static void call_exc_trace(Py_tracefunc, PyObject *, PyFrameObject *); static int maybe_call_line_trace(Py_tracefunc, PyObject *, - PyFrameObject *, int *, int *, int *); + PyFrameObject *, int *, int *, int *, int *); static PyObject * apply_slice(PyObject *, PyObject *, PyObject *); static int assign_slice(PyObject *, PyObject *, @@ -688,12 +688,15 @@ /* when tracing we set things up so that - not (instr_lb <= current_bytecode_offset < instr_ub) + instr_lb <= current_bytecode_offset < instr_ub - is true when the line being executed has changed. The - initial values are such as to make this false the first - time it is tested. */ + is false when the line being executed may have changed. + The initial values are such as to make this false the first + time it is tested. Then maybe_call_line_trace calls + PyCode_CheckLineNumber() to check if the line has actually + changed. */ int instr_ub = -1, instr_lb = 0, instr_prev = -1; + int cur_line = 0; unsigned char *first_instr; PyObject *names; @@ -785,8 +788,11 @@ #ifdef DYNAMIC_EXECUTION_PROFILE #define PREDICT(op) if (0) goto PRED_##op +#define PREDICT_WITH_SIGNALS(op) if (0) goto PRED_##op #else #define PREDICT(op) if (*next_instr == op) goto PRED_##op +#define PREDICT_WITH_SIGNALS(op) \ + if (*next_instr == op && --_Py_Ticker > 0) goto PRED_##op #endif #define PREDICTED(op) PRED_##op: next_instr++ @@ -908,8 +914,8 @@ prediction effectively links the two codes together as if they were a single new opcode; accordingly,f->f_lasti will point to the first code in the pair (for instance, GET_ITER followed by - FOR_ITER is effectively a single opcode and f->f_lasti will point - at to the beginning of the combined pair.) + JUMP_FORWARD is effectively a single opcode and f->f_lasti will + point at to the beginning of the combined pair.) */ next_instr = first_instr + f->f_lasti + 1; stack_pointer = f->f_stacktop; @@ -1027,7 +1033,7 @@ err = maybe_call_line_trace(tstate->c_tracefunc, tstate->c_traceobj, f, &instr_lb, &instr_ub, - &instr_prev); + &instr_prev, &cur_line); /* Reload possibly changed frame fields */ JUMPTO(f->f_lasti); if (f->f_stacktop != NULL) { @@ -1425,7 +1431,7 @@ err = PyList_Append(v, w); Py_DECREF(w); if (err == 0) { - PREDICT(JUMP_ABSOLUTE); + PREDICT(FOR_ITER); continue; } break; @@ -2293,6 +2299,7 @@ if (x != NULL) continue; break; + PREDICTED_WITH_ARG(JUMP_FORWARD); case JUMP_FORWARD: JUMPBY(oparg); goto fast_next_opcode; @@ -2307,14 +2314,22 @@ if (w == Py_False) { Py_DECREF(w); JUMPTO(oparg); +#if FAST_LOOPS /* See JUMP_ABSOLUTE for a description of this define */ goto fast_next_opcode; +#else + continue; +#endif } err = PyObject_IsTrue(w); Py_DECREF(w); if (err > 0) err = 0; - else if (err == 0) + else if (err == 0) { JUMPTO(oparg); +#if FAST_LOOPS /* See JUMP_ABSOLUTE for a description of this define */ + goto fast_next_opcode; +#endif + } else break; continue; @@ -2329,13 +2344,20 @@ if (w == Py_True) { Py_DECREF(w); JUMPTO(oparg); +#if FAST_LOOPS /* See JUMP_ABSOLUTE for a description of this define */ goto fast_next_opcode; +#else + continue; +#endif } err = PyObject_IsTrue(w); Py_DECREF(w); if (err > 0) { err = 0; JUMPTO(oparg); +#if FAST_LOOPS /* See JUMP_ABSOLUTE for a description of this define */ + goto fast_next_opcode; +#endif } else if (err == 0) ; @@ -2390,7 +2412,6 @@ break; continue; - PREDICTED_WITH_ARG(JUMP_ABSOLUTE); case JUMP_ABSOLUTE: JUMPTO(oparg); #if FAST_LOOPS @@ -2413,7 +2434,7 @@ Py_DECREF(v); if (x != NULL) { SET_TOP(x); - PREDICT(FOR_ITER); + PREDICT(JUMP_FORWARD); continue; } STACKADJ(-1); @@ -2426,7 +2447,10 @@ x = (*v->ob_type->tp_iternext)(v); if (x != NULL) { PUSH(x); - PREDICT(STORE_FAST); + JUMPTO(oparg); + /* STORE_FAST is a fast_next_opcode so we must be careful + about not blocking signals in an empty loop. */ + PREDICT_WITH_SIGNALS(STORE_FAST); PREDICT(UNPACK_SEQUENCE); continue; } @@ -2437,9 +2461,8 @@ PyErr_Clear(); } /* iterator ended normally */ - x = v = POP(); + x = v = POP(); Py_DECREF(v); - JUMPBY(oparg); continue; case BREAK_LOOP: @@ -3594,7 +3617,7 @@ static int maybe_call_line_trace(Py_tracefunc func, PyObject *obj, PyFrameObject *frame, int *instr_lb, int *instr_ub, - int *instr_prev) + int *instr_prev, int *cur_line) { int result = 0; @@ -3609,13 +3632,14 @@ line = PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, &bounds); - if (line >= 0) { + if (line >= 0 && line != *cur_line) { frame->f_lineno = line; result = call_trace(func, obj, frame, PyTrace_LINE, Py_None); } *instr_lb = bounds.ap_lower; *instr_ub = bounds.ap_upper; + *cur_line = line; } else if (frame->f_lasti <= *instr_prev) { result = call_trace(func, obj, frame, PyTrace_LINE, Py_None); Index: Python/peephole.c =================================================================== --- Python/peephole.c (revision 70087) +++ Python/peephole.c (working copy) @@ -16,6 +16,7 @@ #define CONDITIONAL_JUMP(op) (op==POP_JUMP_IF_FALSE || op==POP_JUMP_IF_TRUE \ || op==JUMP_IF_FALSE_OR_POP || op==JUMP_IF_TRUE_OR_POP) #define ABSOLUTE_JUMP(op) (op==JUMP_ABSOLUTE || op==CONTINUE_LOOP \ + || op==FOR_ITER \ || op==POP_JUMP_IF_FALSE || op==POP_JUMP_IF_TRUE \ || op==JUMP_IF_FALSE_OR_POP || op==JUMP_IF_TRUE_OR_POP) #define JUMPS_ON_TRUE(op) (op==POP_JUMP_IF_TRUE || op==JUMP_IF_TRUE_OR_POP) @@ -614,16 +615,16 @@ continue; case JUMP_ABSOLUTE: - case CONTINUE_LOOP: case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: case JUMP_IF_FALSE_OR_POP: case JUMP_IF_TRUE_OR_POP: + case CONTINUE_LOOP: + case FOR_ITER: j = addrmap[GETARG(codestr, i)]; SETARG(codestr, i, j); break; - case FOR_ITER: case JUMP_FORWARD: case SETUP_LOOP: case SETUP_EXCEPT: Index: Python/import.c =================================================================== --- Python/import.c (revision 70087) +++ Python/import.c (working copy) @@ -74,9 +74,11 @@ Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) Python 2.7a0: 62181 (optimize conditional branches: introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 2.7a0: 62191 (FOR_ITER optimization, + lnotab format allowing negative line increments) . */ -#define MAGIC (62181 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (62191 | ((long)'\r'<<16) | ((long)'\n'<<24)) /* Magic word as global; note that _PyImport_Init() can change the value of this global to accommodate for alterations of how the Index: Python/compile.c =================================================================== --- Python/compile.c (revision 70087) +++ Python/compile.c (working copy) @@ -1605,12 +1605,12 @@ * constant = 1: "if 1", "if 2", ... * constant = -1: rest */ if (constant == 0) { - if (s->v.If.orelse) + if (asdl_seq_LEN(s->v.If.orelse)) VISIT_SEQ(c, stmt, s->v.If.orelse); } else if (constant == 1) { VISIT_SEQ(c, stmt, s->v.If.body); } else { - if (s->v.If.orelse) { + if (asdl_seq_LEN(s->v.If.orelse)) { next = compiler_new_block(c); if (next == NULL) return 0; @@ -1620,8 +1620,8 @@ VISIT(c, expr, s->v.If.test); ADDOP_JABS(c, POP_JUMP_IF_FALSE, next); VISIT_SEQ(c, stmt, s->v.If.body); - ADDOP_JREL(c, JUMP_FORWARD, end); - if (s->v.If.orelse) { + if (asdl_seq_LEN(s->v.If.orelse)) { + ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, next); VISIT_SEQ(c, stmt, s->v.If.orelse); } @@ -1633,26 +1633,34 @@ static int compiler_for(struct compiler *c, stmt_ty s) { - basicblock *start, *cleanup, *end; + basicblock *start, *body, *tail, *cleanup, *end; + int for_lineno; start = compiler_new_block(c); + body = compiler_new_block(c); + tail = compiler_new_block(c); cleanup = compiler_new_block(c); end = compiler_new_block(c); - if (start == NULL || end == NULL || cleanup == NULL) + if (start == NULL || body == NULL || tail == NULL + || end == NULL || cleanup == NULL) return 0; ADDOP_JREL(c, SETUP_LOOP, end); if (!compiler_push_fblock(c, LOOP, start)) return 0; VISIT(c, expr, s->v.For.iter); + for_lineno = c->u->u_lineno; ADDOP(c, GET_ITER); compiler_use_next_block(c, start); + ADDOP_JREL(c, JUMP_FORWARD, tail); + compiler_use_next_block(c, body); + VISIT(c, expr, s->v.For.target); + VISIT_SEQ(c, stmt, s->v.For.body); + compiler_use_next_block(c, tail); /* for expressions must be traced on each iteration, so we need to set an extra line number. */ c->u->u_lineno_set = false; - ADDOP_JREL(c, FOR_ITER, cleanup); - VISIT(c, expr, s->v.For.target); - VISIT_SEQ(c, stmt, s->v.For.body); - ADDOP_JABS(c, JUMP_ABSOLUTE, start); + c->u->u_lineno = for_lineno; + ADDOP_JABS(c, FOR_ITER, body); compiler_use_next_block(c, cleanup); ADDOP(c, POP_BLOCK); compiler_pop_fblock(c, LOOP, start); @@ -1664,8 +1672,11 @@ static int compiler_while(struct compiler *c, stmt_ty s) { - basicblock *loop, *orelse, *end, *anchor = NULL; + basicblock *loop, *orelse, *end; + /* These are unused if the condition is optimized into a constant */ + basicblock *body = NULL, *tail = NULL, *anchor = NULL; int constant = expr_constant(s->v.While.test); + int while_lineno; if (constant == 0) { if (s->v.While.orelse) @@ -1675,13 +1686,15 @@ loop = compiler_new_block(c); end = compiler_new_block(c); if (constant == -1) { + body = compiler_new_block(c); + tail = compiler_new_block(c); anchor = compiler_new_block(c); - if (anchor == NULL) + if (body == NULL || tail == NULL || anchor == NULL) return 0; } if (loop == NULL || end == NULL) return 0; - if (s->v.While.orelse) { + if (asdl_seq_LEN(s->v.While.orelse)) { orelse = compiler_new_block(c); if (orelse == NULL) return 0; @@ -1690,18 +1703,27 @@ orelse = NULL; ADDOP_JREL(c, SETUP_LOOP, end); + while_lineno = c->u->u_lineno; compiler_use_next_block(c, loop); if (!compiler_push_fblock(c, LOOP, loop)) return 0; if (constant == -1) { + ADDOP_JREL(c, JUMP_FORWARD, tail); + compiler_use_next_block(c, body); + } + VISIT_SEQ(c, stmt, s->v.While.body); + if (constant == -1) { + compiler_use_next_block(c, tail); /* while expressions must be traced on each iteration, so we need to set an extra line number. */ c->u->u_lineno_set = false; + c->u->u_lineno = while_lineno; VISIT(c, expr, s->v.While.test); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, anchor); + ADDOP_JABS(c, POP_JUMP_IF_TRUE, body); } - VISIT_SEQ(c, stmt, s->v.While.body); - ADDOP_JABS(c, JUMP_ABSOLUTE, loop); + else { + ADDOP_JABS(c, JUMP_ABSOLUTE, loop); + } /* XXX should the two POP instructions be in a separate block if there is no else clause ? @@ -2604,24 +2626,23 @@ and then write to the element */ comprehension_ty l; - basicblock *start, *anchor, *skip, *if_cleanup; + basicblock *start, *body, *tail, *anchor; int i, n; start = compiler_new_block(c); - skip = compiler_new_block(c); - if_cleanup = compiler_new_block(c); + body = compiler_new_block(c); + tail = compiler_new_block(c); anchor = compiler_new_block(c); - if (start == NULL || skip == NULL || if_cleanup == NULL || - anchor == NULL) + if (start == NULL || body == NULL || tail == NULL || anchor == NULL) return 0; l = (comprehension_ty)asdl_seq_GET(generators, gen_index); VISIT(c, expr, l->iter); ADDOP(c, GET_ITER); compiler_use_next_block(c, start); - ADDOP_JREL(c, FOR_ITER, anchor); - NEXT_BLOCK(c); + ADDOP_JREL(c, JUMP_FORWARD, tail); + compiler_use_next_block(c, body); VISIT(c, expr, l->target); /* XXX this needs to be cleaned up...a lot! */ @@ -2629,7 +2650,7 @@ for (i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i); VISIT(c, expr, e); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, tail); NEXT_BLOCK(c); } @@ -2641,13 +2662,11 @@ if (gen_index >= asdl_seq_LEN(generators)) { VISIT(c, expr, elt); ADDOP_I(c, LIST_APPEND, gen_index+1); - - compiler_use_next_block(c, skip); } - compiler_use_next_block(c, if_cleanup); - ADDOP_JABS(c, JUMP_ABSOLUTE, start); + compiler_use_next_block(c, tail); + ADDOP_JABS(c, FOR_ITER, body); compiler_use_next_block(c, anchor); - + return 1; } @@ -2669,17 +2688,15 @@ and then write to the element */ comprehension_ty ge; - basicblock *start, *anchor, *skip, *if_cleanup, *end; + basicblock *start, *body, *tail, *end; int i, n; start = compiler_new_block(c); - skip = compiler_new_block(c); - if_cleanup = compiler_new_block(c); - anchor = compiler_new_block(c); + body = compiler_new_block(c); + tail = compiler_new_block(c); end = compiler_new_block(c); - if (start == NULL || skip == NULL || if_cleanup == NULL || - anchor == NULL || end == NULL) + if (start == NULL || body == NULL || tail == NULL || end == NULL) return 0; ge = (comprehension_ty)asdl_seq_GET(generators, gen_index); @@ -2698,8 +2715,8 @@ ADDOP(c, GET_ITER); } compiler_use_next_block(c, start); - ADDOP_JREL(c, FOR_ITER, anchor); - NEXT_BLOCK(c); + ADDOP_JREL(c, JUMP_FORWARD, tail); + compiler_use_next_block(c, body); VISIT(c, expr, ge->target); /* XXX this needs to be cleaned up...a lot! */ @@ -2707,7 +2724,7 @@ for (i = 0; i < n; i++) { expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i); VISIT(c, expr, e); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, tail); NEXT_BLOCK(c); } @@ -2720,12 +2737,9 @@ VISIT(c, expr, elt); ADDOP(c, YIELD_VALUE); ADDOP(c, POP_TOP); - - compiler_use_next_block(c, skip); } - compiler_use_next_block(c, if_cleanup); - ADDOP_JABS(c, JUMP_ABSOLUTE, start); - compiler_use_next_block(c, anchor); + compiler_use_next_block(c, tail); + ADDOP_JABS(c, FOR_ITER, body); ADDOP(c, POP_BLOCK); compiler_pop_fblock(c, LOOP, start); compiler_use_next_block(c, end); @@ -3495,12 +3509,11 @@ /* All about a_lnotab. -c_lnotab is an array of unsigned bytes disguised as a Python string. -It is used to map bytecode offsets to source code line #s (when needed -for tracebacks). +c_lnotab is an array of bytes disguised as a Python string. It is used +to map bytecode offsets to source code line #s (when needed for tracebacks). The array is conceptually a list of - (bytecode offset increment, line number increment) + (unsigned bytecode offset increment, signed line number increment) pairs. The details are important and delicate, best illustrated by example: byte code offset source code line number @@ -3515,14 +3528,17 @@ 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 -The second trick is that an unsigned byte can't hold negative values, or -values larger than 255, so (a) there's a deep assumption that byte code -offsets and their corresponding line #s both increase monotonically, and (b) -if at least one column jumps by more than 255 from one row to the next, more -than one pair is written to the table. In case #b, there's no way to know -from looking at the table later how many were written. That's the delicate -part. A user of c_lnotab desiring to find the source line number -corresponding to a bytecode address A should do something like this +The second trick is that while bytecode offset increments are unsigned +bytes (byte code offsets are assumed to increase monotonically), line +number increments can be negative as well as positive (this is necessary +to evaluate a loop's condition at the end of the loop, saving an +unconditional jump). Thus, if either the bytecode increment jumps by more +than 255, or the line number increment jumps by more than 127 or less than +-127, more than one pair is written to the table. In this case there's no +way to know from looking at the table later how many were written. That's +the delicate part. A user of c_lnotab desiring to find the source line +number corresponding to a bytecode address A should do something like +this: lineno = addr = 0 for addr_incr, line_incr in c_lnotab: @@ -3535,28 +3551,33 @@ the line # increment in each pair generated must be 0 until the remaining addr increment is < 256. So, in the example above, assemble_lnotab (it used to be called com_set_lineno) should not (as was actually done until 2.2) -expand 300, 300 to 255, 255, 45, 45, - but to 255, 0, 45, 255, 0, 45. +expand 300, 300 to 255, 255, 45, 45, + but to 255, 0, 45, 127, 0, 127, 0, 46. */ +#define MAX_LINENO 127 +#define MAX_BYTECODE 255 + static int assemble_lnotab(struct assembler *a, struct instr *i) { - int d_bytecode, d_lineno; + int d_bytecode, d_lineno, d_abs_lineno, d_lineno_sign; int len; unsigned char *lnotab; d_bytecode = a->a_offset - a->a_lineno_off; d_lineno = i->i_lineno - a->a_lineno; + d_abs_lineno = abs(d_lineno); + d_lineno_sign = (d_lineno >= 0) ? 1 : -1; assert(d_bytecode >= 0); - assert(d_lineno >= 0); + assert(d_abs_lineno >= 0); - if(d_bytecode == 0 && d_lineno == 0) + if (d_bytecode == 0 && d_lineno == 0) return 1; - if (d_bytecode > 255) { - int j, nbytes, ncodes = d_bytecode / 255; + if (d_bytecode > MAX_BYTECODE) { + int j, nbytes, ncodes = d_bytecode / MAX_BYTECODE; nbytes = a->a_lnotab_off + 2 * ncodes; len = PyString_GET_SIZE(a->a_lnotab); if (nbytes >= len) { @@ -3574,15 +3595,15 @@ lnotab = (unsigned char *) PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; for (j = 0; j < ncodes; j++) { - *lnotab++ = 255; + *lnotab++ = MAX_BYTECODE; *lnotab++ = 0; } - d_bytecode -= ncodes * 255; + d_bytecode -= ncodes * MAX_BYTECODE; a->a_lnotab_off += ncodes * 2; } - assert(d_bytecode <= 255); - if (d_lineno > 255) { - int j, nbytes, ncodes = d_lineno / 255; + assert(d_bytecode <= MAX_BYTECODE); + if (d_abs_lineno > MAX_LINENO) { + int j, nbytes, ncodes = d_abs_lineno / MAX_LINENO; nbytes = a->a_lnotab_off + 2 * ncodes; len = PyString_GET_SIZE(a->a_lnotab); if (nbytes >= len) { @@ -3600,13 +3621,13 @@ lnotab = (unsigned char *) PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off; *lnotab++ = d_bytecode; - *lnotab++ = 255; + *lnotab++ = MAX_LINENO * d_lineno_sign; d_bytecode = 0; for (j = 1; j < ncodes; j++) { *lnotab++ = 0; - *lnotab++ = 255; + *lnotab++ = MAX_LINENO * d_lineno_sign; } - d_lineno -= ncodes * 255; + d_abs_lineno -= ncodes * MAX_LINENO; a->a_lnotab_off += ncodes * 2; } @@ -3621,11 +3642,11 @@ a->a_lnotab_off += 2; if (d_bytecode) { *lnotab++ = d_bytecode; - *lnotab++ = d_lineno; + *lnotab++ = d_abs_lineno * d_lineno_sign; } else { /* First line of a block; def stmt, etc. */ *lnotab++ = 0; - *lnotab++ = d_lineno; + *lnotab++ = d_abs_lineno * d_lineno_sign; } a->a_lineno = i->i_lineno; a->a_lineno_off = a->a_offset; Index: Include/opcode.h =================================================================== --- Include/opcode.h (revision 70087) +++ Include/opcode.h (working copy) @@ -87,7 +87,7 @@ #define STORE_NAME 90 /* Index in name list */ #define DELETE_NAME 91 /* "" */ #define UNPACK_SEQUENCE 92 /* Number of sequence items */ -#define FOR_ITER 93 +#define FOR_ITER 93 /* Byte index of beginning of loop body */ #define LIST_APPEND 94 #define STORE_ATTR 95 /* Index in name list */ Index: Objects/codeobject.c =================================================================== --- Objects/codeobject.c (revision 70087) +++ Objects/codeobject.c (working copy) @@ -629,7 +629,7 @@ addr += *p++; if (*p) bounds->ap_lower = addr; - line += *p++; + line += (signed char) *p++; --size; } Index: Misc/NEWS =================================================================== --- Misc/NEWS (revision 70087) +++ Misc/NEWS (working copy) @@ -12,6 +12,9 @@ Core and Builtins ----------------- +- Issues #2183, #2459, and #4715: Optimize bytecode generation for + loops and comprehensions. + - Issue #5247: Improve error message when unknown format codes are used when using str.format() with str, unicode, long, int, and float arguments. Index: Lib/compiler/pycodegen.py =================================================================== --- Lib/compiler/pycodegen.py (revision 70087) +++ Lib/compiler/pycodegen.py (working copy) @@ -413,7 +413,7 @@ def visitIf(self, node): end = self.newBlock() numtests = len(node.tests) - for i in range(numtests): + for i in range(numtests - 1): test, suite = node.tests[i] if is_constant_false(test): # XXX will need to check generator stuff here @@ -426,8 +426,22 @@ self.visit(suite) self.emit('JUMP_FORWARD', end) self.startBlock(nextTest) - if node.else_: - self.visit(node.else_) + # The last test can optimize a few things + test, suite = node.tests[numtests - 1] + if is_constant_false(test): + if node.else_: + self.visit(node.else_) + else: + self.set_lineno(test) + self.visit(test) + else_block = self.newBlock() if node.else_ else end + self.emit('POP_JUMP_IF_FALSE', else_block) + self.nextBlock() + self.visit(suite) + if node.else_: + self.emit('JUMP_FORWARD', end) + self.startBlock(else_block) + self.visit(node.else_) self.nextBlock(end) def visitWhile(self, node): @@ -435,22 +449,26 @@ loop = self.newBlock() else_ = self.newBlock() + body = self.newBlock() + tail = self.newBlock() + after = self.newBlock() - after = self.newBlock() self.emit('SETUP_LOOP', after) self.nextBlock(loop) self.setups.push((LOOP, loop)) + self.emit('JUMP_FORWARD', tail) + self.startBlock(body) self.set_lineno(node, force=True) - self.visit(node.test) - self.emit('POP_JUMP_IF_FALSE', else_ or after) - self.nextBlock() self.visit(node.body) - self.emit('JUMP_ABSOLUTE', loop) - self.startBlock(else_) # or just the POPs if not else clause + self.nextBlock(tail) + self.visit(node.test) + self.emit('POP_JUMP_IF_TRUE', body) + + self.nextBlock(else_) # or just the POPs if not else clause self.emit('POP_BLOCK') self.setups.pop() if node.else_: @@ -459,6 +477,8 @@ def visitFor(self, node): start = self.newBlock() + body = self.newBlock() + tail = self.newBlock() anchor = self.newBlock() after = self.newBlock() self.setups.push((LOOP, start)) @@ -470,10 +490,12 @@ self.nextBlock(start) self.set_lineno(node, force=1) - self.emit('FOR_ITER', anchor) + self.emit('JUMP_FORWARD', tail) + self.startBlock(body) self.visit(node.assign) self.visit(node.body) - self.emit('JUMP_ABSOLUTE', start) + self.nextBlock(tail) + self.emit('FOR_ITER', body) self.nextBlock(anchor) self.emit('POP_BLOCK') self.setups.pop() @@ -571,42 +593,37 @@ self.emit('BUILD_LIST', 0) stack = [] - for i, for_ in zip(range(len(node.quals)), node.quals): - start, anchor = self.visit(for_) - cont = None + for for_ in node.quals: + body, tail = self.visit(for_) for if_ in for_.ifs: - if cont is None: - cont = self.newBlock() - self.visit(if_, cont) - stack.insert(0, (start, cont, anchor)) + self.visit(if_, tail) + stack.insert(0, (for_, body, tail)) self.visit(node.expr) self.emit('LIST_APPEND', len(node.quals) + 1) - for start, cont, anchor in stack: - if cont: - self.nextBlock(cont) - self.emit('JUMP_ABSOLUTE', start) - self.startBlock(anchor) + for for_, body, tail in stack: + self.nextBlock(tail) + self.set_lineno(for_, force=True) + self.emit('FOR_ITER', body) + self.nextBlock() def visitListCompFor(self, node): - start = self.newBlock() - anchor = self.newBlock() + body = self.newBlock() + tail = self.newBlock() self.visit(node.list) self.emit('GET_ITER') - self.nextBlock(start) - self.set_lineno(node, force=True) - self.emit('FOR_ITER', anchor) self.nextBlock() + self.emit('JUMP_FORWARD', tail) + self.nextBlock(body) self.visit(node.assign) - return start, anchor + return body, tail def visitListCompIf(self, node, branch): self.set_lineno(node, force=True) self.visit(node.test) self.emit('POP_JUMP_IF_FALSE', branch) - self.newBlock() def _makeClosure(self, gen, args): frees = gen.scope.get_free_vars() @@ -637,24 +654,21 @@ # setup list stack = [] - for i, for_ in zip(range(len(node.quals)), node.quals): - start, anchor, end = self.visit(for_) - cont = None + for for_ in node.quals: + body, tail, end = self.visit(for_) for if_ in for_.ifs: - if cont is None: - cont = self.newBlock() - self.visit(if_, cont) - stack.insert(0, (start, cont, anchor, end)) + self.visit(if_, tail) + stack.insert(0, (for_, body, tail, end)) self.visit(node.expr) self.emit('YIELD_VALUE') self.emit('POP_TOP') - for start, cont, anchor, end in stack: - if cont: - self.nextBlock(cont) - self.emit('JUMP_ABSOLUTE', start) - self.startBlock(anchor) + for for_, body, tail, end in stack: + self.nextBlock(tail) + self.set_lineno(for_, force=True) + self.emit('FOR_ITER', body) + self.nextBlock() self.emit('POP_BLOCK') self.setups.pop() self.nextBlock(end) @@ -663,7 +677,8 @@ def visitGenExprFor(self, node): start = self.newBlock() - anchor = self.newBlock() + body = self.newBlock() + tail = self.newBlock() end = self.newBlock() self.setups.push((LOOP, start)) @@ -676,11 +691,10 @@ self.emit('GET_ITER') self.nextBlock(start) - self.set_lineno(node, force=True) - self.emit('FOR_ITER', anchor) - self.nextBlock() + self.emit('JUMP_FORWARD', tail) + self.nextBlock(body) self.visit(node.assign) - return start, anchor, end + return body, tail, end def visitGenExprIf(self, node, branch): self.set_lineno(node, force=True) Index: Lib/compiler/pyassem.py =================================================================== --- Lib/compiler/pyassem.py (revision 70087) +++ Lib/compiler/pyassem.py (working copy) @@ -620,6 +620,11 @@ # compute deltas addr = self.codeOffset - self.lastoff line = lineno - self.lastline + abs_line = abs(line) + if line >= 0: + sign_line = lambda l: l + else: + sign_line = lambda l: 256 - l # Python assumes that lineno always increases with # increasing bytecode address (lnotab is unsigned char). # Depending on when SET_LINENO instructions are emitted @@ -630,19 +635,18 @@ # after the loading of "b". This works with the C Python # compiler because it only generates a SET_LINENO instruction # for the assignment. - if line >= 0: - push = self.lnotab.append - while addr > 255: - push(255); push(0) - addr -= 255 - while line > 255: - push(addr); push(255) - line -= 255 - addr = 0 - if addr > 0 or line > 0: - push(addr); push(line) - self.lastline = lineno - self.lastoff = self.codeOffset + push = self.lnotab.append + while addr > 255: + push(255); push(0) + addr -= 255 + while abs_line > 127: + push(addr); push(sign_line(127)) + abs_line -= 127 + addr = 0 + if addr > 0 or abs_line > 0: + push(addr); push(sign_line(abs_line)) + self.lastline = lineno + self.lastoff = self.codeOffset def getCode(self): return ''.join(self.code) Index: Lib/opcode.py =================================================================== --- Lib/opcode.py (revision 70087) +++ Lib/opcode.py (working copy) @@ -126,7 +126,7 @@ name_op('STORE_NAME', 90) # Index in name list name_op('DELETE_NAME', 91) # "" def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) +jabs_op('FOR_ITER', 93) def_op('LIST_APPEND', 94) name_op('STORE_ATTR', 95) # Index in name list name_op('DELETE_ATTR', 96) # "" Index: Lib/trace.py =================================================================== --- Lib/trace.py (revision 70087) +++ Lib/trace.py (working copy) @@ -377,7 +377,8 @@ """Return dict where keys are lines in the line number table.""" linenos = {} - line_increments = [ord(c) for c in code.co_lnotab[1::2]] + line_increments = [c if c < 128 else c - 256 + for c in map(ord, code.co_lnotab[1::2])] table_length = len(line_increments) docstring = False Index: Lib/test/test_dis.py =================================================================== --- Lib/test/test_dis.py (revision 70087) +++ Lib/test/test_dis.py (working copy) @@ -22,6 +22,33 @@ _f.func_code.co_firstlineno + 2) +def loop(a): + while a > 0: + a -= 1 + return a + +dis_loop = """\ +%3d 0 SETUP_LOOP 26 (to 29) + 3 JUMP_FORWARD 10 (to 16) + +%3d >> 6 LOAD_FAST 0 (a) + 9 LOAD_CONST 1 (1) + 12 INPLACE_SUBTRACT + 13 STORE_FAST 0 (a) + +%3d >> 16 LOAD_FAST 0 (a) + 19 LOAD_CONST 2 (0) + 22 COMPARE_OP 4 (>) + 25 POP_JUMP_IF_TRUE 6 + 28 POP_BLOCK + +%3d >> 29 LOAD_FAST 0 (a) + 32 RETURN_VALUE +"""%(loop.func_code.co_firstlineno + 1, + loop.func_code.co_firstlineno + 2, + loop.func_code.co_firstlineno + 1, + loop.func_code.co_firstlineno + 3) + def bug708901(): for res in range(1, 10): @@ -35,16 +62,14 @@ %-4d 9 LOAD_CONST 2 (10) 12 CALL_FUNCTION 2 15 GET_ITER - >> 16 FOR_ITER 6 (to 25) - 19 STORE_FAST 0 (res) - - %-4d 22 JUMP_ABSOLUTE 16 - >> 25 POP_BLOCK + 16 JUMP_FORWARD 3 (to 22) + >> 19 STORE_FAST 0 (res) + >> 22 FOR_ITER 19 + 25 POP_BLOCK >> 26 LOAD_CONST 0 (None) 29 RETURN_VALUE """%(bug708901.func_code.co_firstlineno + 1, - bug708901.func_code.co_firstlineno + 2, - bug708901.func_code.co_firstlineno + 3) + bug708901.func_code.co_firstlineno + 2) def bug1333982(x=[]): @@ -59,13 +84,13 @@ 9 BUILD_LIST 0 12 LOAD_FAST 0 (x) 15 GET_ITER - >> 16 FOR_ITER 12 (to 31) - 19 STORE_FAST 1 (s) + 16 JUMP_FORWARD 9 (to 28) + >> 19 STORE_FAST 1 (s) 22 LOAD_FAST 1 (s) 25 LIST_APPEND 2 - 28 JUMP_ABSOLUTE 16 + >> 28 FOR_ITER 19 - %-4d >> 31 LOAD_CONST 2 (1) + %-4d 31 LOAD_CONST 2 (1) 34 BINARY_ADD 35 RAISE_VARARGS 2 @@ -116,6 +141,9 @@ def test_dis(self): self.do_disassembly_test(_f, dis_f) + def test_lineno_goes_backwards_in_loop(self): + self.do_disassembly_test(loop, dis_loop) + def test_bug_708901(self): self.do_disassembly_test(bug708901, dis_bug708901) Index: Lib/test/test_trace.py =================================================================== --- Lib/test/test_trace.py (revision 70087) +++ Lib/test/test_trace.py (working copy) @@ -351,7 +351,7 @@ # issue1750076: "while" expression is skipped by debugger def for_example(): for x in range(2): - pass + y = x self.run_and_compare( for_example, [(0, 'call'), Index: Lib/dis.py =================================================================== --- Lib/dis.py (revision 70087) +++ Lib/dis.py (working copy) @@ -182,7 +182,8 @@ """ byte_increments = [ord(c) for c in code.co_lnotab[0::2]] - line_increments = [ord(c) for c in code.co_lnotab[1::2]] + line_increments = [c if c < 128 else c - 256 + for c in map(ord, code.co_lnotab[1::2])] lastlineno = None lineno = code.co_firstlineno