diff -r c6237edff631 Lib/tokenize.py --- a/Lib/tokenize.py Sat Nov 03 18:29:31 2012 +0200 +++ b/Lib/tokenize.py Sat Nov 03 20:47:40 2012 -0400 @@ -226,10 +226,16 @@ self.prev_col = 0 self.encoding = None - def add_whitespace(self, start): + def add_whitespace(self, start, supply_line_continuation): row, col = start - assert row <= self.prev_row col_offset = col - self.prev_col + # Nearly all newlines are handled by the NL and NEWLINE tokens, + # but explicit line continuations are not, so they're handled here. + if row > self.prev_row: + row_offset = row - self.prev_row + if supply_line_continuation: + self.tokens.append(" \\\n" * row_offset) + col_offset = col # Recalculate the column offset from the start of our new line if col_offset: self.tokens.append(" " * col_offset) @@ -242,7 +248,8 @@ if tok_type == ENCODING: self.encoding = token continue - self.add_whitespace(start) + self.add_whitespace(start, + supply_line_continuation=(tok_type!=ENDMARKER) ) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL):