--- tokenize.py.old 2008-06-11 15:54:32.000000000 +0300 +++ tokenize.py 2008-06-11 16:05:01.000000000 +0300 @@ -188,10 +188,10 @@ if toknum == INDENT: indents.append(tokval) continue - elif toknum == DEDENT: + if toknum == DEDENT: indents.pop() continue - elif toknum in (NEWLINE, COMMENT, NL): + if toknum in (NEWLINE, COMMENT, NL): startline = True elif startline and indents: toks_append(indents[-1]) @@ -222,7 +222,7 @@ contline = None indents = [0] - while 1: # loop over lines in stream + while True: # loop over lines in stream try: line = readline() except StopIteration: @@ -246,20 +246,19 @@ contstr = '' contline = None continue - else: - contstr = contstr + line - contline = contline + line - continue + contstr += line + contline = += line + continue - elif parenlev == 0 and not continued: # new statement + elif not (parenlev or continued): # new statement if not line: break column = 0 while pos < max: # measure leading whitespace - if line[pos] == ' ': column = column + 1 + if line[pos] == ' ': column += 1 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize elif line[pos] == '\f': column = 0 else: break - pos = pos + 1 + pos += 1 if pos == max: break if line[pos] in '#\r\n': # skip comments or blank lines @@ -320,8 +319,7 @@ contstr, needcont = line[start:], 1 contline = line break - else: # ordinary string - yield (STRING, token, spos, epos, line) + yield (STRING, token, spos, epos, line) # ordinary string elif initial in namechars: # ordinary name yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt @@ -333,7 +331,7 @@ else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) - pos = pos + 1 + pos += 1 for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '')