Index: Lib/tokenize.py =================================================================== --- Lib/tokenize.py (revision 60884) +++ Lib/tokenize.py (working copy) @@ -35,6 +35,10 @@ del x del token +SPECIALS = {'[': LSQB, ']': RSQB, '(': LPAR, ')': RPAR, + ';': SEMI, ',': COMMA, '.': DOT, ':': COLON, + '{': LBRACE, '}': RBRACE, '`': BACKQUOTE, + '@': AT} COMMENT = N_TOKENS tok_name[COMMENT] = 'COMMENT' NL = N_TOKENS + 1 @@ -387,7 +391,7 @@ else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 - yield (OP, token, spos, epos, line) + yield (SPECIALS.get(initial, OP), token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) Index: Lib/test/test_tokenize.py =================================================================== --- Lib/test/test_tokenize.py (revision 60884) +++ Lib/test/test_tokenize.py (working copy) @@ -22,7 +22,7 @@ ... " True = False # NEWLINE\\n") NAME 'if' (1, 0) (1, 2) NAME 'False' (1, 3) (1, 8) -OP ':' (1, 8) (1, 9) +COLON ':' (1, 8) (1, 9) NEWLINE '\\n' (1, 9) (1, 10) COMMENT '# NL' (2, 4) (2, 8) NL '\\n' (2, 8) (2, 9) Index: Lib/test/output/test_tokenize =================================================================== --- Lib/test/output/test_tokenize (revision 60884) +++ Lib/test/output/test_tokenize (working copy) @@ -32,73 +32,73 @@ 17,0-17,1: NL '\n' 18,0-18,1: NAME 'a' 18,2-18,3: OP '=' -18,4-18,5: OP '(' +18,4-18,5: LPAR '(' 18,5-18,6: NUMBER '3' -18,6-18,7: OP ',' +18,6-18,7: COMMA ',' 18,8-18,9: NUMBER '4' -18,9-18,10: OP ',' +18,9-18,10: COMMA ',' 18,10-18,11: NL '\n' 19,2-19,3: NUMBER '5' -19,3-19,4: OP ',' +19,3-19,4: COMMA ',' 19,5-19,6: NUMBER '6' -19,6-19,7: OP ')' +19,6-19,7: RPAR ')' 19,7-19,8: NEWLINE '\n' 20,0-20,1: NAME 'y' 20,2-20,3: OP '=' -20,4-20,5: OP '[' +20,4-20,5: LSQB '[' 20,5-20,6: NUMBER '3' -20,6-20,7: OP ',' +20,6-20,7: COMMA ',' 20,8-20,9: NUMBER '4' -20,9-20,10: OP ',' +20,9-20,10: COMMA ',' 20,10-20,11: NL '\n' 21,2-21,3: NUMBER '5' -21,3-21,4: OP ']' +21,3-21,4: RSQB ']' 21,4-21,5: NEWLINE '\n' 22,0-22,1: NAME 'z' 22,2-22,3: OP '=' -22,4-22,5: OP '{' +22,4-22,5: LBRACE '{' 22,5-22,8: STRING "'a'" -22,8-22,9: OP ':' +22,8-22,9: COLON ':' 22,9-22,10: NUMBER '5' -22,10-22,11: OP ',' +22,10-22,11: COMMA ',' 22,11-22,12: NL '\n' 23,2-23,5: STRING "'b'" -23,5-23,6: OP ':' +23,5-23,6: COLON ':' 23,6-23,7: NUMBER '6' -23,7-23,8: OP '}' +23,7-23,8: RBRACE '}' 23,8-23,9: NEWLINE '\n' 24,0-24,1: NAME 'x' 24,2-24,3: OP '=' -24,4-24,5: OP '(' +24,4-24,5: LPAR '(' 24,5-24,8: NAME 'len' -24,8-24,9: OP '(' -24,9-24,10: OP '`' +24,8-24,9: LPAR '(' +24,9-24,10: BACKQUOTE '`' 24,10-24,11: NAME 'y' -24,11-24,12: OP '`' -24,12-24,13: OP ')' +24,11-24,12: BACKQUOTE '`' +24,12-24,13: RPAR ')' 24,14-24,15: OP '+' 24,16-24,17: NUMBER '5' 24,17-24,18: OP '*' 24,18-24,19: NAME 'x' 24,20-24,21: OP '-' 24,22-24,23: NAME 'a' -24,23-24,24: OP '[' +24,23-24,24: LSQB '[' 24,24-24,25: NL '\n' 25,3-25,4: NUMBER '3' -25,5-25,6: OP ']' +25,5-25,6: RSQB ']' 25,6-25,7: NL '\n' 26,3-26,4: OP '-' 26,5-26,6: NAME 'x' 26,7-26,8: OP '+' 26,9-26,12: NAME 'len' -26,12-26,13: OP '(' -26,13-26,14: OP '{' +26,12-26,13: LPAR '(' +26,13-26,14: LBRACE '{' 26,14-26,15: NL '\n' -27,3-27,4: OP '}' +27,3-27,4: RBRACE '}' 27,4-27,5: NL '\n' -28,4-28,5: OP ')' +28,4-28,5: RPAR ')' 28,5-28,6: NL '\n' -29,2-29,3: OP ')' +29,2-29,3: RPAR ')' 29,3-29,4: NEWLINE '\n' 30,0-30,1: NL '\n' 31,0-31,36: COMMENT '# Backslash means line continuation:' @@ -238,29 +238,29 @@ 71,0-71,1: NAME 'x' 71,2-71,3: OP '=' 71,4-71,6: STRING "''" -71,6-71,7: OP ';' +71,6-71,7: SEMI ';' 71,8-71,9: NAME 'y' 71,10-71,11: OP '=' 71,12-71,14: STRING '""' -71,14-71,15: OP ';' +71,14-71,15: SEMI ';' 71,15-71,16: NEWLINE '\n' 72,0-72,1: NAME 'x' 72,2-72,3: OP '=' 72,4-72,8: STRING "'\\''" -72,8-72,9: OP ';' +72,8-72,9: SEMI ';' 72,10-72,11: NAME 'y' 72,12-72,13: OP '=' 72,14-72,17: STRING '"\'"' -72,17-72,18: OP ';' +72,17-72,18: SEMI ';' 72,18-72,19: NEWLINE '\n' 73,0-73,1: NAME 'x' 73,2-73,3: OP '=' 73,4-73,7: STRING '\'"\'' -73,7-73,8: OP ';' +73,7-73,8: SEMI ';' 73,9-73,10: NAME 'y' 73,11-73,12: OP '=' 73,13-73,17: STRING '"\\""' -73,17-73,18: OP ';' +73,17-73,18: SEMI ';' 73,18-73,19: NEWLINE '\n' 74,0-74,1: NAME 'x' 74,2-74,3: OP '=' @@ -289,17 +289,17 @@ 85,0-85,1: NAME 'y' 85,2-85,3: OP '=' 85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\'' -90,3-90,4: OP ';' +90,3-90,4: SEMI ';' 90,4-90,5: NEWLINE '\n' 91,0-91,1: NAME 'y' 91,2-91,3: OP '=' 91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"' -96,1-96,2: OP ';' +96,1-96,2: SEMI ';' 96,2-96,3: NEWLINE '\n' 97,0-97,1: NAME 'y' 97,2-97,3: OP '=' 97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\'' -102,1-102,2: OP ';' +102,1-102,2: SEMI ';' 102,2-102,3: NEWLINE '\n' 103,0-103,1: NAME 'x' 103,2-103,3: OP '=' @@ -386,7 +386,7 @@ 128,13-128,14: NL '\n' 129,0-129,2: NAME 'if' 129,3-129,4: NUMBER '1' -129,4-129,5: OP ':' +129,4-129,5: COLON ':' 129,5-129,6: NEWLINE '\n' 130,0-130,4: INDENT ' ' 130,4-130,5: NAME 'x' @@ -396,7 +396,7 @@ 131,0-131,0: DEDENT '' 131,0-131,2: NAME 'if' 131,3-131,4: NUMBER '1' -131,4-131,5: OP ':' +131,4-131,5: COLON ':' 131,5-131,6: NEWLINE '\n' 132,0-132,8: INDENT ' ' 132,8-132,9: NAME 'x' @@ -406,17 +406,17 @@ 133,0-133,0: DEDENT '' 133,0-133,2: NAME 'if' 133,3-133,4: NUMBER '1' -133,4-133,5: OP ':' +133,4-133,5: COLON ':' 133,5-133,6: NEWLINE '\n' 134,0-134,4: INDENT ' ' 134,4-134,9: NAME 'while' 134,10-134,11: NUMBER '0' -134,11-134,12: OP ':' +134,11-134,12: COLON ':' 134,12-134,13: NEWLINE '\n' 135,0-135,5: INDENT ' ' 135,5-135,7: NAME 'if' 135,8-135,9: NUMBER '0' -135,9-135,10: OP ':' +135,9-135,10: COLON ':' 135,10-135,11: NEWLINE '\n' 136,0-136,11: INDENT ' ' 136,11-136,12: NAME 'x' @@ -432,22 +432,22 @@ 138,0-138,0: DEDENT '' 138,0-138,2: NAME 'if' 138,3-138,4: NUMBER '0' -138,4-138,5: OP ':' +138,4-138,5: COLON ':' 138,5-138,6: NEWLINE '\n' 139,0-139,2: INDENT ' ' 139,2-139,4: NAME 'if' 139,5-139,6: NUMBER '2' -139,6-139,7: OP ':' +139,6-139,7: COLON ':' 139,7-139,8: NEWLINE '\n' 140,0-140,3: INDENT ' ' 140,3-140,8: NAME 'while' 140,9-140,10: NUMBER '0' -140,10-140,11: OP ':' +140,10-140,11: COLON ':' 140,11-140,12: NEWLINE '\n' 141,0-141,8: INDENT ' ' 141,8-141,10: NAME 'if' 141,11-141,12: NUMBER '1' -141,12-141,13: OP ':' +141,12-141,13: COLON ':' 141,13-141,14: NEWLINE '\n' 142,0-142,10: INDENT ' ' 142,10-142,11: NAME 'x' @@ -464,58 +464,58 @@ 146,0-146,0: DEDENT '' 146,0-146,3: NAME 'def' 146,4-146,7: NAME 'd22' -146,7-146,8: OP '(' +146,7-146,8: LPAR '(' 146,8-146,9: NAME 'a' -146,9-146,10: OP ',' +146,9-146,10: COMMA ',' 146,11-146,12: NAME 'b' -146,12-146,13: OP ',' +146,12-146,13: COMMA ',' 146,14-146,15: NAME 'c' 146,15-146,16: OP '=' 146,16-146,17: NUMBER '1' -146,17-146,18: OP ',' +146,17-146,18: COMMA ',' 146,19-146,20: NAME 'd' 146,20-146,21: OP '=' 146,21-146,22: NUMBER '2' -146,22-146,23: OP ')' -146,23-146,24: OP ':' +146,22-146,23: RPAR ')' +146,23-146,24: COLON ':' 146,25-146,29: NAME 'pass' 146,29-146,30: NEWLINE '\n' 147,0-147,3: NAME 'def' 147,4-147,8: NAME 'd01v' -147,8-147,9: OP '(' +147,8-147,9: LPAR '(' 147,9-147,10: NAME 'a' 147,10-147,11: OP '=' 147,11-147,12: NUMBER '1' -147,12-147,13: OP ',' +147,12-147,13: COMMA ',' 147,14-147,15: OP '*' 147,15-147,20: NAME 'restt' -147,20-147,21: OP ',' +147,20-147,21: COMMA ',' 147,22-147,24: OP '**' 147,24-147,29: NAME 'restd' -147,29-147,30: OP ')' -147,30-147,31: OP ':' +147,29-147,30: RPAR ')' +147,30-147,31: COLON ':' 147,32-147,36: NAME 'pass' 147,36-147,37: NEWLINE '\n' 148,0-148,1: NL '\n' -149,0-149,1: OP '(' +149,0-149,1: LPAR '(' 149,1-149,2: NAME 'x' -149,2-149,3: OP ',' +149,2-149,3: COMMA ',' 149,4-149,5: NAME 'y' -149,5-149,6: OP ')' +149,5-149,6: RPAR ')' 149,7-149,9: OP '<>' -149,10-149,11: OP '(' -149,11-149,12: OP '{' +149,10-149,11: LPAR '(' +149,11-149,12: LBRACE '{' 149,12-149,15: STRING "'a'" -149,15-149,16: OP ':' +149,15-149,16: COLON ':' 149,16-149,17: NUMBER '1' -149,17-149,18: OP '}' -149,18-149,19: OP ',' -149,20-149,21: OP '{' +149,17-149,18: RBRACE '}' +149,18-149,19: COMMA ',' +149,20-149,21: LBRACE '{' 149,21-149,24: STRING "'b'" -149,24-149,25: OP ':' +149,24-149,25: COLON ':' 149,25-149,26: NUMBER '2' -149,26-149,27: OP '}' -149,27-149,28: OP ')' +149,26-149,27: RBRACE '}' +149,27-149,28: RPAR ')' 149,28-149,29: NEWLINE '\n' 150,0-150,1: NL '\n' 151,0-151,12: COMMENT '# comparison' @@ -546,7 +546,7 @@ 152,57-152,59: NAME 'is' 152,60-152,63: NAME 'not' 152,64-152,65: NUMBER '1' -152,65-152,66: OP ':' +152,65-152,66: COLON ':' 152,67-152,71: NAME 'pass' 152,71-152,72: NEWLINE '\n' 153,0-153,1: NL '\n' @@ -653,31 +653,31 @@ 172,10-172,11: NL '\n' 173,0-173,6: NAME 'import' 173,7-173,10: NAME 'sys' -173,10-173,11: OP ',' +173,10-173,11: COMMA ',' 173,12-173,16: NAME 'time' 173,16-173,17: NEWLINE '\n' 174,0-174,1: NAME 'x' 174,2-174,3: OP '=' 174,4-174,7: NAME 'sys' -174,7-174,8: OP '.' +174,7-174,8: DOT '.' 174,8-174,15: NAME 'modules' -174,15-174,16: OP '[' +174,15-174,16: LSQB '[' 174,16-174,22: STRING "'time'" -174,22-174,23: OP ']' -174,23-174,24: OP '.' +174,22-174,23: RSQB ']' +174,23-174,24: DOT '.' 174,24-174,28: NAME 'time' -174,28-174,29: OP '(' -174,29-174,30: OP ')' +174,28-174,29: LPAR '(' +174,29-174,30: RPAR ')' 174,30-174,31: NEWLINE '\n' 175,0-175,1: NL '\n' -176,0-176,1: OP '@' +176,0-176,1: AT '@' 176,1-176,13: NAME 'staticmethod' 176,13-176,14: NEWLINE '\n' 177,0-177,3: NAME 'def' 177,4-177,7: NAME 'foo' -177,7-177,8: OP '(' -177,8-177,9: OP ')' -177,9-177,10: OP ':' +177,7-177,8: LPAR '(' +177,8-177,9: RPAR ')' +177,9-177,10: COLON ':' 177,11-177,15: NAME 'pass' 177,15-177,16: NEWLINE '\n' 178,0-178,1: NL '\n'