Index: Python/mystrtoul.c =================================================================== --- Python/mystrtoul.c (revision 54387) +++ Python/mystrtoul.c (working copy) @@ -119,6 +119,14 @@ ++str; base = 16; } + else if (*str == 't' || *str == 'T') { + ++str; + base = 8; + } + else if (*str == 'b' || *str == 'B') { + ++str; + base = 2; + } else base = 8; } @@ -133,6 +141,20 @@ ++str; } break; + case 8: + if (*str == '0') { + ++str; + if (*str == 't' || *str == 'T') + ++str; + } + break; + case 2: + if (*str == '0') { + ++str; + if (*str == 'b' || *str == 'B') + ++str; + } + break; } /* catch silly bases */ Index: Objects/longobject.c =================================================================== --- Objects/longobject.c (revision 54387) +++ Objects/longobject.c (working copy) @@ -1455,10 +1455,17 @@ base = 10; else if (str[1] == 'x' || str[1] == 'X') base = 16; + else if (str[1] == 't' || str[1] == 'T') + base = 8; + else if (str[1] == 'b' || str[1] == 'B') + base = 2; else base = 8; } - if (base == 16 && str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) + if (str[0] == '0' && + ((base == 16 && (str[1] == 'x' || str[1] == 'X')) || + (base == 8 && (str[1] == 't' || str[1] == 'T')) || + (base == 2 && (str[1] == 'b' || str[1] == 'B')))) str += 2; start = str; Index: Parser/tokenizer.c =================================================================== --- Parser/tokenizer.c (revision 54387) +++ Parser/tokenizer.c (working copy) @@ -1328,6 +1328,18 @@ c = tok_nextc(tok); } while (isxdigit(c)); } + else if (c == 'b' || c == 'B') { + do { + c = tok_nextc(tok); + } while (c == '0' || c == '1'); + } + else if (c == 't' || c == 'T') { + /* Separate from the no-prefix case because + we don't try to deal with floats */ + do { + c = tok_nextc(tok); + } while ('0' <= c && c < '8'); + } else { int found_decimal = 0; /* Octal; c is first char of it */ Index: Lib/tokenize.py =================================================================== --- Lib/tokenize.py (revision 54387) +++ Lib/tokenize.py (working copy) @@ -51,9 +51,10 @@ Name = r'[a-zA-Z_]\w*' Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[0-7]*[lL]?' +Binnumber = r'0[bB][01]*[lL]?' +Octnumber = r'0[tT]?[0-7]*[lL]?' Decnumber = r'[1-9]\d*[lL]?' -Intnumber = group(Hexnumber, Octnumber, Decnumber) +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) Exponent = r'[eE][-+]?\d+' Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) Expfloat = r'\d+' + Exponent Index: Lib/test/tokenize_tests.txt =================================================================== --- Lib/test/tokenize_tests.txt (revision 54387) +++ Lib/test/tokenize_tests.txt (working copy) @@ -42,6 +42,9 @@ -2147483647-1 != 020000000000 037777777777 != -1 0xffffffff != -1 +0t37777777777 != -1 +-0t1234567 == 0T001234567 +0b10101 == 0B00010101 # Long integers x = 0L @@ -52,6 +55,10 @@ x = 077777777777777777l x = 123456789012345678901234567890L x = 123456789012345678901234567890l +x = 0t77777777777777777L +x = 0T77777777777777777l +x = 0b111111010111101011L +x = 0B111010101111111111l # trailing small L # Floating-point numbers x = 3.14 Index: Lib/test/output/test_tokenize =================================================================== --- Lib/test/output/test_tokenize (revision 54387) +++ Lib/test/output/test_tokenize (working copy) @@ -148,537 +148,568 @@ 44,14-44,15: OP '-' 44,15-44,16: NUMBER '1' 44,16-44,17: NEWLINE '\n' -45,0-45,1: NL '\n' -46,0-46,15: COMMENT '# Long integers' -46,15-46,16: NL '\n' -47,0-47,1: NAME 'x' -47,2-47,3: OP '=' -47,4-47,6: NUMBER '0L' -47,6-47,7: NEWLINE '\n' -48,0-48,1: NAME 'x' -48,2-48,3: OP '=' -48,4-48,6: NUMBER '0l' -48,6-48,7: NEWLINE '\n' -49,0-49,1: NAME 'x' -49,2-49,3: OP '=' -49,4-49,23: NUMBER '0xffffffffffffffffL' -49,23-49,24: NEWLINE '\n' +45,0-45,13: NUMBER '0t37777777777' +45,14-45,16: OP '!=' +45,17-45,18: OP '-' +45,18-45,19: NUMBER '1' +45,19-45,20: NEWLINE '\n' +46,0-46,1: OP '-' +46,1-46,10: NUMBER '0t1234567' +46,11-46,13: OP '==' +46,14-46,25: NUMBER '0T001234567' +46,25-46,26: NEWLINE '\n' +47,0-47,7: NUMBER '0b10101' +47,8-47,10: OP '==' +47,11-47,21: NUMBER '0B00010101' +47,21-47,22: NEWLINE '\n' +48,0-48,1: NL '\n' +49,0-49,15: COMMENT '# Long integers' +49,15-49,16: NL '\n' 50,0-50,1: NAME 'x' 50,2-50,3: OP '=' -50,4-50,23: NUMBER '0xffffffffffffffffl' -50,23-50,24: NEWLINE '\n' +50,4-50,6: NUMBER '0L' +50,6-50,7: NEWLINE '\n' 51,0-51,1: NAME 'x' 51,2-51,3: OP '=' -51,4-51,23: NUMBER '077777777777777777L' -51,23-51,24: NEWLINE '\n' +51,4-51,6: NUMBER '0l' +51,6-51,7: NEWLINE '\n' 52,0-52,1: NAME 'x' 52,2-52,3: OP '=' -52,4-52,23: NUMBER '077777777777777777l' +52,4-52,23: NUMBER '0xffffffffffffffffL' 52,23-52,24: NEWLINE '\n' 53,0-53,1: NAME 'x' 53,2-53,3: OP '=' -53,4-53,35: NUMBER '123456789012345678901234567890L' -53,35-53,36: NEWLINE '\n' +53,4-53,23: NUMBER '0xffffffffffffffffl' +53,23-53,24: NEWLINE '\n' 54,0-54,1: NAME 'x' 54,2-54,3: OP '=' -54,4-54,35: NUMBER '123456789012345678901234567890l' -54,35-54,36: NEWLINE '\n' -55,0-55,1: NL '\n' -56,0-56,24: COMMENT '# Floating-point numbers' -56,24-56,25: NL '\n' +54,4-54,23: NUMBER '077777777777777777L' +54,23-54,24: NEWLINE '\n' +55,0-55,1: NAME 'x' +55,2-55,3: OP '=' +55,4-55,23: NUMBER '077777777777777777l' +55,23-55,24: NEWLINE '\n' +56,0-56,1: NAME 'x' +56,2-56,3: OP '=' +56,4-56,35: NUMBER '123456789012345678901234567890L' +56,35-56,36: NEWLINE '\n' 57,0-57,1: NAME 'x' 57,2-57,3: OP '=' -57,4-57,8: NUMBER '3.14' -57,8-57,9: NEWLINE '\n' +57,4-57,35: NUMBER '123456789012345678901234567890l' +57,35-57,36: NEWLINE '\n' 58,0-58,1: NAME 'x' 58,2-58,3: OP '=' -58,4-58,8: NUMBER '314.' -58,8-58,9: NEWLINE '\n' +58,4-58,24: NUMBER '0t77777777777777777L' +58,24-58,25: NEWLINE '\n' 59,0-59,1: NAME 'x' 59,2-59,3: OP '=' -59,4-59,9: NUMBER '0.314' -59,9-59,10: NEWLINE '\n' -60,0-60,17: COMMENT '# XXX x = 000.314' -60,17-60,18: NL '\n' +59,4-59,24: NUMBER '0T77777777777777777l' +59,24-59,25: NEWLINE '\n' +60,0-60,1: NAME 'x' +60,2-60,3: OP '=' +60,4-60,25: NUMBER '0b111111010111101011L' +60,25-60,26: NEWLINE '\n' 61,0-61,1: NAME 'x' 61,2-61,3: OP '=' -61,4-61,8: NUMBER '.314' -61,8-61,9: NEWLINE '\n' -62,0-62,1: NAME 'x' -62,2-62,3: OP '=' -62,4-62,8: NUMBER '3e14' -62,8-62,9: NEWLINE '\n' -63,0-63,1: NAME 'x' -63,2-63,3: OP '=' -63,4-63,8: NUMBER '3E14' -63,8-63,9: NEWLINE '\n' +61,4-61,25: NUMBER '0B111010101111111111l' +61,26-61,44: COMMENT '# trailing small L' +61,44-61,45: NEWLINE '\n' +62,0-62,1: NL '\n' +63,0-63,24: COMMENT '# Floating-point numbers' +63,24-63,25: NL '\n' 64,0-64,1: NAME 'x' 64,2-64,3: OP '=' -64,4-64,9: NUMBER '3e-14' -64,9-64,10: NEWLINE '\n' +64,4-64,8: NUMBER '3.14' +64,8-64,9: NEWLINE '\n' 65,0-65,1: NAME 'x' 65,2-65,3: OP '=' -65,4-65,9: NUMBER '3e+14' -65,9-65,10: NEWLINE '\n' +65,4-65,8: NUMBER '314.' +65,8-65,9: NEWLINE '\n' 66,0-66,1: NAME 'x' 66,2-66,3: OP '=' -66,4-66,9: NUMBER '3.e14' +66,4-66,9: NUMBER '0.314' 66,9-66,10: NEWLINE '\n' -67,0-67,1: NAME 'x' -67,2-67,3: OP '=' -67,4-67,9: NUMBER '.3e14' -67,9-67,10: NEWLINE '\n' +67,0-67,17: COMMENT '# XXX x = 000.314' +67,17-67,18: NL '\n' 68,0-68,1: NAME 'x' 68,2-68,3: OP '=' -68,4-68,9: NUMBER '3.1e4' -68,9-68,10: NEWLINE '\n' -69,0-69,1: NL '\n' -70,0-70,17: COMMENT '# String literals' -70,17-70,18: NL '\n' +68,4-68,8: NUMBER '.314' +68,8-68,9: NEWLINE '\n' +69,0-69,1: NAME 'x' +69,2-69,3: OP '=' +69,4-69,8: NUMBER '3e14' +69,8-69,9: NEWLINE '\n' +70,0-70,1: NAME 'x' +70,2-70,3: OP '=' +70,4-70,8: NUMBER '3E14' +70,8-70,9: NEWLINE '\n' 71,0-71,1: NAME 'x' 71,2-71,3: OP '=' -71,4-71,6: STRING "''" -71,6-71,7: OP ';' -71,8-71,9: NAME 'y' -71,10-71,11: OP '=' -71,12-71,14: STRING '""' -71,14-71,15: OP ';' -71,15-71,16: NEWLINE '\n' +71,4-71,9: NUMBER '3e-14' +71,9-71,10: NEWLINE '\n' 72,0-72,1: NAME 'x' 72,2-72,3: OP '=' -72,4-72,8: STRING "'\\''" -72,8-72,9: OP ';' -72,10-72,11: NAME 'y' -72,12-72,13: OP '=' -72,14-72,17: STRING '"\'"' -72,17-72,18: OP ';' -72,18-72,19: NEWLINE '\n' +72,4-72,9: NUMBER '3e+14' +72,9-72,10: NEWLINE '\n' 73,0-73,1: NAME 'x' 73,2-73,3: OP '=' -73,4-73,7: STRING '\'"\'' -73,7-73,8: OP ';' -73,9-73,10: NAME 'y' -73,11-73,12: OP '=' -73,13-73,17: STRING '"\\""' -73,17-73,18: OP ';' -73,18-73,19: NEWLINE '\n' +73,4-73,9: NUMBER '3.e14' +73,9-73,10: NEWLINE '\n' 74,0-74,1: NAME 'x' 74,2-74,3: OP '=' -74,4-74,32: STRING '"doesn\'t \\"shrink\\" does it"' -74,32-74,33: NEWLINE '\n' -75,0-75,1: NAME 'y' +74,4-74,9: NUMBER '.3e14' +74,9-74,10: NEWLINE '\n' +75,0-75,1: NAME 'x' 75,2-75,3: OP '=' -75,4-75,31: STRING '\'doesn\\\'t "shrink" does it\'' -75,31-75,32: NEWLINE '\n' -76,0-76,1: NAME 'x' -76,2-76,3: OP '=' -76,4-76,32: STRING '"does \\"shrink\\" doesn\'t it"' -76,32-76,33: NEWLINE '\n' -77,0-77,1: NAME 'y' -77,2-77,3: OP '=' -77,4-77,31: STRING '\'does "shrink" doesn\\\'t it\'' -77,31-77,32: NEWLINE '\n' +75,4-75,9: NUMBER '3.1e4' +75,9-75,10: NEWLINE '\n' +76,0-76,1: NL '\n' +77,0-77,17: COMMENT '# String literals' +77,17-77,18: NL '\n' 78,0-78,1: NAME 'x' 78,2-78,3: OP '=' -78,4-83,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""' -83,3-83,4: NEWLINE '\n' +78,4-78,6: STRING "''" +78,6-78,7: OP ';' +78,8-78,9: NAME 'y' +78,10-78,11: OP '=' +78,12-78,14: STRING '""' +78,14-78,15: OP ';' +78,15-78,16: NEWLINE '\n' +79,0-79,1: NAME 'x' +79,2-79,3: OP '=' +79,4-79,8: STRING "'\\''" +79,8-79,9: OP ';' +79,10-79,11: NAME 'y' +79,12-79,13: OP '=' +79,14-79,17: STRING '"\'"' +79,17-79,18: OP ';' +79,18-79,19: NEWLINE '\n' +80,0-80,1: NAME 'x' +80,2-80,3: OP '=' +80,4-80,7: STRING '\'"\'' +80,7-80,8: OP ';' +80,9-80,10: NAME 'y' +80,11-80,12: OP '=' +80,13-80,17: STRING '"\\""' +80,17-80,18: OP ';' +80,18-80,19: NEWLINE '\n' +81,0-81,1: NAME 'x' +81,2-81,3: OP '=' +81,4-81,32: STRING '"doesn\'t \\"shrink\\" does it"' +81,32-81,33: NEWLINE '\n' +82,0-82,1: NAME 'y' +82,2-82,3: OP '=' +82,4-82,31: STRING '\'doesn\\\'t "shrink" does it\'' +82,31-82,32: NEWLINE '\n' +83,0-83,1: NAME 'x' +83,2-83,3: OP '=' +83,4-83,32: STRING '"does \\"shrink\\" doesn\'t it"' +83,32-83,33: NEWLINE '\n' 84,0-84,1: NAME 'y' 84,2-84,3: OP '=' -84,4-84,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\'' -84,63-84,64: NEWLINE '\n' -85,0-85,1: NAME 'y' +84,4-84,31: STRING '\'does "shrink" doesn\\\'t it\'' +84,31-84,32: NEWLINE '\n' +85,0-85,1: NAME 'x' 85,2-85,3: OP '=' -85,4-90,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\'' -90,3-90,4: OP ';' -90,4-90,5: NEWLINE '\n' +85,4-90,3: STRING '"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""' +90,3-90,4: NEWLINE '\n' 91,0-91,1: NAME 'y' 91,2-91,3: OP '=' -91,4-96,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"' -96,1-96,2: OP ';' -96,2-96,3: NEWLINE '\n' -97,0-97,1: NAME 'y' -97,2-97,3: OP '=' -97,4-102,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\'' -102,1-102,2: OP ';' -102,2-102,3: NEWLINE '\n' -103,0-103,1: NAME 'x' -103,2-103,3: OP '=' -103,4-103,9: STRING "r'\\\\'" -103,10-103,11: OP '+' -103,12-103,17: STRING "R'\\\\'" -103,17-103,18: NEWLINE '\n' -104,0-104,1: NAME 'x' +91,4-91,63: STRING '\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\'' +91,63-91,64: NEWLINE '\n' +92,0-92,1: NAME 'y' +92,2-92,3: OP '=' +92,4-97,3: STRING '\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\'' +97,3-97,4: OP ';' +97,4-97,5: NEWLINE '\n' +98,0-98,1: NAME 'y' +98,2-98,3: OP '=' +98,4-103,1: STRING '"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"' +103,1-103,2: OP ';' +103,2-103,3: NEWLINE '\n' +104,0-104,1: NAME 'y' 104,2-104,3: OP '=' -104,4-104,9: STRING "r'\\''" -104,10-104,11: OP '+' -104,12-104,14: STRING "''" -104,14-104,15: NEWLINE '\n' -105,0-105,1: NAME 'y' -105,2-105,3: OP '=' -105,4-107,6: STRING "r'''\nfoo bar \\\\\nbaz'''" -107,7-107,8: OP '+' -107,9-108,6: STRING "R'''\nfoo'''" -108,6-108,7: NEWLINE '\n' -109,0-109,1: NAME 'y' -109,2-109,3: OP '=' -109,4-111,3: STRING 'r"""foo\nbar \\\\ baz\n"""' -111,4-111,5: OP '+' -111,6-112,3: STRING "R'''spam\n'''" -112,3-112,4: NEWLINE '\n' -113,0-113,1: NAME 'x' -113,2-113,3: OP '=' -113,4-113,10: STRING "u'abc'" -113,11-113,12: OP '+' -113,13-113,19: STRING "U'ABC'" -113,19-113,20: NEWLINE '\n' -114,0-114,1: NAME 'y' -114,2-114,3: OP '=' -114,4-114,10: STRING 'u"abc"' -114,11-114,12: OP '+' -114,13-114,19: STRING 'U"ABC"' -114,19-114,20: NEWLINE '\n' -115,0-115,1: NAME 'x' -115,2-115,3: OP '=' -115,4-115,11: STRING "ur'abc'" -115,12-115,13: OP '+' -115,14-115,21: STRING "Ur'ABC'" -115,22-115,23: OP '+' -115,24-115,31: STRING "uR'ABC'" -115,32-115,33: OP '+' -115,34-115,41: STRING "UR'ABC'" -115,41-115,42: NEWLINE '\n' +104,4-109,1: STRING '\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\'' +109,1-109,2: OP ';' +109,2-109,3: NEWLINE '\n' +110,0-110,1: NAME 'x' +110,2-110,3: OP '=' +110,4-110,9: STRING "r'\\\\'" +110,10-110,11: OP '+' +110,12-110,17: STRING "R'\\\\'" +110,17-110,18: NEWLINE '\n' +111,0-111,1: NAME 'x' +111,2-111,3: OP '=' +111,4-111,9: STRING "r'\\''" +111,10-111,11: OP '+' +111,12-111,14: STRING "''" +111,14-111,15: NEWLINE '\n' +112,0-112,1: NAME 'y' +112,2-112,3: OP '=' +112,4-114,6: STRING "r'''\nfoo bar \\\\\nbaz'''" +114,7-114,8: OP '+' +114,9-115,6: STRING "R'''\nfoo'''" +115,6-115,7: NEWLINE '\n' 116,0-116,1: NAME 'y' 116,2-116,3: OP '=' -116,4-116,11: STRING 'ur"abc"' -116,12-116,13: OP '+' -116,14-116,21: STRING 'Ur"ABC"' -116,22-116,23: OP '+' -116,24-116,31: STRING 'uR"ABC"' -116,32-116,33: OP '+' -116,34-116,41: STRING 'UR"ABC"' -116,41-116,42: NEWLINE '\n' -117,0-117,1: NAME 'x' -117,2-117,3: OP '=' -117,4-117,10: STRING "ur'\\\\'" -117,11-117,12: OP '+' -117,13-117,19: STRING "UR'\\\\'" -117,19-117,20: NEWLINE '\n' -118,0-118,1: NAME 'x' -118,2-118,3: OP '=' -118,4-118,10: STRING "ur'\\''" -118,11-118,12: OP '+' -118,13-118,15: STRING "''" -118,15-118,16: NEWLINE '\n' -119,0-119,1: NAME 'y' -119,2-119,3: OP '=' -119,4-121,6: STRING "ur'''\nfoo bar \\\\\nbaz'''" -121,7-121,8: OP '+' -121,9-122,6: STRING "UR'''\nfoo'''" -122,6-122,7: NEWLINE '\n' +116,4-118,3: STRING 'r"""foo\nbar \\\\ baz\n"""' +118,4-118,5: OP '+' +118,6-119,3: STRING "R'''spam\n'''" +119,3-119,4: NEWLINE '\n' +120,0-120,1: NAME 'x' +120,2-120,3: OP '=' +120,4-120,10: STRING "u'abc'" +120,11-120,12: OP '+' +120,13-120,19: STRING "U'ABC'" +120,19-120,20: NEWLINE '\n' +121,0-121,1: NAME 'y' +121,2-121,3: OP '=' +121,4-121,10: STRING 'u"abc"' +121,11-121,12: OP '+' +121,13-121,19: STRING 'U"ABC"' +121,19-121,20: NEWLINE '\n' +122,0-122,1: NAME 'x' +122,2-122,3: OP '=' +122,4-122,11: STRING "ur'abc'" +122,12-122,13: OP '+' +122,14-122,21: STRING "Ur'ABC'" +122,22-122,23: OP '+' +122,24-122,31: STRING "uR'ABC'" +122,32-122,33: OP '+' +122,34-122,41: STRING "UR'ABC'" +122,41-122,42: NEWLINE '\n' 123,0-123,1: NAME 'y' 123,2-123,3: OP '=' -123,4-125,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""' -125,4-125,5: OP '+' -125,6-126,3: STRING "uR'''spam\n'''" -126,3-126,4: NEWLINE '\n' -127,0-127,1: NL '\n' -128,0-128,13: COMMENT '# Indentation' -128,13-128,14: NL '\n' -129,0-129,2: NAME 'if' -129,3-129,4: NUMBER '1' -129,4-129,5: OP ':' -129,5-129,6: NEWLINE '\n' -130,0-130,4: INDENT ' ' -130,4-130,5: NAME 'x' -130,6-130,7: OP '=' -130,8-130,9: NUMBER '2' -130,9-130,10: NEWLINE '\n' -131,0-131,0: DEDENT '' -131,0-131,2: NAME 'if' -131,3-131,4: NUMBER '1' -131,4-131,5: OP ':' -131,5-131,6: NEWLINE '\n' -132,0-132,8: INDENT ' ' -132,8-132,9: NAME 'x' -132,10-132,11: OP '=' -132,12-132,13: NUMBER '2' -132,13-132,14: NEWLINE '\n' -133,0-133,0: DEDENT '' -133,0-133,2: NAME 'if' -133,3-133,4: NUMBER '1' -133,4-133,5: OP ':' -133,5-133,6: NEWLINE '\n' -134,0-134,4: INDENT ' ' -134,4-134,9: NAME 'while' -134,10-134,11: NUMBER '0' -134,11-134,12: OP ':' -134,12-134,13: NEWLINE '\n' -135,0-135,5: INDENT ' ' -135,5-135,7: NAME 'if' -135,8-135,9: NUMBER '0' -135,9-135,10: OP ':' -135,10-135,11: NEWLINE '\n' -136,0-136,11: INDENT ' ' -136,11-136,12: NAME 'x' -136,13-136,14: OP '=' -136,15-136,16: NUMBER '2' -136,16-136,17: NEWLINE '\n' -137,5-137,5: DEDENT '' -137,5-137,6: NAME 'x' -137,7-137,8: OP '=' -137,9-137,10: NUMBER '2' -137,10-137,11: NEWLINE '\n' +123,4-123,11: STRING 'ur"abc"' +123,12-123,13: OP '+' +123,14-123,21: STRING 'Ur"ABC"' +123,22-123,23: OP '+' +123,24-123,31: STRING 'uR"ABC"' +123,32-123,33: OP '+' +123,34-123,41: STRING 'UR"ABC"' +123,41-123,42: NEWLINE '\n' +124,0-124,1: NAME 'x' +124,2-124,3: OP '=' +124,4-124,10: STRING "ur'\\\\'" +124,11-124,12: OP '+' +124,13-124,19: STRING "UR'\\\\'" +124,19-124,20: NEWLINE '\n' +125,0-125,1: NAME 'x' +125,2-125,3: OP '=' +125,4-125,10: STRING "ur'\\''" +125,11-125,12: OP '+' +125,13-125,15: STRING "''" +125,15-125,16: NEWLINE '\n' +126,0-126,1: NAME 'y' +126,2-126,3: OP '=' +126,4-128,6: STRING "ur'''\nfoo bar \\\\\nbaz'''" +128,7-128,8: OP '+' +128,9-129,6: STRING "UR'''\nfoo'''" +129,6-129,7: NEWLINE '\n' +130,0-130,1: NAME 'y' +130,2-130,3: OP '=' +130,4-132,3: STRING 'Ur"""foo\nbar \\\\ baz\n"""' +132,4-132,5: OP '+' +132,6-133,3: STRING "uR'''spam\n'''" +133,3-133,4: NEWLINE '\n' +134,0-134,1: NL '\n' +135,0-135,13: COMMENT '# Indentation' +135,13-135,14: NL '\n' +136,0-136,2: NAME 'if' +136,3-136,4: NUMBER '1' +136,4-136,5: OP ':' +136,5-136,6: NEWLINE '\n' +137,0-137,4: INDENT ' ' +137,4-137,5: NAME 'x' +137,6-137,7: OP '=' +137,8-137,9: NUMBER '2' +137,9-137,10: NEWLINE '\n' 138,0-138,0: DEDENT '' -138,0-138,0: DEDENT '' 138,0-138,2: NAME 'if' -138,3-138,4: NUMBER '0' +138,3-138,4: NUMBER '1' 138,4-138,5: OP ':' 138,5-138,6: NEWLINE '\n' -139,0-139,2: INDENT ' ' -139,2-139,4: NAME 'if' -139,5-139,6: NUMBER '2' -139,6-139,7: OP ':' -139,7-139,8: NEWLINE '\n' -140,0-140,3: INDENT ' ' -140,3-140,8: NAME 'while' -140,9-140,10: NUMBER '0' -140,10-140,11: OP ':' -140,11-140,12: NEWLINE '\n' -141,0-141,8: INDENT ' ' -141,8-141,10: NAME 'if' -141,11-141,12: NUMBER '1' -141,12-141,13: OP ':' -141,13-141,14: NEWLINE '\n' -142,0-142,10: INDENT ' ' -142,10-142,11: NAME 'x' -142,12-142,13: OP '=' -142,14-142,15: NUMBER '2' -142,15-142,16: NEWLINE '\n' -143,0-143,1: NL '\n' -144,0-144,11: COMMENT '# Operators' -144,11-144,12: NL '\n' -145,0-145,1: NL '\n' -146,0-146,0: DEDENT '' -146,0-146,0: DEDENT '' -146,0-146,0: DEDENT '' -146,0-146,0: DEDENT '' -146,0-146,3: NAME 'def' -146,4-146,7: NAME 'd22' -146,7-146,8: OP '(' -146,8-146,9: NAME 'a' -146,9-146,10: OP ',' -146,11-146,12: NAME 'b' -146,12-146,13: OP ',' -146,14-146,15: NAME 'c' -146,15-146,16: OP '=' -146,16-146,17: NUMBER '1' -146,17-146,18: OP ',' -146,19-146,20: NAME 'd' -146,20-146,21: OP '=' -146,21-146,22: NUMBER '2' -146,22-146,23: OP ')' -146,23-146,24: OP ':' -146,25-146,29: NAME 'pass' -146,29-146,30: NEWLINE '\n' -147,0-147,3: NAME 'def' -147,4-147,8: NAME 'd01v' -147,8-147,9: OP '(' -147,9-147,10: NAME 'a' -147,10-147,11: OP '=' -147,11-147,12: NUMBER '1' -147,12-147,13: OP ',' -147,14-147,15: OP '*' -147,15-147,20: NAME 'restt' -147,20-147,21: OP ',' -147,22-147,24: OP '**' -147,24-147,29: NAME 'restd' -147,29-147,30: OP ')' -147,30-147,31: OP ':' -147,32-147,36: NAME 'pass' -147,36-147,37: NEWLINE '\n' -148,0-148,1: NL '\n' -149,0-149,1: OP '(' -149,1-149,2: NAME 'x' -149,2-149,3: OP ',' -149,4-149,5: NAME 'y' -149,5-149,6: OP ')' -149,7-149,9: OP '<>' -149,10-149,11: OP '(' -149,11-149,12: OP '{' -149,12-149,15: STRING "'a'" -149,15-149,16: OP ':' -149,16-149,17: NUMBER '1' -149,17-149,18: OP '}' -149,18-149,19: OP ',' -149,20-149,21: OP '{' -149,21-149,24: STRING "'b'" -149,24-149,25: OP ':' -149,25-149,26: NUMBER '2' -149,26-149,27: OP '}' -149,27-149,28: OP ')' -149,28-149,29: NEWLINE '\n' +139,0-139,8: INDENT ' ' +139,8-139,9: NAME 'x' +139,10-139,11: OP '=' +139,12-139,13: NUMBER '2' +139,13-139,14: NEWLINE '\n' +140,0-140,0: DEDENT '' +140,0-140,2: NAME 'if' +140,3-140,4: NUMBER '1' +140,4-140,5: OP ':' +140,5-140,6: NEWLINE '\n' +141,0-141,4: INDENT ' ' +141,4-141,9: NAME 'while' +141,10-141,11: NUMBER '0' +141,11-141,12: OP ':' +141,12-141,13: NEWLINE '\n' +142,0-142,5: INDENT ' ' +142,5-142,7: NAME 'if' +142,8-142,9: NUMBER '0' +142,9-142,10: OP ':' +142,10-142,11: NEWLINE '\n' +143,0-143,11: INDENT ' ' +143,11-143,12: NAME 'x' +143,13-143,14: OP '=' +143,15-143,16: NUMBER '2' +143,16-143,17: NEWLINE '\n' +144,5-144,5: DEDENT '' +144,5-144,6: NAME 'x' +144,7-144,8: OP '=' +144,9-144,10: NUMBER '2' +144,10-144,11: NEWLINE '\n' +145,0-145,0: DEDENT '' +145,0-145,0: DEDENT '' +145,0-145,2: NAME 'if' +145,3-145,4: NUMBER '0' +145,4-145,5: OP ':' +145,5-145,6: NEWLINE '\n' +146,0-146,2: INDENT ' ' +146,2-146,4: NAME 'if' +146,5-146,6: NUMBER '2' +146,6-146,7: OP ':' +146,7-146,8: NEWLINE '\n' +147,0-147,3: INDENT ' ' +147,3-147,8: NAME 'while' +147,9-147,10: NUMBER '0' +147,10-147,11: OP ':' +147,11-147,12: NEWLINE '\n' +148,0-148,8: INDENT ' ' +148,8-148,10: NAME 'if' +148,11-148,12: NUMBER '1' +148,12-148,13: OP ':' +148,13-148,14: NEWLINE '\n' +149,0-149,10: INDENT ' ' +149,10-149,11: NAME 'x' +149,12-149,13: OP '=' +149,14-149,15: NUMBER '2' +149,15-149,16: NEWLINE '\n' 150,0-150,1: NL '\n' -151,0-151,12: COMMENT '# comparison' -151,12-151,13: NL '\n' -152,0-152,2: NAME 'if' -152,3-152,4: NUMBER '1' -152,5-152,6: OP '<' -152,7-152,8: NUMBER '1' -152,9-152,10: OP '>' -152,11-152,12: NUMBER '1' -152,13-152,15: OP '==' -152,16-152,17: NUMBER '1' -152,18-152,20: OP '>=' -152,21-152,22: NUMBER '1' -152,23-152,25: OP '<=' -152,26-152,27: NUMBER '1' -152,28-152,30: OP '<>' -152,31-152,32: NUMBER '1' -152,33-152,35: OP '!=' -152,36-152,37: NUMBER '1' -152,38-152,40: NAME 'in' -152,41-152,42: NUMBER '1' -152,43-152,46: NAME 'not' -152,47-152,49: NAME 'in' -152,50-152,51: NUMBER '1' -152,52-152,54: NAME 'is' -152,55-152,56: NUMBER '1' -152,57-152,59: NAME 'is' -152,60-152,63: NAME 'not' -152,64-152,65: NUMBER '1' -152,65-152,66: OP ':' -152,67-152,71: NAME 'pass' -152,71-152,72: NEWLINE '\n' -153,0-153,1: NL '\n' -154,0-154,8: COMMENT '# binary' -154,8-154,9: NL '\n' -155,0-155,1: NAME 'x' -155,2-155,3: OP '=' -155,4-155,5: NUMBER '1' -155,6-155,7: OP '&' -155,8-155,9: NUMBER '1' -155,9-155,10: NEWLINE '\n' -156,0-156,1: NAME 'x' -156,2-156,3: OP '=' -156,4-156,5: NUMBER '1' -156,6-156,7: OP '^' -156,8-156,9: NUMBER '1' -156,9-156,10: NEWLINE '\n' -157,0-157,1: NAME 'x' -157,2-157,3: OP '=' -157,4-157,5: NUMBER '1' -157,6-157,7: OP '|' -157,8-157,9: NUMBER '1' -157,9-157,10: NEWLINE '\n' -158,0-158,1: NL '\n' -159,0-159,7: COMMENT '# shift' -159,7-159,8: NL '\n' -160,0-160,1: NAME 'x' -160,2-160,3: OP '=' -160,4-160,5: NUMBER '1' -160,6-160,8: OP '<<' -160,9-160,10: NUMBER '1' -160,11-160,13: OP '>>' -160,14-160,15: NUMBER '1' -160,15-160,16: NEWLINE '\n' -161,0-161,1: NL '\n' -162,0-162,10: COMMENT '# additive' -162,10-162,11: NL '\n' +151,0-151,11: COMMENT '# Operators' +151,11-151,12: NL '\n' +152,0-152,1: NL '\n' +153,0-153,0: DEDENT '' +153,0-153,0: DEDENT '' +153,0-153,0: DEDENT '' +153,0-153,0: DEDENT '' +153,0-153,3: NAME 'def' +153,4-153,7: NAME 'd22' +153,7-153,8: OP '(' +153,8-153,9: NAME 'a' +153,9-153,10: OP ',' +153,11-153,12: NAME 'b' +153,12-153,13: OP ',' +153,14-153,15: NAME 'c' +153,15-153,16: OP '=' +153,16-153,17: NUMBER '1' +153,17-153,18: OP ',' +153,19-153,20: NAME 'd' +153,20-153,21: OP '=' +153,21-153,22: NUMBER '2' +153,22-153,23: OP ')' +153,23-153,24: OP ':' +153,25-153,29: NAME 'pass' +153,29-153,30: NEWLINE '\n' +154,0-154,3: NAME 'def' +154,4-154,8: NAME 'd01v' +154,8-154,9: OP '(' +154,9-154,10: NAME 'a' +154,10-154,11: OP '=' +154,11-154,12: NUMBER '1' +154,12-154,13: OP ',' +154,14-154,15: OP '*' +154,15-154,20: NAME 'restt' +154,20-154,21: OP ',' +154,22-154,24: OP '**' +154,24-154,29: NAME 'restd' +154,29-154,30: OP ')' +154,30-154,31: OP ':' +154,32-154,36: NAME 'pass' +154,36-154,37: NEWLINE '\n' +155,0-155,1: NL '\n' +156,0-156,1: OP '(' +156,1-156,2: NAME 'x' +156,2-156,3: OP ',' +156,4-156,5: NAME 'y' +156,5-156,6: OP ')' +156,7-156,9: OP '<>' +156,10-156,11: OP '(' +156,11-156,12: OP '{' +156,12-156,15: STRING "'a'" +156,15-156,16: OP ':' +156,16-156,17: NUMBER '1' +156,17-156,18: OP '}' +156,18-156,19: OP ',' +156,20-156,21: OP '{' +156,21-156,24: STRING "'b'" +156,24-156,25: OP ':' +156,25-156,26: NUMBER '2' +156,26-156,27: OP '}' +156,27-156,28: OP ')' +156,28-156,29: NEWLINE '\n' +157,0-157,1: NL '\n' +158,0-158,12: COMMENT '# comparison' +158,12-158,13: NL '\n' +159,0-159,2: NAME 'if' +159,3-159,4: NUMBER '1' +159,5-159,6: OP '<' +159,7-159,8: NUMBER '1' +159,9-159,10: OP '>' +159,11-159,12: NUMBER '1' +159,13-159,15: OP '==' +159,16-159,17: NUMBER '1' +159,18-159,20: OP '>=' +159,21-159,22: NUMBER '1' +159,23-159,25: OP '<=' +159,26-159,27: NUMBER '1' +159,28-159,30: OP '<>' +159,31-159,32: NUMBER '1' +159,33-159,35: OP '!=' +159,36-159,37: NUMBER '1' +159,38-159,40: NAME 'in' +159,41-159,42: NUMBER '1' +159,43-159,46: NAME 'not' +159,47-159,49: NAME 'in' +159,50-159,51: NUMBER '1' +159,52-159,54: NAME 'is' +159,55-159,56: NUMBER '1' +159,57-159,59: NAME 'is' +159,60-159,63: NAME 'not' +159,64-159,65: NUMBER '1' +159,65-159,66: OP ':' +159,67-159,71: NAME 'pass' +159,71-159,72: NEWLINE '\n' +160,0-160,1: NL '\n' +161,0-161,8: COMMENT '# binary' +161,8-161,9: NL '\n' +162,0-162,1: NAME 'x' +162,2-162,3: OP '=' +162,4-162,5: NUMBER '1' +162,6-162,7: OP '&' +162,8-162,9: NUMBER '1' +162,9-162,10: NEWLINE '\n' 163,0-163,1: NAME 'x' 163,2-163,3: OP '=' 163,4-163,5: NUMBER '1' -163,6-163,7: OP '-' +163,6-163,7: OP '^' 163,8-163,9: NUMBER '1' -163,10-163,11: OP '+' -163,12-163,13: NUMBER '1' -163,14-163,15: OP '-' -163,16-163,17: NUMBER '1' -163,18-163,19: OP '+' -163,20-163,21: NUMBER '1' -163,21-163,22: NEWLINE '\n' -164,0-164,1: NL '\n' -165,0-165,16: COMMENT '# multiplicative' -165,16-165,17: NL '\n' -166,0-166,1: NAME 'x' -166,2-166,3: OP '=' -166,4-166,5: NUMBER '1' -166,6-166,7: OP '/' -166,8-166,9: NUMBER '1' -166,10-166,11: OP '*' -166,12-166,13: NUMBER '1' -166,14-166,15: OP '%' -166,16-166,17: NUMBER '1' -166,17-166,18: NEWLINE '\n' -167,0-167,1: NL '\n' -168,0-168,7: COMMENT '# unary' -168,7-168,8: NL '\n' -169,0-169,1: NAME 'x' -169,2-169,3: OP '=' -169,4-169,5: OP '~' -169,5-169,6: NUMBER '1' -169,7-169,8: OP '^' -169,9-169,10: NUMBER '1' -169,11-169,12: OP '&' -169,13-169,14: NUMBER '1' -169,15-169,16: OP '|' -169,17-169,18: NUMBER '1' -169,19-169,20: OP '&' -169,21-169,22: NUMBER '1' -169,23-169,24: OP '^' -169,25-169,26: OP '-' -169,26-169,27: NUMBER '1' -169,27-169,28: NEWLINE '\n' +163,9-163,10: NEWLINE '\n' +164,0-164,1: NAME 'x' +164,2-164,3: OP '=' +164,4-164,5: NUMBER '1' +164,6-164,7: OP '|' +164,8-164,9: NUMBER '1' +164,9-164,10: NEWLINE '\n' +165,0-165,1: NL '\n' +166,0-166,7: COMMENT '# shift' +166,7-166,8: NL '\n' +167,0-167,1: NAME 'x' +167,2-167,3: OP '=' +167,4-167,5: NUMBER '1' +167,6-167,8: OP '<<' +167,9-167,10: NUMBER '1' +167,11-167,13: OP '>>' +167,14-167,15: NUMBER '1' +167,15-167,16: NEWLINE '\n' +168,0-168,1: NL '\n' +169,0-169,10: COMMENT '# additive' +169,10-169,11: NL '\n' 170,0-170,1: NAME 'x' 170,2-170,3: OP '=' -170,4-170,5: OP '-' -170,5-170,6: NUMBER '1' -170,6-170,7: OP '*' -170,7-170,8: NUMBER '1' -170,8-170,9: OP '/' -170,9-170,10: NUMBER '1' -170,11-170,12: OP '+' -170,13-170,14: NUMBER '1' -170,14-170,15: OP '*' -170,15-170,16: NUMBER '1' -170,17-170,18: OP '-' -170,19-170,20: OP '-' -170,20-170,21: OP '-' -170,21-170,22: OP '-' -170,22-170,23: NUMBER '1' -170,23-170,24: OP '*' -170,24-170,25: NUMBER '1' -170,25-170,26: NEWLINE '\n' +170,4-170,5: NUMBER '1' +170,6-170,7: OP '-' +170,8-170,9: NUMBER '1' +170,10-170,11: OP '+' +170,12-170,13: NUMBER '1' +170,14-170,15: OP '-' +170,16-170,17: NUMBER '1' +170,18-170,19: OP '+' +170,20-170,21: NUMBER '1' +170,21-170,22: NEWLINE '\n' 171,0-171,1: NL '\n' -172,0-172,10: COMMENT '# selector' -172,10-172,11: NL '\n' -173,0-173,6: NAME 'import' -173,7-173,10: NAME 'sys' -173,10-173,11: OP ',' -173,12-173,16: NAME 'time' -173,16-173,17: NEWLINE '\n' -174,0-174,1: NAME 'x' -174,2-174,3: OP '=' -174,4-174,7: NAME 'sys' -174,7-174,8: OP '.' -174,8-174,15: NAME 'modules' -174,15-174,16: OP '[' -174,16-174,22: STRING "'time'" -174,22-174,23: OP ']' -174,23-174,24: OP '.' -174,24-174,28: NAME 'time' -174,28-174,29: OP '(' -174,29-174,30: OP ')' -174,30-174,31: NEWLINE '\n' -175,0-175,1: NL '\n' -176,0-176,1: OP '@' -176,1-176,13: NAME 'staticmethod' -176,13-176,14: NEWLINE '\n' -177,0-177,3: NAME 'def' -177,4-177,7: NAME 'foo' -177,7-177,8: OP '(' -177,8-177,9: OP ')' -177,9-177,10: OP ':' -177,11-177,15: NAME 'pass' -177,15-177,16: NEWLINE '\n' +172,0-172,16: COMMENT '# multiplicative' +172,16-172,17: NL '\n' +173,0-173,1: NAME 'x' +173,2-173,3: OP '=' +173,4-173,5: NUMBER '1' +173,6-173,7: OP '/' +173,8-173,9: NUMBER '1' +173,10-173,11: OP '*' +173,12-173,13: NUMBER '1' +173,14-173,15: OP '%' +173,16-173,17: NUMBER '1' +173,17-173,18: NEWLINE '\n' +174,0-174,1: NL '\n' +175,0-175,7: COMMENT '# unary' +175,7-175,8: NL '\n' +176,0-176,1: NAME 'x' +176,2-176,3: OP '=' +176,4-176,5: OP '~' +176,5-176,6: NUMBER '1' +176,7-176,8: OP '^' +176,9-176,10: NUMBER '1' +176,11-176,12: OP '&' +176,13-176,14: NUMBER '1' +176,15-176,16: OP '|' +176,17-176,18: NUMBER '1' +176,19-176,20: OP '&' +176,21-176,22: NUMBER '1' +176,23-176,24: OP '^' +176,25-176,26: OP '-' +176,26-176,27: NUMBER '1' +176,27-176,28: NEWLINE '\n' +177,0-177,1: NAME 'x' +177,2-177,3: OP '=' +177,4-177,5: OP '-' +177,5-177,6: NUMBER '1' +177,6-177,7: OP '*' +177,7-177,8: NUMBER '1' +177,8-177,9: OP '/' +177,9-177,10: NUMBER '1' +177,11-177,12: OP '+' +177,13-177,14: NUMBER '1' +177,14-177,15: OP '*' +177,15-177,16: NUMBER '1' +177,17-177,18: OP '-' +177,19-177,20: OP '-' +177,20-177,21: OP '-' +177,21-177,22: OP '-' +177,22-177,23: NUMBER '1' +177,23-177,24: OP '*' +177,24-177,25: NUMBER '1' +177,25-177,26: NEWLINE '\n' 178,0-178,1: NL '\n' -179,0-179,0: ENDMARKER '' +179,0-179,10: COMMENT '# selector' +179,10-179,11: NL '\n' +180,0-180,6: NAME 'import' +180,7-180,10: NAME 'sys' +180,10-180,11: OP ',' +180,12-180,16: NAME 'time' +180,16-180,17: NEWLINE '\n' +181,0-181,1: NAME 'x' +181,2-181,3: OP '=' +181,4-181,7: NAME 'sys' +181,7-181,8: OP '.' +181,8-181,15: NAME 'modules' +181,15-181,16: OP '[' +181,16-181,22: STRING "'time'" +181,22-181,23: OP ']' +181,23-181,24: OP '.' +181,24-181,28: NAME 'time' +181,28-181,29: OP '(' +181,29-181,30: OP ')' +181,30-181,31: NEWLINE '\n' +182,0-182,1: NL '\n' +183,0-183,1: OP '@' +183,1-183,13: NAME 'staticmethod' +183,13-183,14: NEWLINE '\n' +184,0-184,3: NAME 'def' +184,4-184,7: NAME 'foo' +184,7-184,8: OP '(' +184,8-184,9: OP ')' +184,9-184,10: OP ':' +184,11-184,15: NAME 'pass' +184,15-184,16: NEWLINE '\n' +185,0-185,1: NL '\n' +186,0-186,0: ENDMARKER '' Index: Lib/test/test_compile.py =================================================================== --- Lib/test/test_compile.py (revision 54387) +++ Lib/test/test_compile.py (working copy) @@ -177,7 +177,9 @@ def test_literals_with_leading_zeroes(self): for arg in ["077787", "0xj", "0x.", "0e", "090000000000000", - "080000000000000", "000000000000009", "000000000000008"]: + "080000000000000", "000000000000009", "000000000000008", + "0b42", "0BADCAFE", "0t123456789", "0b1.1", "0t4.2", + "0b101j2", "0t153j2", "0b100e1", "0t777e1"]: self.assertRaises(SyntaxError, eval, arg) self.assertEqual(eval("0777"), 511) @@ -205,6 +207,10 @@ self.assertEqual(eval("000000000000007"), 7) self.assertEqual(eval("000000000000008."), 8.) self.assertEqual(eval("000000000000009."), 9.) + self.assertEqual(eval("0B101010"), 42) + self.assertEqual(eval("-0b000000000010"), -2) + self.assertEqual(eval("0t777"), 511) + self.assertEqual(eval("-0T0000010"), -8) def test_unary_minus(self): # Verify treatment of unary minus on negative numbers SF bug #660455