Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(2950)

Side by Side Diff: Parser/tokenizer.c

Issue 15096: Drop support for the "ur" string prefix
Patch Set: Created 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« Lib/test/test_tokenize.py ('K') | « Misc/NEWS ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 1
2 /* Tokenizer implementation */ 2 /* Tokenizer implementation */
3 3
4 #include "Python.h" 4 #include "Python.h"
5 #include "pgenheaders.h" 5 #include "pgenheaders.h"
6 6
7 #include <ctype.h> 7 #include <ctype.h>
8 #include <assert.h> 8 #include <assert.h>
9 9
10 #include "tokenizer.h" 10 #include "tokenizer.h"
(...skipping 1394 matching lines...) Expand 10 before | Expand all | Expand 10 after
1405 c = tok_nextc(tok); 1405 c = tok_nextc(tok);
1406 1406
1407 /* Check for EOF and errors now */ 1407 /* Check for EOF and errors now */
1408 if (c == EOF) { 1408 if (c == EOF) {
1409 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; 1409 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
1410 } 1410 }
1411 1411
1412 /* Identifier (most frequent token!) */ 1412 /* Identifier (most frequent token!) */
1413 nonascii = 0; 1413 nonascii = 0;
1414 if (is_potential_identifier_start(c)) { 1414 if (is_potential_identifier_start(c)) {
1415 /* Process b"", r"", u"", br"", rb"" and ur"" */ 1415 /* Process b"", r"", u"", br"" and rb"" */
1416 int saw_b = 0, saw_r = 0, saw_u = 0; 1416 int saw_b = 0, saw_r = 0, saw_u = 0;
1417 while (1) { 1417 while (1) {
1418 if (!(saw_b || saw_u) && (c == 'b' || c == 'B')) 1418 if (!(saw_b || saw_u) && (c == 'b' || c == 'B'))
1419 saw_b = 1; 1419 saw_b = 1;
1420 /* Since this is a backwards compatibility support literal we don't 1420 /* Since this is a backwards compatibility support literal we don't
1421 want to support it in arbitrary order like byte literals. */ 1421 want to support it in arbitrary order like byte literals. */
1422 else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U')) 1422 else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U'))
1423 saw_u = 1; 1423 saw_u = 1;
1424 else if (!saw_r && (c == 'r' || c == 'R')) 1424 /* ur"" and ru"" are not supported */
1425 else if (!(saw_r || saw_u) && (c == 'r' || c == 'R'))
1425 saw_r = 1; 1426 saw_r = 1;
1426 else 1427 else
1427 break; 1428 break;
1428 c = tok_nextc(tok); 1429 c = tok_nextc(tok);
1429 if (c == '"' || c == '\'') 1430 if (c == '"' || c == '\'')
1430 goto letter_quote; 1431 goto letter_quote;
1431 } 1432 }
1432 while (is_potential_identifier_char(c)) { 1433 while (is_potential_identifier_char(c)) {
1433 if (c >= 128) 1434 if (c >= 128)
1434 nonascii = 1; 1435 nonascii = 1;
(...skipping 332 matching lines...) Expand 10 before | Expand all | Expand 10 after
1767 1768
1768 void 1769 void
1769 tok_dump(int type, char *start, char *end) 1770 tok_dump(int type, char *start, char *end)
1770 { 1771 {
1771 printf("%s", _PyParser_TokenNames[type]); 1772 printf("%s", _PyParser_TokenNames[type]);
1772 if (type == NAME || type == NUMBER || type == STRING || type == OP) 1773 if (type == NAME || type == NUMBER || type == STRING || type == OP)
1773 printf("(%.*s)", (int)(end - start), start); 1774 printf("(%.*s)", (int)(end - start), start);
1774 } 1775 }
1775 1776
1776 #endif 1777 #endif
OLDNEW
« Lib/test/test_tokenize.py ('K') | « Misc/NEWS ('k') | no next file » | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld cbc36f91f3f7