| OLD | NEW |
| 1 | 1 |
| 2 /* Tokenizer implementation */ | 2 /* Tokenizer implementation */ |
| 3 | 3 |
| 4 #include "Python.h" | 4 #include "Python.h" |
| 5 #include "pgenheaders.h" | 5 #include "pgenheaders.h" |
| 6 | 6 |
| 7 #include <ctype.h> | 7 #include <ctype.h> |
| 8 #include <assert.h> | 8 #include <assert.h> |
| 9 | 9 |
| 10 #include "tokenizer.h" | 10 #include "tokenizer.h" |
| (...skipping 1394 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1405 c = tok_nextc(tok); | 1405 c = tok_nextc(tok); |
| 1406 | 1406 |
| 1407 /* Check for EOF and errors now */ | 1407 /* Check for EOF and errors now */ |
| 1408 if (c == EOF) { | 1408 if (c == EOF) { |
| 1409 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; | 1409 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; |
| 1410 } | 1410 } |
| 1411 | 1411 |
| 1412 /* Identifier (most frequent token!) */ | 1412 /* Identifier (most frequent token!) */ |
| 1413 nonascii = 0; | 1413 nonascii = 0; |
| 1414 if (is_potential_identifier_start(c)) { | 1414 if (is_potential_identifier_start(c)) { |
| 1415 /* Process b"", r"", u"", br"", rb"" and ur"" */ | 1415 /* Process b"", r"", u"", br"" and rb"" */ |
| 1416 int saw_b = 0, saw_r = 0, saw_u = 0; | 1416 int saw_b = 0, saw_r = 0, saw_u = 0; |
| 1417 while (1) { | 1417 while (1) { |
| 1418 if (!(saw_b || saw_u) && (c == 'b' || c == 'B')) | 1418 if (!(saw_b || saw_u) && (c == 'b' || c == 'B')) |
| 1419 saw_b = 1; | 1419 saw_b = 1; |
| 1420 /* Since this is a backwards compatibility support literal we don't | 1420 /* Since this is a backwards compatibility support literal we don't |
| 1421 want to support it in arbitrary order like byte literals. */ | 1421 want to support it in arbitrary order like byte literals. */ |
| 1422 else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U')) | 1422 else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U')) |
| 1423 saw_u = 1; | 1423 saw_u = 1; |
| 1424 else if (!saw_r && (c == 'r' || c == 'R')) | 1424 /* ur"" and ru"" are not supported */ |
| 1425 else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) |
| 1425 saw_r = 1; | 1426 saw_r = 1; |
| 1426 else | 1427 else |
| 1427 break; | 1428 break; |
| 1428 c = tok_nextc(tok); | 1429 c = tok_nextc(tok); |
| 1429 if (c == '"' || c == '\'') | 1430 if (c == '"' || c == '\'') |
| 1430 goto letter_quote; | 1431 goto letter_quote; |
| 1431 } | 1432 } |
| 1432 while (is_potential_identifier_char(c)) { | 1433 while (is_potential_identifier_char(c)) { |
| 1433 if (c >= 128) | 1434 if (c >= 128) |
| 1434 nonascii = 1; | 1435 nonascii = 1; |
| (...skipping 332 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1767 | 1768 |
| 1768 void | 1769 void |
| 1769 tok_dump(int type, char *start, char *end) | 1770 tok_dump(int type, char *start, char *end) |
| 1770 { | 1771 { |
| 1771 printf("%s", _PyParser_TokenNames[type]); | 1772 printf("%s", _PyParser_TokenNames[type]); |
| 1772 if (type == NAME || type == NUMBER || type == STRING || type == OP) | 1773 if (type == NAME || type == NUMBER || type == STRING || type == OP) |
| 1773 printf("(%.*s)", (int)(end - start), start); | 1774 printf("(%.*s)", (int)(end - start), start); |
| 1774 } | 1775 } |
| 1775 | 1776 |
| 1776 #endif | 1777 #endif |
| OLD | NEW |