Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(14)

Delta Between Two Patch Sets: Parser/tokenizer.c

Issue 3353: make built-in tokenizer available via Python C API
Left Patch Set: Created 4 years, 11 months ago
Right Patch Set: Created 4 years, 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Parser/pgen.c ('k') | Parser/tokenizer.h » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 1
2 /* Tokenizer implementation */ 2 /* Tokenizer implementation */
3 3
4 #include "Python.h" 4 #include "Python.h"
5 #include "pgenheaders.h" 5 #include "pgenheaders.h"
6 6
7 #include <ctype.h> 7 #include <ctype.h>
8 #include <assert.h> 8 #include <assert.h>
9 9
10 #include "tokenizer.h" 10 #include "tokenizer.h"
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after
126 tok->indstack[0] = 0; 126 tok->indstack[0] = 0;
127 tok->atbol = 1; 127 tok->atbol = 1;
128 tok->pendin = 0; 128 tok->pendin = 0;
129 tok->prompt = tok->nextprompt = NULL; 129 tok->prompt = tok->nextprompt = NULL;
130 tok->lineno = 0; 130 tok->lineno = 0;
131 tok->level = 0; 131 tok->level = 0;
132 tok->altwarning = 1; 132 tok->altwarning = 1;
133 tok->alterror = 1; 133 tok->alterror = 1;
134 tok->alttabsize = 1; 134 tok->alttabsize = 1;
135 tok->altindstack[0] = 0; 135 tok->altindstack[0] = 0;
136 tok->decoding_state = STATE_INIT; 136 tok->decoding_state = PYTOKENIZER_STATE_INIT;
137 tok->decoding_erred = 0; 137 tok->decoding_erred = 0;
138 tok->read_coding_spec = 0; 138 tok->read_coding_spec = 0;
139 tok->enc = NULL; 139 tok->enc = NULL;
140 tok->encoding = NULL; 140 tok->encoding = NULL;
141 tok->cont_line = 0; 141 tok->cont_line = 0;
142 #ifndef PGEN 142 #ifndef PGEN
143 tok->filename = NULL; 143 tok->filename = NULL;
144 tok->decoding_readline = NULL; 144 tok->decoding_readline = NULL;
145 tok->decoding_buffer = NULL; 145 tok->decoding_buffer = NULL;
146 #endif 146 #endif
(...skipping 153 matching lines...) Expand 10 before | Expand all | Expand 10 after
300 /* Stop checking coding spec after a line containing 300 /* Stop checking coding spec after a line containing
301 * anything except a comment. */ 301 * anything except a comment. */
302 tok->read_coding_spec = 1; 302 tok->read_coding_spec = 1;
303 break; 303 break;
304 } 304 }
305 } 305 }
306 return 1; 306 return 1;
307 } 307 }
308 tok->read_coding_spec = 1; 308 tok->read_coding_spec = 1;
309 if (tok->encoding == NULL) { 309 if (tok->encoding == NULL) {
310 assert(tok->decoding_state == STATE_RAW); 310 assert(tok->decoding_state == PYTOKENIZER_STATE_RAW);
311 if (strcmp(cs, "utf-8") == 0) { 311 if (strcmp(cs, "utf-8") == 0) {
312 tok->encoding = cs; 312 tok->encoding = cs;
313 } else { 313 } else {
314 r = set_readline(tok, cs); 314 r = set_readline(tok, cs);
315 if (r) { 315 if (r) {
316 tok->encoding = cs; 316 tok->encoding = cs;
317 tok->decoding_state = STATE_NORMAL; 317 tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
318 } 318 }
319 else { 319 else {
320 PyErr_Format(PyExc_SyntaxError, 320 PyErr_Format(PyExc_SyntaxError,
321 "encoding problem: %s", cs); 321 "encoding problem: %s", cs);
322 PyMem_FREE(cs); 322 PyMem_FREE(cs);
323 } 323 }
324 } 324 }
325 } else { /* then, compare cs with BOM */ 325 } else { /* then, compare cs with BOM */
326 r = (strcmp(tok->encoding, cs) == 0); 326 r = (strcmp(tok->encoding, cs) == 0);
327 if (!r) 327 if (!r)
328 PyErr_Format(PyExc_SyntaxError, 328 PyErr_Format(PyExc_SyntaxError,
329 "encoding problem: %s with BOM", cs); 329 "encoding problem: %s with BOM", cs);
330 PyMem_FREE(cs); 330 PyMem_FREE(cs);
331 } 331 }
332 return r; 332 return r;
333 } 333 }
334 334
335 /* See whether the file starts with a BOM. If it does, 335 /* See whether the file starts with a BOM. If it does,
336 invoke the set_readline function with the new encoding. 336 invoke the set_readline function with the new encoding.
337 Return 1 on success, 0 on failure. */ 337 Return 1 on success, 0 on failure. */
338 338
339 static int 339 static int
340 check_bom(int get_char(PyTokenizer_State *), 340 check_bom(int get_char(PyTokenizer_State *),
341 void unget_char(int, PyTokenizer_State *), 341 void unget_char(int, PyTokenizer_State *),
342 int set_readline(PyTokenizer_State *, const char *), 342 int set_readline(PyTokenizer_State *, const char *),
343 PyTokenizer_State *tok) 343 PyTokenizer_State *tok)
344 { 344 {
345 int ch1, ch2, ch3; 345 int ch1, ch2, ch3;
346 ch1 = get_char(tok); 346 ch1 = get_char(tok);
347 tok->decoding_state = STATE_RAW; 347 tok->decoding_state = PYTOKENIZER_STATE_RAW;
348 if (ch1 == EOF) { 348 if (ch1 == EOF) {
349 return 1; 349 return 1;
350 } else if (ch1 == 0xEF) { 350 } else if (ch1 == 0xEF) {
351 ch2 = get_char(tok); 351 ch2 = get_char(tok);
352 if (ch2 != 0xBB) { 352 if (ch2 != 0xBB) {
353 unget_char(ch2, tok); 353 unget_char(ch2, tok);
354 unget_char(ch1, tok); 354 unget_char(ch1, tok);
355 return 1; 355 return 1;
356 } 356 }
357 ch3 = get_char(tok); 357 ch3 = get_char(tok);
358 if (ch3 != 0xBF) { 358 if (ch3 != 0xBF) {
359 unget_char(ch3, tok); 359 unget_char(ch3, tok);
360 unget_char(ch2, tok); 360 unget_char(ch2, tok);
361 unget_char(ch1, tok); 361 unget_char(ch1, tok);
362 return 1; 362 return 1;
363 } 363 }
364 #if 0 364 #if 0
365 /* Disable support for UTF-16 BOMs until a decision 365 /* Disable support for UTF-16 BOMs until a decision
366 is made whether this needs to be supported. */ 366 is made whether this needs to be supported. */
367 } else if (ch1 == 0xFE) { 367 } else if (ch1 == 0xFE) {
368 ch2 = get_char(tok); 368 ch2 = get_char(tok);
369 if (ch2 != 0xFF) { 369 if (ch2 != 0xFF) {
370 unget_char(ch2, tok); 370 unget_char(ch2, tok);
371 unget_char(ch1, tok); 371 unget_char(ch1, tok);
372 return 1; 372 return 1;
373 } 373 }
374 if (!set_readline(tok, "utf-16-be")) 374 if (!set_readline(tok, "utf-16-be"))
375 return 0; 375 return 0;
376 tok->decoding_state = STATE_NORMAL; 376 tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
377 } else if (ch1 == 0xFF) { 377 } else if (ch1 == 0xFF) {
378 ch2 = get_char(tok); 378 ch2 = get_char(tok);
379 if (ch2 != 0xFE) { 379 if (ch2 != 0xFE) {
380 unget_char(ch2, tok); 380 unget_char(ch2, tok);
381 unget_char(ch1, tok); 381 unget_char(ch1, tok);
382 return 1; 382 return 1;
383 } 383 }
384 if (!set_readline(tok, "utf-16-le")) 384 if (!set_readline(tok, "utf-16-le"))
385 return 0; 385 return 0;
386 tok->decoding_state = STATE_NORMAL; 386 tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
387 #endif 387 #endif
388 } else { 388 } else {
389 unget_char(ch1, tok); 389 unget_char(ch1, tok);
390 return 1; 390 return 1;
391 } 391 }
392 if (tok->encoding != NULL) 392 if (tok->encoding != NULL)
393 PyMem_FREE(tok->encoding); 393 PyMem_FREE(tok->encoding);
394 tok->encoding = new_string("utf-8", 5, tok); 394 tok->encoding = new_string("utf-8", 5, tok);
395 if (!tok->encoding) 395 if (!tok->encoding)
396 return 0; 396 return 0;
(...skipping 176 matching lines...) Expand 10 before | Expand all | Expand 10 after
573 573
574 /* Read a line of input from TOK. Determine encoding 574 /* Read a line of input from TOK. Determine encoding
575 if necessary. */ 575 if necessary. */
576 576
577 static char * 577 static char *
578 decoding_fgets(char *s, int size, PyTokenizer_State *tok) 578 decoding_fgets(char *s, int size, PyTokenizer_State *tok)
579 { 579 {
580 char *line = NULL; 580 char *line = NULL;
581 int badchar = 0; 581 int badchar = 0;
582 for (;;) { 582 for (;;) {
583 if (tok->decoding_state == STATE_NORMAL) { 583 if (tok->decoding_state == PYTOKENIZER_STATE_NORMAL) {
584 /* We already have a codec associated with 584 /* We already have a codec associated with
585 this input. */ 585 this input. */
586 line = fp_readl(s, size, tok); 586 line = fp_readl(s, size, tok);
587 break; 587 break;
588 } else if (tok->decoding_state == STATE_RAW) { 588 } else if (tok->decoding_state == PYTOKENIZER_STATE_RAW) {
589 /* We want a 'raw' read. */ 589 /* We want a 'raw' read. */
590 line = Py_UniversalNewlineFgets(s, size, 590 line = Py_UniversalNewlineFgets(s, size,
591 tok->fp, NULL); 591 tok->fp, NULL);
592 break; 592 break;
593 } else { 593 } else {
594 /* We have not yet determined the encoding. 594 /* We have not yet determined the encoding.
595 If an encoding is found, use the file-pointer 595 If an encoding is found, use the file-pointer
596 reader functions from now on. */ 596 reader functions from now on. */
597 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) 597 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
598 return error_ret(tok); 598 return error_ret(tok);
599 assert(tok->decoding_state != STATE_INIT); 599 assert(tok->decoding_state != PYTOKENIZER_STATE_INIT);
600 } 600 }
601 } 601 }
602 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) { 602 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
603 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) { 603 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
604 return error_ret(tok); 604 return error_ret(tok);
605 } 605 }
606 } 606 }
607 #ifndef PGEN 607 #ifndef PGEN
608 /* The default encoding is UTF-8, so make sure we don't have any 608 /* The default encoding is UTF-8, so make sure we don't have any
609 non-UTF-8 sequences in it. */ 609 non-UTF-8 sequences in it. */
(...skipping 17 matching lines...) Expand all
627 badchar, tok->filename, tok->lineno + 1); 627 badchar, tok->filename, tok->lineno + 1);
628 return error_ret(tok); 628 return error_ret(tok);
629 } 629 }
630 #endif 630 #endif
631 return line; 631 return line;
632 } 632 }
633 633
634 static int 634 static int
635 decoding_feof(PyTokenizer_State *tok) 635 decoding_feof(PyTokenizer_State *tok)
636 { 636 {
637 if (tok->decoding_state != STATE_NORMAL) { 637 if (tok->decoding_state != PYTOKENIZER_STATE_NORMAL) {
638 return feof(tok->fp); 638 return feof(tok->fp);
639 } else { 639 } else {
640 PyObject* buf = tok->decoding_buffer; 640 PyObject* buf = tok->decoding_buffer;
641 if (buf == NULL) { 641 if (buf == NULL) {
642 buf = PyObject_CallObject(tok->decoding_readline, NULL); 642 buf = PyObject_CallObject(tok->decoding_readline, NULL);
643 if (buf == NULL) { 643 if (buf == NULL) {
644 error_ret(tok); 644 error_ret(tok);
645 return 1; 645 return 1;
646 } else { 646 } else {
647 tok->decoding_buffer = buf; 647 tok->decoding_buffer = buf;
(...skipping 170 matching lines...) Expand 10 before | Expand all | Expand 10 after
818 PyTokenizer_State *tok = tok_new(); 818 PyTokenizer_State *tok = tok_new();
819 if (tok == NULL) 819 if (tok == NULL)
820 return NULL; 820 return NULL;
821 #ifndef PGEN 821 #ifndef PGEN
822 tok->input = str = translate_newlines(str, exec_input, tok); 822 tok->input = str = translate_newlines(str, exec_input, tok);
823 #endif 823 #endif
824 if (str == NULL) { 824 if (str == NULL) {
825 PyTokenizer_Free(tok); 825 PyTokenizer_Free(tok);
826 return NULL; 826 return NULL;
827 } 827 }
828 tok->decoding_state = STATE_RAW; 828 tok->decoding_state = PYTOKENIZER_STATE_RAW;
829 tok->read_coding_spec = 1; 829 tok->read_coding_spec = 1;
830 tok->enc = NULL; 830 tok->enc = NULL;
831 tok->str = str; 831 tok->str = str;
832 tok->encoding = (char *)PyMem_MALLOC(6); 832 tok->encoding = (char *)PyMem_MALLOC(6);
833 if (!tok->encoding) { 833 if (!tok->encoding) {
834 PyTokenizer_Free(tok); 834 PyTokenizer_Free(tok);
835 return NULL; 835 return NULL;
836 } 836 }
837 strcpy(tok->encoding, "utf-8"); 837 strcpy(tok->encoding, "utf-8");
838 838
(...skipping 22 matching lines...) Expand all
861 tok->nextprompt = ps2; 861 tok->nextprompt = ps2;
862 if (enc != NULL) { 862 if (enc != NULL) {
863 /* Must copy encoding declaration since it 863 /* Must copy encoding declaration since it
864 gets copied into the parse tree. */ 864 gets copied into the parse tree. */
865 tok->encoding = PyMem_MALLOC(strlen(enc)+1); 865 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
866 if (!tok->encoding) { 866 if (!tok->encoding) {
867 PyTokenizer_Free(tok); 867 PyTokenizer_Free(tok);
868 return NULL; 868 return NULL;
869 } 869 }
870 strcpy(tok->encoding, enc); 870 strcpy(tok->encoding, enc);
871 tok->decoding_state = STATE_NORMAL; 871 tok->decoding_state = PYTOKENIZER_STATE_NORMAL;
872 } 872 }
873 return tok; 873 return tok;
874 } 874 }
875 875
876 876
877 /* Free a tok_state structure */ 877 /* Free a tok_state structure */
878 878
879 void 879 void
880 PyTokenizer_Free(PyTokenizer_State *tok) 880 PyTokenizer_Free(PyTokenizer_State *tok)
881 { 881 {
(...skipping 221 matching lines...) Expand 10 before | Expand all | Expand 10 after
1103 } 1103 }
1104 } 1104 }
1105 1105
1106 1106
1107 /* Return the token corresponding to a single character */ 1107 /* Return the token corresponding to a single character */
1108 1108
1109 int 1109 int
1110 PyToken_OneChar(int c) 1110 PyToken_OneChar(int c)
1111 { 1111 {
1112 switch (c) { 1112 switch (c) {
1113 case '(': return LPAR; 1113 case '(': return PYTOK_LPAR;
1114 case ')': return RPAR; 1114 case ')': return PYTOK_RPAR;
1115 case '[': return LSQB; 1115 case '[': return PYTOK_LSQB;
1116 case ']': return RSQB; 1116 case ']': return PYTOK_RSQB;
1117 case ':': return COLON; 1117 case ':': return PYTOK_COLON;
1118 case ',': return COMMA; 1118 case ',': return PYTOK_COMMA;
1119 case ';': return SEMI; 1119 case ';': return PYTOK_SEMI;
1120 case '+': return PLUS; 1120 case '+': return PYTOK_PLUS;
1121 case '-': return MINUS; 1121 case '-': return PYTOK_MINUS;
1122 case '*': return STAR; 1122 case '*': return PYTOK_STAR;
1123 case '/': return SLASH; 1123 case '/': return PYTOK_SLASH;
1124 case '|': return VBAR; 1124 case '|': return PYTOK_VBAR;
1125 case '&': return AMPER; 1125 case '&': return PYTOK_AMPER;
1126 case '<': return LESS; 1126 case '<': return PYTOK_LESS;
1127 case '>': return GREATER; 1127 case '>': return PYTOK_GREATER;
1128 case '=': return EQUAL; 1128 case '=': return PYTOK_EQUAL;
1129 case '.': return DOT; 1129 case '.': return PYTOK_DOT;
1130 case '%': return PERCENT; 1130 case '%': return PYTOK_PERCENT;
1131 case '{': return LBRACE; 1131 case '{': return PYTOK_LBRACE;
1132 case '}': return RBRACE; 1132 case '}': return PYTOK_RBRACE;
1133 case '^': return CIRCUMFLEX; 1133 case '^': return PYTOK_CIRCUMFLEX;
1134 case '~': return TILDE; 1134 case '~': return PYTOK_TILDE;
1135 case '@': return AT; 1135 case '@': return PYTOK_AT;
1136 default: return OP; 1136 default: return PYTOK_OP;
1137 } 1137 }
1138 } 1138 }
1139 1139
1140 1140
1141 int 1141 int
1142 PyToken_TwoChars(int c1, int c2) 1142 PyToken_TwoChars(int c1, int c2)
1143 { 1143 {
1144 switch (c1) { 1144 switch (c1) {
1145 case '=': 1145 case '=':
1146 switch (c2) { 1146 switch (c2) {
1147 case '=': return EQEQUAL; 1147 case '=': return PYTOK_EQEQUAL;
1148 } 1148 }
1149 break; 1149 break;
1150 case '!': 1150 case '!':
1151 switch (c2) { 1151 switch (c2) {
1152 case '=': return NOTEQUAL; 1152 case '=': return PYTOK_NOTEQUAL;
1153 } 1153 }
1154 break; 1154 break;
1155 case '<': 1155 case '<':
1156 switch (c2) { 1156 switch (c2) {
1157 case '>': return NOTEQUAL; 1157 case '>': return PYTOK_NOTEQUAL;
1158 case '=': return LESSEQUAL; 1158 case '=': return PYTOK_LESSEQUAL;
1159 case '<': return LEFTSHIFT; 1159 case '<': return PYTOK_LEFTSHIFT;
1160 } 1160 }
1161 break; 1161 break;
1162 case '>': 1162 case '>':
1163 switch (c2) { 1163 switch (c2) {
1164 case '=': return GREATEREQUAL; 1164 case '=': return PYTOK_GREATEREQUAL;
1165 case '>': return RIGHTSHIFT; 1165 case '>': return PYTOK_RIGHTSHIFT;
1166 } 1166 }
1167 break; 1167 break;
1168 case '+': 1168 case '+':
1169 switch (c2) { 1169 switch (c2) {
1170 case '=': return PLUSEQUAL; 1170 case '=': return PYTOK_PLUSEQUAL;
1171 } 1171 }
1172 break; 1172 break;
1173 case '-': 1173 case '-':
1174 switch (c2) { 1174 switch (c2) {
1175 case '=': return MINEQUAL; 1175 case '=': return PYTOK_MINEQUAL;
1176 case '>': return RARROW; 1176 case '>': return PYTOK_RARROW;
1177 } 1177 }
1178 break; 1178 break;
1179 case '*': 1179 case '*':
1180 switch (c2) { 1180 switch (c2) {
1181 case '*': return DOUBLESTAR; 1181 case '*': return PYTOK_DOUBLESTAR;
1182 case '=': return STAREQUAL; 1182 case '=': return PYTOK_STAREQUAL;
1183 } 1183 }
1184 break; 1184 break;
1185 case '/': 1185 case '/':
1186 switch (c2) { 1186 switch (c2) {
1187 case '/': return DOUBLESLASH; 1187 case '/': return PYTOK_DOUBLESLASH;
1188 case '=': return SLASHEQUAL; 1188 case '=': return PYTOK_SLASHEQUAL;
1189 } 1189 }
1190 break; 1190 break;
1191 case '|': 1191 case '|':
1192 switch (c2) { 1192 switch (c2) {
1193 case '=': return VBAREQUAL; 1193 case '=': return PYTOK_VBAREQUAL;
1194 } 1194 }
1195 break; 1195 break;
1196 case '%': 1196 case '%':
1197 switch (c2) { 1197 switch (c2) {
1198 case '=': return PERCENTEQUAL; 1198 case '=': return PYTOK_PERCENTEQUAL;
1199 } 1199 }
1200 break; 1200 break;
1201 case '&': 1201 case '&':
1202 switch (c2) { 1202 switch (c2) {
1203 case '=': return AMPEREQUAL; 1203 case '=': return PYTOK_AMPEREQUAL;
1204 } 1204 }
1205 break; 1205 break;
1206 case '^': 1206 case '^':
1207 switch (c2) { 1207 switch (c2) {
1208 case '=': return CIRCUMFLEXEQUAL; 1208 case '=': return PYTOK_CIRCUMFLEXEQUAL;
1209 } 1209 }
1210 break; 1210 break;
1211 case '@': 1211 case '@':
1212 switch (c2) { 1212 switch (c2) {
1213 case '=': return ATEQUAL; 1213 case '=': return PYTOK_ATEQUAL;
1214 } 1214 }
1215 break; 1215 break;
1216 } 1216 }
1217 return OP; 1217 return PYTOK_OP;
1218 } 1218 }
1219 1219
1220 int 1220 int
1221 PyToken_ThreeChars(int c1, int c2, int c3) 1221 PyToken_ThreeChars(int c1, int c2, int c3)
1222 { 1222 {
1223 switch (c1) { 1223 switch (c1) {
1224 case '<': 1224 case '<':
1225 switch (c2) { 1225 switch (c2) {
1226 case '<': 1226 case '<':
1227 switch (c3) { 1227 switch (c3) {
1228 case '=': 1228 case '=':
1229 return LEFTSHIFTEQUAL; 1229 return PYTOK_LEFTSHIFTEQUAL;
1230 } 1230 }
1231 break; 1231 break;
1232 } 1232 }
1233 break; 1233 break;
1234 case '>': 1234 case '>':
1235 switch (c2) { 1235 switch (c2) {
1236 case '>': 1236 case '>':
1237 switch (c3) { 1237 switch (c3) {
1238 case '=': 1238 case '=':
1239 return RIGHTSHIFTEQUAL; 1239 return PYTOK_RIGHTSHIFTEQUAL;
1240 } 1240 }
1241 break; 1241 break;
1242 } 1242 }
1243 break; 1243 break;
1244 case '*': 1244 case '*':
1245 switch (c2) { 1245 switch (c2) {
1246 case '*': 1246 case '*':
1247 switch (c3) { 1247 switch (c3) {
1248 case '=': 1248 case '=':
1249 return DOUBLESTAREQUAL; 1249 return PYTOK_DOUBLESTAREQUAL;
1250 } 1250 }
1251 break; 1251 break;
1252 } 1252 }
1253 break; 1253 break;
1254 case '/': 1254 case '/':
1255 switch (c2) { 1255 switch (c2) {
1256 case '/': 1256 case '/':
1257 switch (c3) { 1257 switch (c3) {
1258 case '=': 1258 case '=':
1259 return DOUBLESLASHEQUAL; 1259 return PYTOK_DOUBLESLASHEQUAL;
1260 } 1260 }
1261 break; 1261 break;
1262 } 1262 }
1263 break; 1263 break;
1264 case '.': 1264 case '.':
1265 switch (c2) { 1265 switch (c2) {
1266 case '.': 1266 case '.':
1267 switch (c3) { 1267 switch (c3) {
1268 case '.': 1268 case '.':
1269 return ELLIPSIS; 1269 return PYTOK_ELLIPSIS;
1270 } 1270 }
1271 break; 1271 break;
1272 } 1272 }
1273 break; 1273 break;
1274 } 1274 }
1275 return OP; 1275 return PYTOK_OP;
1276 } 1276 }
1277 1277
1278 static int 1278 static int
1279 indenterror(PyTokenizer_State *tok) 1279 indenterror(PyTokenizer_State *tok)
1280 { 1280 {
1281 if (tok->alterror) { 1281 if (tok->alterror) {
1282 tok->done = E_TABSPACE; 1282 tok->done = E_TABSPACE;
1283 tok->cur = tok->inp; 1283 tok->cur = tok->inp;
1284 return 1; 1284 return 1;
1285 } 1285 }
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after
1354 } 1354 }
1355 else if (c == '\014') /* Control-L (formfeed) */ 1355 else if (c == '\014') /* Control-L (formfeed) */
1356 col = altcol = 0; /* For Emacs users */ 1356 col = altcol = 0; /* For Emacs users */
1357 else 1357 else
1358 break; 1358 break;
1359 } 1359 }
1360 tok_backup(tok, c); 1360 tok_backup(tok, c);
1361 if (c == '#' || c == '\n') { 1361 if (c == '#' || c == '\n') {
1362 /* Lines with only whitespace and/or comments 1362 /* Lines with only whitespace and/or comments
1363 shouldn't affect the indentation and are 1363 shouldn't affect the indentation and are
1364 not passed to the parser as NEWLINE tokens, 1364 not passed to the parser as PYTOK_NEWLINE tokens,
1365 except *totally* empty lines in interactive 1365 except *totally* empty lines in interactive
1366 mode, which signal the end of a command group. */ 1366 mode, which signal the end of a command group. */
1367 if (col == 0 && c == '\n' && tok->prompt != NULL) 1367 if (col == 0 && c == '\n' && tok->prompt != NULL)
1368 blankline = 0; /* Let it through */ 1368 blankline = 0; /* Let it through */
1369 else 1369 else
1370 blankline = 1; /* Ignore completely */ 1370 blankline = 1; /* Ignore completely */
1371 /* We can't jump back right here since we still 1371 /* We can't jump back right here since we still
1372 may need to skip to the end of a comment */ 1372 may need to skip to the end of a comment */
1373 } 1373 }
1374 if (!blankline && tok->level == 0) { 1374 if (!blankline && tok->level == 0) {
1375 if (col == tok->indstack[tok->indent]) { 1375 if (col == tok->indstack[tok->indent]) {
1376 /* No change */ 1376 /* No change */
1377 if (altcol != tok->altindstack[tok->indent]) { 1377 if (altcol != tok->altindstack[tok->indent]) {
1378 if (indenterror(tok)) 1378 if (indenterror(tok))
1379 return ERRORTOKEN; 1379 return PYTOK_ERRORTOKEN;
1380 } 1380 }
1381 } 1381 }
1382 else if (col > tok->indstack[tok->indent]) { 1382 else if (col > tok->indstack[tok->indent]) {
1383 /* Indent -- always one */ 1383 /* Indent -- always one */
1384 if (tok->indent+1 >= MAXINDENT) { 1384 if (tok->indent+1 >= PYTOKENIZER_MAXINDENT) {
1385 tok->done = E_TOODEEP; 1385 tok->done = E_TOODEEP;
1386 tok->cur = tok->inp; 1386 tok->cur = tok->inp;
1387 return ERRORTOKEN; 1387 return PYTOK_ERRORTOKEN;
1388 } 1388 }
1389 if (altcol <= tok->altindstack[tok->indent]) { 1389 if (altcol <= tok->altindstack[tok->indent]) {
1390 if (indenterror(tok)) 1390 if (indenterror(tok))
1391 return ERRORTOKEN; 1391 return PYTOK_ERRORTOKEN;
1392 } 1392 }
1393 tok->pendin++; 1393 tok->pendin++;
1394 tok->indstack[++tok->indent] = col; 1394 tok->indstack[++tok->indent] = col;
1395 tok->altindstack[tok->indent] = altcol; 1395 tok->altindstack[tok->indent] = altcol;
1396 } 1396 }
1397 else /* col < tok->indstack[tok->indent] */ { 1397 else /* col < tok->indstack[tok->indent] */ {
1398 /* Dedent -- any number, must be consistent */ 1398 /* Dedent -- any number, must be consistent */
1399 while (tok->indent > 0 && 1399 while (tok->indent > 0 &&
1400 col < tok->indstack[tok->indent]) { 1400 col < tok->indstack[tok->indent]) {
1401 tok->pendin--; 1401 tok->pendin--;
1402 tok->indent--; 1402 tok->indent--;
1403 } 1403 }
1404 if (col != tok->indstack[tok->indent]) { 1404 if (col != tok->indstack[tok->indent]) {
1405 tok->done = E_DEDENT; 1405 tok->done = E_DEDENT;
1406 tok->cur = tok->inp; 1406 tok->cur = tok->inp;
1407 return ERRORTOKEN; 1407 return PYTOK_ERRORTOKEN;
1408 } 1408 }
1409 if (altcol != tok->altindstack[tok->indent]) { 1409 if (altcol != tok->altindstack[tok->indent]) {
1410 if (indenterror(tok)) 1410 if (indenterror(tok))
1411 return ERRORTOKEN; 1411 return PYTOK_ERRORTOKEN;
1412 } 1412 }
1413 } 1413 }
1414 } 1414 }
1415 } 1415 }
1416 1416
1417 tok->start = tok->cur; 1417 tok->start = tok->cur;
1418 1418
1419 /* Return pending indents/dedents */ 1419 /* Return pending indents/dedents */
1420 if (tok->pendin != 0) { 1420 if (tok->pendin != 0) {
1421 if (tok->pendin < 0) { 1421 if (tok->pendin < 0) {
1422 tok->pendin++; 1422 tok->pendin++;
1423 return DEDENT; 1423 return PYTOK_DEDENT;
1424 } 1424 }
1425 else { 1425 else {
1426 tok->pendin--; 1426 tok->pendin--;
1427 return INDENT; 1427 return PYTOK_INDENT;
1428 } 1428 }
1429 } 1429 }
1430 1430
1431 again: 1431 again:
1432 tok->start = NULL; 1432 tok->start = NULL;
1433 /* Skip spaces */ 1433 /* Skip spaces */
1434 do { 1434 do {
1435 c = tok_nextc(tok); 1435 c = tok_nextc(tok);
1436 } while (c == ' ' || c == '\t' || c == '\014'); 1436 } while (c == ' ' || c == '\t' || c == '\014');
1437 1437
1438 /* Set start of current token */ 1438 /* Set start of current token */
1439 tok->start = tok->cur - 1; 1439 tok->start = tok->cur - 1;
1440 1440
1441 /* Skip comment */ 1441 /* Skip comment */
1442 if (c == '#') 1442 if (c == '#')
1443 while (c != EOF && c != '\n') 1443 while (c != EOF && c != '\n')
1444 c = tok_nextc(tok); 1444 c = tok_nextc(tok);
1445 1445
1446 /* Check for EOF and errors now */ 1446 /* Check for EOF and errors now */
1447 if (c == EOF) { 1447 if (c == EOF) {
1448 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; 1448 return tok->done == E_EOF ? PYTOK_ENDMARKER : PYTOK_ERRORTOKEN;
1449 } 1449 }
1450 1450
1451 /* Identifier (most frequent token!) */ 1451 /* Identifier (most frequent token!) */
1452 nonascii = 0; 1452 nonascii = 0;
1453 if (is_potential_identifier_start(c)) { 1453 if (is_potential_identifier_start(c)) {
1454 /* Process b"", r"", u"", br"" and rb"" */ 1454 /* Process b"", r"", u"", br"" and rb"" */
1455 int saw_b = 0, saw_r = 0, saw_u = 0; 1455 int saw_b = 0, saw_r = 0, saw_u = 0;
1456 while (1) { 1456 while (1) {
1457 if (!(saw_b || saw_u) && (c == 'b' || c == 'B')) 1457 if (!(saw_b || saw_u) && (c == 'b' || c == 'B'))
1458 saw_b = 1; 1458 saw_b = 1;
(...skipping 12 matching lines...) Expand all
1471 } 1471 }
1472 while (is_potential_identifier_char(c)) { 1472 while (is_potential_identifier_char(c)) {
1473 if (c >= 128) 1473 if (c >= 128)
1474 nonascii = 1; 1474 nonascii = 1;
1475 c = tok_nextc(tok); 1475 c = tok_nextc(tok);
1476 } 1476 }
1477 tok_backup(tok, c); 1477 tok_backup(tok, c);
1478 if (nonascii && 1478 if (nonascii &&
1479 !verify_identifier(tok)) { 1479 !verify_identifier(tok)) {
1480 tok->done = E_IDENTIFIER; 1480 tok->done = E_IDENTIFIER;
1481 return ERRORTOKEN; 1481 return PYTOK_ERRORTOKEN;
1482 } 1482 }
1483 *p_start = tok->start; 1483 *p_start = tok->start;
1484 *p_end = tok->cur; 1484 *p_end = tok->cur;
1485 return NAME; 1485 return PYTOK_NAME;
1486 } 1486 }
1487 1487
1488 /* Newline */ 1488 /* Newline */
1489 if (c == '\n') { 1489 if (c == '\n') {
1490 tok->atbol = 1; 1490 tok->atbol = 1;
1491 if (blankline || tok->level > 0) 1491 if (blankline || tok->level > 0)
1492 goto nextline; 1492 goto nextline;
1493 *p_start = tok->start; 1493 *p_start = tok->start;
1494 *p_end = tok->cur - 1; /* Leave '\n' out of the string */ 1494 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
1495 tok->cont_line = 0; 1495 tok->cont_line = 0;
1496 return NEWLINE; 1496 return PYTOK_NEWLINE;
1497 } 1497 }
1498 1498
1499 /* Period or number starting with period? */ 1499 /* Period or number starting with period? */
1500 if (c == '.') { 1500 if (c == '.') {
1501 c = tok_nextc(tok); 1501 c = tok_nextc(tok);
1502 if (isdigit(c)) { 1502 if (isdigit(c)) {
1503 goto fraction; 1503 goto fraction;
1504 } else if (c == '.') { 1504 } else if (c == '.') {
1505 c = tok_nextc(tok); 1505 c = tok_nextc(tok);
1506 if (c == '.') { 1506 if (c == '.') {
1507 *p_start = tok->start; 1507 *p_start = tok->start;
1508 *p_end = tok->cur; 1508 *p_end = tok->cur;
1509 return ELLIPSIS; 1509 return PYTOK_ELLIPSIS;
1510 } else { 1510 } else {
1511 tok_backup(tok, c); 1511 tok_backup(tok, c);
1512 } 1512 }
1513 tok_backup(tok, '.'); 1513 tok_backup(tok, '.');
1514 } else { 1514 } else {
1515 tok_backup(tok, c); 1515 tok_backup(tok, c);
1516 } 1516 }
1517 *p_start = tok->start; 1517 *p_start = tok->start;
1518 *p_end = tok->cur; 1518 *p_end = tok->cur;
1519 return DOT; 1519 return PYTOK_DOT;
1520 } 1520 }
1521 1521
1522 /* Number */ 1522 /* Number */
1523 if (isdigit(c)) { 1523 if (isdigit(c)) {
1524 if (c == '0') { 1524 if (c == '0') {
1525 /* Hex, octal or binary -- maybe. */ 1525 /* Hex, octal or binary -- maybe. */
1526 c = tok_nextc(tok); 1526 c = tok_nextc(tok);
1527 if (c == '.') 1527 if (c == '.')
1528 goto fraction; 1528 goto fraction;
1529 if (c == 'j' || c == 'J') 1529 if (c == 'j' || c == 'J')
1530 goto imaginary; 1530 goto imaginary;
1531 if (c == 'x' || c == 'X') { 1531 if (c == 'x' || c == 'X') {
1532 1532
1533 /* Hex */ 1533 /* Hex */
1534 c = tok_nextc(tok); 1534 c = tok_nextc(tok);
1535 if (!isxdigit(c)) { 1535 if (!isxdigit(c)) {
1536 tok->done = E_TOKEN; 1536 tok->done = E_TOKEN;
1537 tok_backup(tok, c); 1537 tok_backup(tok, c);
1538 return ERRORTOKEN; 1538 return PYTOK_ERRORTOKEN;
1539 } 1539 }
1540 do { 1540 do {
1541 c = tok_nextc(tok); 1541 c = tok_nextc(tok);
1542 } while (isxdigit(c)); 1542 } while (isxdigit(c));
1543 } 1543 }
1544 else if (c == 'o' || c == 'O') { 1544 else if (c == 'o' || c == 'O') {
1545 /* Octal */ 1545 /* Octal */
1546 c = tok_nextc(tok); 1546 c = tok_nextc(tok);
1547 if (c < '0' || c >= '8') { 1547 if (c < '0' || c >= '8') {
1548 tok->done = E_TOKEN; 1548 tok->done = E_TOKEN;
1549 tok_backup(tok, c); 1549 tok_backup(tok, c);
1550 return ERRORTOKEN; 1550 return PYTOK_ERRORTOKEN;
1551 } 1551 }
1552 do { 1552 do {
1553 c = tok_nextc(tok); 1553 c = tok_nextc(tok);
1554 } while ('0' <= c && c < '8'); 1554 } while ('0' <= c && c < '8');
1555 } 1555 }
1556 else if (c == 'b' || c == 'B') { 1556 else if (c == 'b' || c == 'B') {
1557 /* Binary */ 1557 /* Binary */
1558 c = tok_nextc(tok); 1558 c = tok_nextc(tok);
1559 if (c != '0' && c != '1') { 1559 if (c != '0' && c != '1') {
1560 tok->done = E_TOKEN; 1560 tok->done = E_TOKEN;
1561 tok_backup(tok, c); 1561 tok_backup(tok, c);
1562 return ERRORTOKEN; 1562 return PYTOK_ERRORTOKEN;
1563 } 1563 }
1564 do { 1564 do {
1565 c = tok_nextc(tok); 1565 c = tok_nextc(tok);
1566 } while (c == '0' || c == '1'); 1566 } while (c == '0' || c == '1');
1567 } 1567 }
1568 else { 1568 else {
1569 int nonzero = 0; 1569 int nonzero = 0;
1570 /* maybe old-style octal; c is first char of it */ 1570 /* maybe old-style octal; c is first char of it */
1571 /* in any case, allow '0' as a literal */ 1571 /* in any case, allow '0' as a literal */
1572 while (c == '0') 1572 while (c == '0')
1573 c = tok_nextc(tok); 1573 c = tok_nextc(tok);
1574 while (isdigit(c)) { 1574 while (isdigit(c)) {
1575 nonzero = 1; 1575 nonzero = 1;
1576 c = tok_nextc(tok); 1576 c = tok_nextc(tok);
1577 } 1577 }
1578 if (c == '.') 1578 if (c == '.')
1579 goto fraction; 1579 goto fraction;
1580 else if (c == 'e' || c == 'E') 1580 else if (c == 'e' || c == 'E')
1581 goto exponent; 1581 goto exponent;
1582 else if (c == 'j' || c == 'J') 1582 else if (c == 'j' || c == 'J')
1583 goto imaginary; 1583 goto imaginary;
1584 else if (nonzero) { 1584 else if (nonzero) {
1585 tok->done = E_TOKEN; 1585 tok->done = E_TOKEN;
1586 tok_backup(tok, c); 1586 tok_backup(tok, c);
1587 return ERRORTOKEN; 1587 return PYTOK_ERRORTOKEN;
1588 } 1588 }
1589 } 1589 }
1590 } 1590 }
1591 else { 1591 else {
1592 /* Decimal */ 1592 /* Decimal */
1593 do { 1593 do {
1594 c = tok_nextc(tok); 1594 c = tok_nextc(tok);
1595 } while (isdigit(c)); 1595 } while (isdigit(c));
1596 { 1596 {
1597 /* Accept floating point numbers. */ 1597 /* Accept floating point numbers. */
1598 if (c == '.') { 1598 if (c == '.') {
1599 fraction: 1599 fraction:
1600 /* Fraction */ 1600 /* Fraction */
1601 do { 1601 do {
1602 c = tok_nextc(tok); 1602 c = tok_nextc(tok);
1603 } while (isdigit(c)); 1603 } while (isdigit(c));
1604 } 1604 }
1605 if (c == 'e' || c == 'E') { 1605 if (c == 'e' || c == 'E') {
1606 int e; 1606 int e;
1607 exponent: 1607 exponent:
1608 e = c; 1608 e = c;
1609 /* Exponent part */ 1609 /* Exponent part */
1610 c = tok_nextc(tok); 1610 c = tok_nextc(tok);
1611 if (c == '+' || c == '-') { 1611 if (c == '+' || c == '-') {
1612 c = tok_nextc(tok); 1612 c = tok_nextc(tok);
1613 if (!isdigit(c)) { 1613 if (!isdigit(c)) {
1614 tok->done = E_TOKEN; 1614 tok->done = E_TOKEN;
1615 tok_backup(tok, c); 1615 tok_backup(tok, c);
1616 return ERRORTOKEN; 1616 return PYTOK_ERRORTOKEN;
1617 } 1617 }
1618 } else if (!isdigit(c)) { 1618 } else if (!isdigit(c)) {
1619 tok_backup(tok, c); 1619 tok_backup(tok, c);
1620 tok_backup(tok, e); 1620 tok_backup(tok, e);
1621 *p_start = tok->start; 1621 *p_start = tok->start;
1622 *p_end = tok->cur; 1622 *p_end = tok->cur;
1623 return NUMBER; 1623 return PYTOK_NUMBER;
1624 } 1624 }
1625 do { 1625 do {
1626 c = tok_nextc(tok); 1626 c = tok_nextc(tok);
1627 } while (isdigit(c)); 1627 } while (isdigit(c));
1628 } 1628 }
1629 if (c == 'j' || c == 'J') 1629 if (c == 'j' || c == 'J')
1630 /* Imaginary part */ 1630 /* Imaginary part */
1631 imaginary: 1631 imaginary:
1632 c = tok_nextc(tok); 1632 c = tok_nextc(tok);
1633 } 1633 }
1634 } 1634 }
1635 tok_backup(tok, c); 1635 tok_backup(tok, c);
1636 *p_start = tok->start; 1636 *p_start = tok->start;
1637 *p_end = tok->cur; 1637 *p_end = tok->cur;
1638 return NUMBER; 1638 return PYTOK_NUMBER;
1639 } 1639 }
1640 1640
1641 letter_quote: 1641 letter_quote:
1642 /* String */ 1642 /* String */
1643 if (c == '\'' || c == '"') { 1643 if (c == '\'' || c == '"') {
1644 int quote = c; 1644 int quote = c;
1645 int quote_size = 1; /* 1 or 3 */ 1645 int quote_size = 1; /* 1 or 3 */
1646 int end_quote_size = 0; 1646 int end_quote_size = 0;
1647 1647
1648 /* Find the quote size and start of string */ 1648 /* Find the quote size and start of string */
(...skipping 10 matching lines...) Expand all
1659 1659
1660 /* Get rest of string */ 1660 /* Get rest of string */
1661 while (end_quote_size != quote_size) { 1661 while (end_quote_size != quote_size) {
1662 c = tok_nextc(tok); 1662 c = tok_nextc(tok);
1663 if (c == EOF) { 1663 if (c == EOF) {
1664 if (quote_size == 3) 1664 if (quote_size == 3)
1665 tok->done = E_EOFS; 1665 tok->done = E_EOFS;
1666 else 1666 else
1667 tok->done = E_EOLS; 1667 tok->done = E_EOLS;
1668 tok->cur = tok->inp; 1668 tok->cur = tok->inp;
1669 return ERRORTOKEN; 1669 return PYTOK_ERRORTOKEN;
1670 } 1670 }
1671 if (quote_size == 1 && c == '\n') { 1671 if (quote_size == 1 && c == '\n') {
1672 tok->done = E_EOLS; 1672 tok->done = E_EOLS;
1673 tok->cur = tok->inp; 1673 tok->cur = tok->inp;
1674 return ERRORTOKEN; 1674 return PYTOK_ERRORTOKEN;
1675 } 1675 }
1676 if (c == quote) 1676 if (c == quote)
1677 end_quote_size += 1; 1677 end_quote_size += 1;
1678 else { 1678 else {
1679 end_quote_size = 0; 1679 end_quote_size = 0;
1680 if (c == '\\') 1680 if (c == '\\')
1681 c = tok_nextc(tok); /* skip escaped char */ 1681 c = tok_nextc(tok); /* skip escaped char */
1682 } 1682 }
1683 } 1683 }
1684 1684
1685 *p_start = tok->start; 1685 *p_start = tok->start;
1686 *p_end = tok->cur; 1686 *p_end = tok->cur;
1687 return STRING; 1687 return PYTOK_STRING;
1688 } 1688 }
1689 1689
1690 /* Line continuation */ 1690 /* Line continuation */
1691 if (c == '\\') { 1691 if (c == '\\') {
1692 c = tok_nextc(tok); 1692 c = tok_nextc(tok);
1693 if (c != '\n') { 1693 if (c != '\n') {
1694 tok->done = E_LINECONT; 1694 tok->done = E_LINECONT;
1695 tok->cur = tok->inp; 1695 tok->cur = tok->inp;
1696 return ERRORTOKEN; 1696 return PYTOK_ERRORTOKEN;
1697 } 1697 }
1698 tok->cont_line = 1; 1698 tok->cont_line = 1;
1699 goto again; /* Read next line */ 1699 goto again; /* Read next line */
1700 } 1700 }
1701 1701
1702 /* Check for two-character token */ 1702 /* Check for two-character token */
1703 { 1703 {
1704 int c2 = tok_nextc(tok); 1704 int c2 = tok_nextc(tok);
1705 int token = PyToken_TwoChars(c, c2); 1705 int token = PyToken_TwoChars(c, c2);
1706 if (token != OP) { 1706 if (token != PYTOK_OP) {
1707 int c3 = tok_nextc(tok); 1707 int c3 = tok_nextc(tok);
1708 int token3 = PyToken_ThreeChars(c, c2, c3); 1708 int token3 = PyToken_ThreeChars(c, c2, c3);
1709 if (token3 != OP) { 1709 if (token3 != PYTOK_OP) {
1710 token = token3; 1710 token = token3;
1711 } else { 1711 } else {
1712 tok_backup(tok, c3); 1712 tok_backup(tok, c3);
1713 } 1713 }
1714 *p_start = tok->start; 1714 *p_start = tok->start;
1715 *p_end = tok->cur; 1715 *p_end = tok->cur;
1716 return token; 1716 return token;
1717 } 1717 }
1718 tok_backup(tok, c2); 1718 tok_backup(tok, c2);
1719 } 1719 }
(...skipping 16 matching lines...) Expand all
1736 *p_start = tok->start; 1736 *p_start = tok->start;
1737 *p_end = tok->cur; 1737 *p_end = tok->cur;
1738 return PyToken_OneChar(c); 1738 return PyToken_OneChar(c);
1739 } 1739 }
1740 1740
1741 int 1741 int
1742 PyTokenizer_Get(PyTokenizer_State *tok, char **p_start, char **p_end) 1742 PyTokenizer_Get(PyTokenizer_State *tok, char **p_start, char **p_end)
1743 { 1743 {
1744 int result = tok_get(tok, p_start, p_end); 1744 int result = tok_get(tok, p_start, p_end);
1745 if (tok->decoding_erred) { 1745 if (tok->decoding_erred) {
1746 result = ERRORTOKEN; 1746 result = PYTOK_ERRORTOKEN;
1747 tok->done = E_DECODE; 1747 tok->done = E_DECODE;
1748 } 1748 }
1749 return result; 1749 return result;
1750 } 1750 }
1751 1751
1752 /* Get the encoding of a Python file. Check for the coding cookie and check if 1752 /* Get the encoding of a Python file. Check for the coding cookie and check if
1753 the file starts with a BOM. 1753 the file starts with a BOM.
1754 1754
1755 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the 1755 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
1756 encoding in the first or second line of the file (in which case the encoding 1756 encoding in the first or second line of the file (in which case the encoding
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
1816 { 1816 {
1817 return PyTokenizer_FindEncodingFilename(fd, NULL); 1817 return PyTokenizer_FindEncodingFilename(fd, NULL);
1818 } 1818 }
1819 1819
1820 #ifdef Py_DEBUG 1820 #ifdef Py_DEBUG
1821 1821
1822 void 1822 void
1823 tok_dump(int type, char *start, char *end) 1823 tok_dump(int type, char *start, char *end)
1824 { 1824 {
1825 printf("%s", _PyParser_TokenNames[type]); 1825 printf("%s", _PyParser_TokenNames[type]);
1826 if (type == NAME || type == NUMBER || type == STRING || type == OP) 1826 if (type == PYTOK_NAME || type == PYTOK_NUMBER || type == PYTOK_STRING || ty pe == PYTOK_OP)
1827 printf("(%.*s)", (int)(end - start), start); 1827 printf("(%.*s)", (int)(end - start), start);
1828 } 1828 }
1829 1829
1830 #endif 1830 #endif
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+