Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(5)

Delta Between Two Patch Sets: Parser/parsetok.c

Issue 3353: make built-in tokenizer available via Python C API
Left Patch Set: Created 4 years, 12 months ago
Right Patch Set: Created 4 years, 12 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Parser/parser.c ('k') | Parser/pgen.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 1
2 /* Parser-tokenizer link implementation */ 2 /* Parser-tokenizer link implementation */
3 3
4 #include "pgenheaders.h" 4 #include "pgenheaders.h"
5 #include "tokenizer.h" 5 #include "tokenizer.h"
6 #include "node.h" 6 #include "node.h"
7 #include "grammar.h" 7 #include "grammar.h"
8 #include "parser.h" 8 #include "parser.h"
9 #include "parsetok.h" 9 #include "parsetok.h"
10 #include "errcode.h" 10 #include "errcode.h"
(...skipping 188 matching lines...) Expand 10 before | Expand all | Expand 10 after
199 #endif 199 #endif
200 200
201 for (;;) { 201 for (;;) {
202 char *a, *b; 202 char *a, *b;
203 int type; 203 int type;
204 size_t len; 204 size_t len;
205 char *str; 205 char *str;
206 int col_offset; 206 int col_offset;
207 207
208 type = PyTokenizer_Get(tok, &a, &b); 208 type = PyTokenizer_Get(tok, &a, &b);
209 if (type == ERRORTOKEN) { 209 if (type == PYTOK_ERRORTOKEN) {
210 err_ret->error = tok->done; 210 err_ret->error = tok->done;
211 break; 211 break;
212 } 212 }
213 if (type == ENDMARKER && started) { 213 if (type == PYTOK_ENDMARKER && started) {
214 type = NEWLINE; /* Add an extra newline */ 214 type = PYTOK_NEWLINE; /* Add an extra newline */
215 started = 0; 215 started = 0;
216 /* Add the right number of dedent tokens, 216 /* Add the right number of dedent tokens,
217 except if a certain flag is given -- 217 except if a certain flag is given --
218 codeop.py uses this. */ 218 codeop.py uses this. */
219 if (tok->indent && 219 if (tok->indent &&
220 !(*flags & PyPARSE_DONT_IMPLY_DEDENT)) 220 !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
221 { 221 {
222 tok->pendin = -tok->indent; 222 tok->pendin = -tok->indent;
223 tok->indent = 0; 223 tok->indent = 0;
224 } 224 }
225 } 225 }
226 else 226 else
227 started = 1; 227 started = 1;
228 len = b - a; /* XXX this may compute NULL - NULL */ 228 len = b - a; /* XXX this may compute NULL - NULL */
229 str = (char *) PyObject_MALLOC(len + 1); 229 str = (char *) PyObject_MALLOC(len + 1);
230 if (str == NULL) { 230 if (str == NULL) {
231 err_ret->error = E_NOMEM; 231 err_ret->error = E_NOMEM;
232 break; 232 break;
233 } 233 }
234 if (len > 0) 234 if (len > 0)
235 strncpy(str, a, len); 235 strncpy(str, a, len);
236 str[len] = '\0'; 236 str[len] = '\0';
237 237
238 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 238 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
239 if (type == NOTEQUAL) { 239 if (type == PYTOK_NOTEQUAL) {
240 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 240 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
241 strcmp(str, "!=")) { 241 strcmp(str, "!=")) {
242 PyObject_FREE(str); 242 PyObject_FREE(str);
243 err_ret->error = E_SYNTAX; 243 err_ret->error = E_SYNTAX;
244 break; 244 break;
245 } 245 }
246 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 246 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
247 strcmp(str, "<>")) { 247 strcmp(str, "<>")) {
248 PyObject_FREE(str); 248 PyObject_FREE(str);
249 err_ret->text = "with Barry as BDFL, use '<>' " 249 err_ret->text = "with Barry as BDFL, use '<>' "
(...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after
374 else { 374 else {
375 err_ret->filename = PyUnicode_FromString("<string>"); 375 err_ret->filename = PyUnicode_FromString("<string>");
376 if (err_ret->filename == NULL) { 376 if (err_ret->filename == NULL) {
377 err_ret->error = E_ERROR; 377 err_ret->error = E_ERROR;
378 return -1; 378 return -1;
379 } 379 }
380 } 380 }
381 #endif 381 #endif
382 return 0; 382 return 0;
383 } 383 }
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+