Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(76753)

Delta Between Two Patch Sets: Parser/parsetok.c

Issue 11619: On Windows, don't encode filenames in the import machinery
Left Patch Set: Created 8 years, 4 months ago
Right Patch Set: Created 5 years, 11 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Modules/symtablemodule.c ('k') | Python/ast.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 1
2 /* Parser-tokenizer link implementation */ 2 /* Parser-tokenizer link implementation */
3 3
4 #include "pgenheaders.h" 4 #include "pgenheaders.h"
5 #include "tokenizer.h" 5 #include "tokenizer.h"
6 #include "node.h" 6 #include "node.h"
7 #include "grammar.h" 7 #include "grammar.h"
8 #include "parser.h" 8 #include "parser.h"
9 #include "parsetok.h" 9 #include "parsetok.h"
10 #include "errcode.h" 10 #include "errcode.h"
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after
182 182
183 static node * 183 static node *
184 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, 184 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
185 int *flags) 185 int *flags)
186 { 186 {
187 parser_state *ps; 187 parser_state *ps;
188 node *n; 188 node *n;
189 int started = 0; 189 int started = 0;
190 190
191 if ((ps = PyParser_New(g, start)) == NULL) { 191 if ((ps = PyParser_New(g, start)) == NULL) {
192 fprintf(stderr, "no mem for new parser\n");
193 err_ret->error = E_NOMEM; 192 err_ret->error = E_NOMEM;
194 PyTokenizer_Free(tok); 193 PyTokenizer_Free(tok);
195 return NULL; 194 return NULL;
196 } 195 }
197 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 196 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
198 if (*flags & PyPARSE_BARRY_AS_BDFL) 197 if (*flags & PyPARSE_BARRY_AS_BDFL)
199 ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL; 198 ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL;
200 #endif 199 #endif
201 200
202 for (;;) { 201 for (;;) {
(...skipping 19 matching lines...) Expand all
222 { 221 {
223 tok->pendin = -tok->indent; 222 tok->pendin = -tok->indent;
224 tok->indent = 0; 223 tok->indent = 0;
225 } 224 }
226 } 225 }
227 else 226 else
228 started = 1; 227 started = 1;
229 len = b - a; /* XXX this may compute NULL - NULL */ 228 len = b - a; /* XXX this may compute NULL - NULL */
230 str = (char *) PyObject_MALLOC(len + 1); 229 str = (char *) PyObject_MALLOC(len + 1);
231 if (str == NULL) { 230 if (str == NULL) {
232 fprintf(stderr, "no mem for next token\n");
233 err_ret->error = E_NOMEM; 231 err_ret->error = E_NOMEM;
234 break; 232 break;
235 } 233 }
236 if (len > 0) 234 if (len > 0)
237 strncpy(str, a, len); 235 strncpy(str, a, len);
238 str[len] = '\0'; 236 str[len] = '\0';
239 237
240 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 238 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
241 if (type == NOTEQUAL) { 239 if (type == NOTEQUAL) {
242 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 240 if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
243 strcmp(str, "!=")) { 241 strcmp(str, "!=")) {
242 PyObject_FREE(str);
244 err_ret->error = E_SYNTAX; 243 err_ret->error = E_SYNTAX;
245 break; 244 break;
246 } 245 }
247 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) && 246 else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
248 strcmp(str, "<>")) { 247 strcmp(str, "<>")) {
248 PyObject_FREE(str);
249 err_ret->text = "with Barry as BDFL, use '<>' " 249 err_ret->text = "with Barry as BDFL, use '<>' "
250 "instead of '!='"; 250 "instead of '!='";
251 err_ret->error = E_SYNTAX; 251 err_ret->error = E_SYNTAX;
252 break; 252 break;
253 } 253 }
254 } 254 }
255 #endif 255 #endif
256 if (a >= tok->line_start) 256 if (a >= tok->line_start)
257 col_offset = a - tok->line_start; 257 col_offset = a - tok->line_start;
258 else 258 else
259 col_offset = -1; 259 col_offset = -1;
260 260
261 if ((err_ret->error = 261 if ((err_ret->error =
262 PyParser_AddToken(ps, (int)type, str, 262 PyParser_AddToken(ps, (int)type, str,
263 tok->lineno, col_offset, 263 tok->lineno, col_offset,
264 &(err_ret->expected))) != E_OK) { 264 &(err_ret->expected))) != E_OK) {
265 if (err_ret->error != E_DONE) { 265 if (err_ret->error != E_DONE) {
266 PyObject_FREE(str); 266 PyObject_FREE(str);
267 err_ret->token = type; 267 err_ret->token = type;
268 } 268 }
269 break; 269 break;
270 } 270 }
271 } 271 }
272 272
273 if (err_ret->error == E_DONE) { 273 if (err_ret->error == E_DONE) {
274 n = ps->p_tree; 274 n = ps->p_tree;
275 ps->p_tree = NULL; 275 ps->p_tree = NULL;
276
277 #ifndef PGEN
278 /* Check that the source for a single input statement really
279 is a single statement by looking at what is left in the
280 buffer after parsing. Trailing whitespace and comments
281 are OK. */
282 if (start == single_input) {
283 char *cur = tok->cur;
284 char c = *tok->cur;
285
286 for (;;) {
287 while (c == ' ' || c == '\t' || c == '\n' || c == '\014')
288 c = *++cur;
289
290 if (!c)
291 break;
292
293 if (c != '#') {
294 err_ret->error = E_BADSINGLE;
295 PyNode_Free(n);
296 n = NULL;
297 break;
298 }
299
300 /* Suck up comment. */
301 while (c && c != '\n')
302 c = *++cur;
303 }
304 }
305 #endif
276 } 306 }
277 else 307 else
278 n = NULL; 308 n = NULL;
279 309
280 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD 310 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
281 *flags = ps->p_flags; 311 *flags = ps->p_flags;
282 #endif 312 #endif
283 PyParser_Delete(ps); 313 PyParser_Delete(ps);
284 314
285 if (n == NULL) { 315 if (n == NULL) {
286 if (tok->lineno <= 1 && tok->done == E_EOF) 316 if (tok->done == E_EOF)
287 err_ret->error = E_EOF; 317 err_ret->error = E_EOF;
288 err_ret->lineno = tok->lineno; 318 err_ret->lineno = tok->lineno;
289 if (tok->buf != NULL) { 319 if (tok->buf != NULL) {
290 size_t len; 320 size_t len;
291 assert(tok->cur - tok->buf < INT_MAX); 321 assert(tok->cur - tok->buf < INT_MAX);
292 err_ret->offset = (int)(tok->cur - tok->buf); 322 err_ret->offset = (int)(tok->cur - tok->buf);
293 len = tok->inp - tok->buf; 323 len = tok->inp - tok->buf;
294 err_ret->text = (char *) PyObject_MALLOC(len + 1); 324 err_ret->text = (char *) PyObject_MALLOC(len + 1);
295 if (err_ret->text != NULL) { 325 if (err_ret->text != NULL) {
296 if (len > 0) 326 if (len > 0)
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
343 else { 373 else {
344 err_ret->filename = PyUnicode_FromString("<string>"); 374 err_ret->filename = PyUnicode_FromString("<string>");
345 if (err_ret->filename == NULL) { 375 if (err_ret->filename == NULL) {
346 err_ret->error = E_ERROR; 376 err_ret->error = E_ERROR;
347 return -1; 377 return -1;
348 } 378 }
349 } 379 }
350 #endif 380 #endif
351 return 0; 381 return 0;
352 } 382 }
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+