Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(154529)

Delta Between Two Patch Sets: Python/ast.c

Issue 11619: On Windows, don't encode filenames in the import machinery
Left Patch Set: Created 8 years, 7 months ago
Right Patch Set: Created 6 years, 3 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Parser/parsetok.c ('k') | Python/compile.c » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 /* 1 /*
2 * This file includes functions to transform a concrete syntax tree (CST) to 2 * This file includes functions to transform a concrete syntax tree (CST) to
3 * an abstract syntax tree (AST). The main function is PyAST_FromNode(). 3 * an abstract syntax tree (AST). The main function is PyAST_FromNode().
4 * 4 *
5 */ 5 */
6 #include "Python.h" 6 #include "Python.h"
7 #include "Python-ast.h" 7 #include "Python-ast.h"
8 #include "grammar.h"
9 #include "node.h" 8 #include "node.h"
10 #include "ast.h" 9 #include "ast.h"
11 #include "token.h" 10 #include "token.h"
11
12 #include <assert.h>
13
14 static int validate_stmts(asdl_seq *);
15 static int validate_exprs(asdl_seq *, expr_context_ty, int);
16 static int validate_nonempty_seq(asdl_seq *, const char *, const char *);
17 static int validate_stmt(stmt_ty);
18 static int validate_expr(expr_ty, expr_context_ty);
19
20 static int
21 validate_comprehension(asdl_seq *gens)
22 {
23 int i;
24 if (!asdl_seq_LEN(gens)) {
25 PyErr_SetString(PyExc_ValueError, "comprehension with no generators");
26 return 0;
27 }
28 for (i = 0; i < asdl_seq_LEN(gens); i++) {
29 comprehension_ty comp = asdl_seq_GET(gens, i);
30 if (!validate_expr(comp->target, Store) ||
31 !validate_expr(comp->iter, Load) ||
32 !validate_exprs(comp->ifs, Load, 0))
33 return 0;
34 }
35 return 1;
36 }
37
38 static int
39 validate_slice(slice_ty slice)
40 {
41 switch (slice->kind) {
42 case Slice_kind:
43 return (!slice->v.Slice.lower || validate_expr(slice->v.Slice.lower, Loa d)) &&
44 (!slice->v.Slice.upper || validate_expr(slice->v.Slice.upper, Load)) &&
45 (!slice->v.Slice.step || validate_expr(slice->v.Slice.step, Load));
46 case ExtSlice_kind: {
47 int i;
48 if (!validate_nonempty_seq(slice->v.ExtSlice.dims, "dims", "ExtSlice"))
49 return 0;
50 for (i = 0; i < asdl_seq_LEN(slice->v.ExtSlice.dims); i++)
51 if (!validate_slice(asdl_seq_GET(slice->v.ExtSlice.dims, i)))
52 return 0;
53 return 1;
54 }
55 case Index_kind:
56 return validate_expr(slice->v.Index.value, Load);
57 default:
58 PyErr_SetString(PyExc_SystemError, "unknown slice node");
59 return 0;
60 }
61 }
62
63 static int
64 validate_keywords(asdl_seq *keywords)
65 {
66 int i;
67 for (i = 0; i < asdl_seq_LEN(keywords); i++)
68 if (!validate_expr(((keyword_ty)asdl_seq_GET(keywords, i))->value, Load) )
69 return 0;
70 return 1;
71 }
72
73 static int
74 validate_args(asdl_seq *args)
75 {
76 int i;
77 for (i = 0; i < asdl_seq_LEN(args); i++) {
78 arg_ty arg = asdl_seq_GET(args, i);
79 if (arg->annotation && !validate_expr(arg->annotation, Load))
80 return 0;
81 }
82 return 1;
83 }
84
85 static const char *
86 expr_context_name(expr_context_ty ctx)
87 {
88 switch (ctx) {
89 case Load:
90 return "Load";
91 case Store:
92 return "Store";
93 case Del:
94 return "Del";
95 case AugLoad:
96 return "AugLoad";
97 case AugStore:
98 return "AugStore";
99 case Param:
100 return "Param";
101 default:
102 assert(0);
103 return "(unknown)";
104 }
105 }
106
107 static int
108 validate_arguments(arguments_ty args)
109 {
110 if (!validate_args(args->args))
111 return 0;
112 if (args->vararg && args->vararg->annotation
113 && !validate_expr(args->vararg->annotation, Load)) {
114 return 0;
115 }
116 if (!validate_args(args->kwonlyargs))
117 return 0;
118 if (args->kwarg && args->kwarg->annotation
119 && !validate_expr(args->kwarg->annotation, Load)) {
120 return 0;
121 }
122 if (asdl_seq_LEN(args->defaults) > asdl_seq_LEN(args->args)) {
123 PyErr_SetString(PyExc_ValueError, "more positional defaults than args on arguments");
124 return 0;
125 }
126 if (asdl_seq_LEN(args->kw_defaults) != asdl_seq_LEN(args->kwonlyargs)) {
127 PyErr_SetString(PyExc_ValueError, "length of kwonlyargs is not the same as "
128 "kw_defaults on arguments");
129 return 0;
130 }
131 return validate_exprs(args->defaults, Load, 0) && validate_exprs(args->kw_de faults, Load, 1);
132 }
133
134 static int
135 validate_expr(expr_ty exp, expr_context_ty ctx)
136 {
137 int check_ctx = 1;
138 expr_context_ty actual_ctx;
139
140 /* First check expression context. */
141 switch (exp->kind) {
142 case Attribute_kind:
143 actual_ctx = exp->v.Attribute.ctx;
144 break;
145 case Subscript_kind:
146 actual_ctx = exp->v.Subscript.ctx;
147 break;
148 case Starred_kind:
149 actual_ctx = exp->v.Starred.ctx;
150 break;
151 case Name_kind:
152 actual_ctx = exp->v.Name.ctx;
153 break;
154 case List_kind:
155 actual_ctx = exp->v.List.ctx;
156 break;
157 case Tuple_kind:
158 actual_ctx = exp->v.Tuple.ctx;
159 break;
160 default:
161 if (ctx != Load) {
162 PyErr_Format(PyExc_ValueError, "expression which can't be "
163 "assigned to in %s context", expr_context_name(ctx));
164 return 0;
165 }
166 check_ctx = 0;
167 }
168 if (check_ctx && actual_ctx != ctx) {
169 PyErr_Format(PyExc_ValueError, "expression must have %s context but has %s instead",
170 expr_context_name(ctx), expr_context_name(actual_ctx));
171 return 0;
172 }
173
174 /* Now validate expression. */
175 switch (exp->kind) {
176 case BoolOp_kind:
177 if (asdl_seq_LEN(exp->v.BoolOp.values) < 2) {
178 PyErr_SetString(PyExc_ValueError, "BoolOp with less than 2 values");
179 return 0;
180 }
181 return validate_exprs(exp->v.BoolOp.values, Load, 0);
182 case BinOp_kind:
183 return validate_expr(exp->v.BinOp.left, Load) &&
184 validate_expr(exp->v.BinOp.right, Load);
185 case UnaryOp_kind:
186 return validate_expr(exp->v.UnaryOp.operand, Load);
187 case Lambda_kind:
188 return validate_arguments(exp->v.Lambda.args) &&
189 validate_expr(exp->v.Lambda.body, Load);
190 case IfExp_kind:
191 return validate_expr(exp->v.IfExp.test, Load) &&
192 validate_expr(exp->v.IfExp.body, Load) &&
193 validate_expr(exp->v.IfExp.orelse, Load);
194 case Dict_kind:
195 if (asdl_seq_LEN(exp->v.Dict.keys) != asdl_seq_LEN(exp->v.Dict.values)) {
196 PyErr_SetString(PyExc_ValueError,
197 "Dict doesn't have the same number of keys as values ");
198 return 0;
199 }
200 return validate_exprs(exp->v.Dict.keys, Load, 0) &&
201 validate_exprs(exp->v.Dict.values, Load, 0);
202 case Set_kind:
203 return validate_exprs(exp->v.Set.elts, Load, 0);
204 #define COMP(NAME) \
205 case NAME ## _kind: \
206 return validate_comprehension(exp->v.NAME.generators) && \
207 validate_expr(exp->v.NAME.elt, Load);
208 COMP(ListComp)
209 COMP(SetComp)
210 COMP(GeneratorExp)
211 #undef COMP
212 case DictComp_kind:
213 return validate_comprehension(exp->v.DictComp.generators) &&
214 validate_expr(exp->v.DictComp.key, Load) &&
215 validate_expr(exp->v.DictComp.value, Load);
216 case Yield_kind:
217 return !exp->v.Yield.value || validate_expr(exp->v.Yield.value, Load);
218 case YieldFrom_kind:
219 return validate_expr(exp->v.YieldFrom.value, Load);
220 case Compare_kind:
221 if (!asdl_seq_LEN(exp->v.Compare.comparators)) {
222 PyErr_SetString(PyExc_ValueError, "Compare with no comparators");
223 return 0;
224 }
225 if (asdl_seq_LEN(exp->v.Compare.comparators) !=
226 asdl_seq_LEN(exp->v.Compare.ops)) {
227 PyErr_SetString(PyExc_ValueError, "Compare has a different number "
228 "of comparators and operands");
229 return 0;
230 }
231 return validate_exprs(exp->v.Compare.comparators, Load, 0) &&
232 validate_expr(exp->v.Compare.left, Load);
233 case Call_kind:
234 return validate_expr(exp->v.Call.func, Load) &&
235 validate_exprs(exp->v.Call.args, Load, 0) &&
236 validate_keywords(exp->v.Call.keywords) &&
237 (!exp->v.Call.starargs || validate_expr(exp->v.Call.starargs, Load)) &&
238 (!exp->v.Call.kwargs || validate_expr(exp->v.Call.kwargs, Load));
239 case Num_kind: {
240 PyObject *n = exp->v.Num.n;
241 if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) &&
242 !PyComplex_CheckExact(n)) {
243 PyErr_SetString(PyExc_TypeError, "non-numeric type in Num");
244 return 0;
245 }
246 return 1;
247 }
248 case Str_kind: {
249 PyObject *s = exp->v.Str.s;
250 if (!PyUnicode_CheckExact(s)) {
251 PyErr_SetString(PyExc_TypeError, "non-string type in Str");
252 return 0;
253 }
254 return 1;
255 }
256 case Bytes_kind: {
257 PyObject *b = exp->v.Bytes.s;
258 if (!PyBytes_CheckExact(b)) {
259 PyErr_SetString(PyExc_TypeError, "non-bytes type in Bytes");
260 return 0;
261 }
262 return 1;
263 }
264 case Attribute_kind:
265 return validate_expr(exp->v.Attribute.value, Load);
266 case Subscript_kind:
267 return validate_slice(exp->v.Subscript.slice) &&
268 validate_expr(exp->v.Subscript.value, Load);
269 case Starred_kind:
270 return validate_expr(exp->v.Starred.value, ctx);
271 case List_kind:
272 return validate_exprs(exp->v.List.elts, ctx, 0);
273 case Tuple_kind:
274 return validate_exprs(exp->v.Tuple.elts, ctx, 0);
275 /* These last cases don't have any checking. */
276 case Name_kind:
277 case NameConstant_kind:
278 case Ellipsis_kind:
279 return 1;
280 default:
281 PyErr_SetString(PyExc_SystemError, "unexpected expression");
282 return 0;
283 }
284 }
285
286 static int
287 validate_nonempty_seq(asdl_seq *seq, const char *what, const char *owner)
288 {
289 if (asdl_seq_LEN(seq))
290 return 1;
291 PyErr_Format(PyExc_ValueError, "empty %s on %s", what, owner);
292 return 0;
293 }
294
295 static int
296 validate_assignlist(asdl_seq *targets, expr_context_ty ctx)
297 {
298 return validate_nonempty_seq(targets, "targets", ctx == Del ? "Delete" : "As sign") &&
299 validate_exprs(targets, ctx, 0);
300 }
301
302 static int
303 validate_body(asdl_seq *body, const char *owner)
304 {
305 return validate_nonempty_seq(body, "body", owner) && validate_stmts(body);
306 }
307
308 static int
309 validate_stmt(stmt_ty stmt)
310 {
311 int i;
312 switch (stmt->kind) {
313 case FunctionDef_kind:
314 return validate_body(stmt->v.FunctionDef.body, "FunctionDef") &&
315 validate_arguments(stmt->v.FunctionDef.args) &&
316 validate_exprs(stmt->v.FunctionDef.decorator_list, Load, 0) &&
317 (!stmt->v.FunctionDef.returns ||
318 validate_expr(stmt->v.FunctionDef.returns, Load));
319 case ClassDef_kind:
320 return validate_body(stmt->v.ClassDef.body, "ClassDef") &&
321 validate_exprs(stmt->v.ClassDef.bases, Load, 0) &&
322 validate_keywords(stmt->v.ClassDef.keywords) &&
323 validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0) &&
324 (!stmt->v.ClassDef.starargs || validate_expr(stmt->v.ClassDef.starar gs, Load)) &&
325 (!stmt->v.ClassDef.kwargs || validate_expr(stmt->v.ClassDef.kwargs, Load));
326 case Return_kind:
327 return !stmt->v.Return.value || validate_expr(stmt->v.Return.value, Load );
328 case Delete_kind:
329 return validate_assignlist(stmt->v.Delete.targets, Del);
330 case Assign_kind:
331 return validate_assignlist(stmt->v.Assign.targets, Store) &&
332 validate_expr(stmt->v.Assign.value, Load);
333 case AugAssign_kind:
334 return validate_expr(stmt->v.AugAssign.target, Store) &&
335 validate_expr(stmt->v.AugAssign.value, Load);
336 case For_kind:
337 return validate_expr(stmt->v.For.target, Store) &&
338 validate_expr(stmt->v.For.iter, Load) &&
339 validate_body(stmt->v.For.body, "For") &&
340 validate_stmts(stmt->v.For.orelse);
341 case While_kind:
342 return validate_expr(stmt->v.While.test, Load) &&
343 validate_body(stmt->v.While.body, "While") &&
344 validate_stmts(stmt->v.While.orelse);
345 case If_kind:
346 return validate_expr(stmt->v.If.test, Load) &&
347 validate_body(stmt->v.If.body, "If") &&
348 validate_stmts(stmt->v.If.orelse);
349 case With_kind:
350 if (!validate_nonempty_seq(stmt->v.With.items, "items", "With"))
351 return 0;
352 for (i = 0; i < asdl_seq_LEN(stmt->v.With.items); i++) {
353 withitem_ty item = asdl_seq_GET(stmt->v.With.items, i);
354 if (!validate_expr(item->context_expr, Load) ||
355 (item->optional_vars && !validate_expr(item->optional_vars, Stor e)))
356 return 0;
357 }
358 return validate_body(stmt->v.With.body, "With");
359 case Raise_kind:
360 if (stmt->v.Raise.exc) {
361 return validate_expr(stmt->v.Raise.exc, Load) &&
362 (!stmt->v.Raise.cause || validate_expr(stmt->v.Raise.cause, Load ));
363 }
364 if (stmt->v.Raise.cause) {
365 PyErr_SetString(PyExc_ValueError, "Raise with cause but no exception ");
366 return 0;
367 }
368 return 1;
369 case Try_kind:
370 if (!validate_body(stmt->v.Try.body, "Try"))
371 return 0;
372 if (!asdl_seq_LEN(stmt->v.Try.handlers) &&
373 !asdl_seq_LEN(stmt->v.Try.finalbody)) {
374 PyErr_SetString(PyExc_ValueError, "Try has neither except handlers n or finalbody");
375 return 0;
376 }
377 if (!asdl_seq_LEN(stmt->v.Try.handlers) &&
378 asdl_seq_LEN(stmt->v.Try.orelse)) {
379 PyErr_SetString(PyExc_ValueError, "Try has orelse but no except hand lers");
380 return 0;
381 }
382 for (i = 0; i < asdl_seq_LEN(stmt->v.Try.handlers); i++) {
383 excepthandler_ty handler = asdl_seq_GET(stmt->v.Try.handlers, i);
384 if ((handler->v.ExceptHandler.type &&
385 !validate_expr(handler->v.ExceptHandler.type, Load)) ||
386 !validate_body(handler->v.ExceptHandler.body, "ExceptHandler"))
387 return 0;
388 }
389 return (!asdl_seq_LEN(stmt->v.Try.finalbody) ||
390 validate_stmts(stmt->v.Try.finalbody)) &&
391 (!asdl_seq_LEN(stmt->v.Try.orelse) ||
392 validate_stmts(stmt->v.Try.orelse));
393 case Assert_kind:
394 return validate_expr(stmt->v.Assert.test, Load) &&
395 (!stmt->v.Assert.msg || validate_expr(stmt->v.Assert.msg, Load));
396 case Import_kind:
397 return validate_nonempty_seq(stmt->v.Import.names, "names", "Import");
398 case ImportFrom_kind:
399 if (stmt->v.ImportFrom.level < -1) {
400 PyErr_SetString(PyExc_ValueError, "ImportFrom level less than -1");
401 return 0;
402 }
403 return validate_nonempty_seq(stmt->v.ImportFrom.names, "names", "ImportF rom");
404 case Global_kind:
405 return validate_nonempty_seq(stmt->v.Global.names, "names", "Global");
406 case Nonlocal_kind:
407 return validate_nonempty_seq(stmt->v.Nonlocal.names, "names", "Nonlocal" );
408 case Expr_kind:
409 return validate_expr(stmt->v.Expr.value, Load);
410 case Pass_kind:
411 case Break_kind:
412 case Continue_kind:
413 return 1;
414 default:
415 PyErr_SetString(PyExc_SystemError, "unexpected statement");
416 return 0;
417 }
418 }
419
420 static int
421 validate_stmts(asdl_seq *seq)
422 {
423 int i;
424 for (i = 0; i < asdl_seq_LEN(seq); i++) {
425 stmt_ty stmt = asdl_seq_GET(seq, i);
426 if (stmt) {
427 if (!validate_stmt(stmt))
428 return 0;
429 }
430 else {
431 PyErr_SetString(PyExc_ValueError,
432 "None disallowed in statement list");
433 return 0;
434 }
435 }
436 return 1;
437 }
438
439 static int
440 validate_exprs(asdl_seq *exprs, expr_context_ty ctx, int null_ok)
441 {
442 int i;
443 for (i = 0; i < asdl_seq_LEN(exprs); i++) {
444 expr_ty expr = asdl_seq_GET(exprs, i);
445 if (expr) {
446 if (!validate_expr(expr, ctx))
447 return 0;
448 }
449 else if (!null_ok) {
450 PyErr_SetString(PyExc_ValueError,
451 "None disallowed in expression list");
452 return 0;
453 }
454
455 }
456 return 1;
457 }
458
459 int
460 PyAST_Validate(mod_ty mod)
461 {
462 int res = 0;
463
464 switch (mod->kind) {
465 case Module_kind:
466 res = validate_stmts(mod->v.Module.body);
467 break;
468 case Interactive_kind:
469 res = validate_stmts(mod->v.Interactive.body);
470 break;
471 case Expression_kind:
472 res = validate_expr(mod->v.Expression.body, Load);
473 break;
474 case Suite_kind:
475 PyErr_SetString(PyExc_ValueError, "Suite is not valid in the CPython com piler");
476 break;
477 default:
478 PyErr_SetString(PyExc_SystemError, "impossible module node");
479 res = 0;
480 break;
481 }
482 return res;
483 }
484
485 /* This is done here, so defines like "test" don't interfere with AST use above. */
486 #include "grammar.h"
12 #include "parsetok.h" 487 #include "parsetok.h"
13 #include "graminit.h" 488 #include "graminit.h"
14
15 #include <assert.h>
16 489
17 /* Data structure used internally */ 490 /* Data structure used internally */
18 struct compiling { 491 struct compiling {
19 char *c_encoding; /* source encoding */ 492 char *c_encoding; /* source encoding */
20 PyArena *c_arena; /* arena for allocating memeory */ 493 PyArena *c_arena; /* arena for allocating memeory */
494 PyObject *c_filename; /* filename */
495 PyObject *c_normalize; /* Normalization function from unicodedata. */
496 PyObject *c_normalize_args; /* Normalization argument tuple. */
21 }; 497 };
22 498
23 static asdl_seq *seq_for_testlist(struct compiling *, const node *); 499 static asdl_seq *seq_for_testlist(struct compiling *, const node *);
24 static expr_ty ast_for_expr(struct compiling *, const node *); 500 static expr_ty ast_for_expr(struct compiling *, const node *);
25 static stmt_ty ast_for_stmt(struct compiling *, const node *); 501 static stmt_ty ast_for_stmt(struct compiling *, const node *);
26 static asdl_seq *ast_for_suite(struct compiling *, const node *); 502 static asdl_seq *ast_for_suite(struct compiling *, const node *);
27 static asdl_seq *ast_for_exprlist(struct compiling *, const node *, 503 static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
28 expr_context_ty); 504 expr_context_ty);
29 static expr_ty ast_for_testlist(struct compiling *, const node *); 505 static expr_ty ast_for_testlist(struct compiling *, const node *);
30 static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *); 506 static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
31 507
32 /* Note different signature for ast_for_call */ 508 /* Note different signature for ast_for_call */
33 static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); 509 static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
34 510
35 static PyObject *parsenumber(struct compiling *, const char *); 511 static PyObject *parsenumber(struct compiling *, const char *);
36 static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode); 512 static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode);
37 static PyObject *parsestrplus(struct compiling *, const node *n, 513 static PyObject *parsestrplus(struct compiling *, const node *n,
38 int *bytesmode); 514 int *bytesmode);
39 515
40 #ifndef LINENO
41 #define LINENO(n) ((n)->n_lineno)
42 #endif
43
44 #define COMP_GENEXP 0 516 #define COMP_GENEXP 0
45 #define COMP_LISTCOMP 1 517 #define COMP_LISTCOMP 1
46 #define COMP_SETCOMP 2 518 #define COMP_SETCOMP 2
47 519
520 static int
521 init_normalization(struct compiling *c)
522 {
523 PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
524 if (!m)
525 return 0;
526 c->c_normalize = PyObject_GetAttrString(m, "normalize");
527 Py_DECREF(m);
528 if (!c->c_normalize)
529 return 0;
530 c->c_normalize_args = Py_BuildValue("(sN)", "NFKC", Py_None);
531 if (!c->c_normalize_args) {
532 Py_CLEAR(c->c_normalize);
533 return 0;
534 }
535 PyTuple_SET_ITEM(c->c_normalize_args, 1, NULL);
536 return 1;
537 }
538
48 static identifier 539 static identifier
49 new_identifier(const char* n, PyArena *arena) 540 new_identifier(const char *n, struct compiling *c)
50 { 541 {
51 PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL); 542 PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
52 Py_UNICODE *u;
53 if (!id) 543 if (!id)
54 return NULL; 544 return NULL;
55 u = PyUnicode_AS_UNICODE(id); 545 /* PyUnicode_DecodeUTF8 should always return a ready string. */
546 assert(PyUnicode_IS_READY(id));
56 /* Check whether there are non-ASCII characters in the 547 /* Check whether there are non-ASCII characters in the
57 identifier; if so, normalize to NFKC. */ 548 identifier; if so, normalize to NFKC. */
58 for (; *u; u++) { 549 if (!PyUnicode_IS_ASCII(id)) {
59 if (*u >= 128) { 550 PyObject *id2;
60 PyObject *m = PyImport_ImportModuleNoBlock("unicodedata"); 551 if (!c->c_normalize && !init_normalization(c)) {
61 PyObject *id2;
62 if (!m)
63 return NULL;
64 id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
65 Py_DECREF(m);
66 if (!id2)
67 return NULL;
68 Py_DECREF(id); 552 Py_DECREF(id);
69 id = id2; 553 return NULL;
70 break; 554 }
71 } 555 PyTuple_SET_ITEM(c->c_normalize_args, 1, id);
556 id2 = PyObject_Call(c->c_normalize, c->c_normalize_args, NULL);
557 Py_DECREF(id);
558 if (!id2)
559 return NULL;
560 id = id2;
72 } 561 }
73 PyUnicode_InternInPlace(&id); 562 PyUnicode_InternInPlace(&id);
74 PyArena_AddPyObject(arena, id); 563 if (PyArena_AddPyObject(c->c_arena, id) < 0) {
564 Py_DECREF(id);
565 return NULL;
566 }
75 return id; 567 return id;
76 } 568 }
77 569
78 #define NEW_IDENTIFIER(n) new_identifier(STR(n), c->c_arena) 570 #define NEW_IDENTIFIER(n) new_identifier(STR(n), c)
79
80 /* This routine provides an invalid object for the syntax error.
81 The outermost routine must unpack this error and create the
82 proper object. We do this so that we don't have to pass
83 the filename to everything function.
84
85 XXX Maybe we should just pass the filename...
86 */
87 571
88 static int 572 static int
89 ast_error(const node *n, const char *errstr) 573 ast_error(struct compiling *c, const node *n, const char *errmsg)
90 { 574 {
91 PyObject *u = Py_BuildValue("zii", errstr, LINENO(n), n->n_col_offset); 575 PyObject *value, *errstr, *loc, *tmp;
92 if (!u) 576
93 return 0; 577 loc = PyErr_ProgramTextObject(c->c_filename, LINENO(n));
94 PyErr_SetObject(PyExc_SyntaxError, u);
95 Py_DECREF(u);
96 return 0;
97 }
98
99 static void
100 ast_error_finish(PyObject *filename)
101 {
102 PyObject *type, *value, *tback, *errstr, *offset, *loc, *tmp;
103 long lineno;
104
105 assert(PyErr_Occurred());
106 if (!PyErr_ExceptionMatches(PyExc_SyntaxError))
107 return;
108
109 PyErr_Fetch(&type, &value, &tback);
110 errstr = PyTuple_GetItem(value, 0);
111 if (!errstr)
112 return;
113 Py_INCREF(errstr);
114 lineno = PyLong_AsLong(PyTuple_GetItem(value, 1));
115 if (lineno == -1) {
116 Py_DECREF(errstr);
117 return;
118 }
119 offset = PyTuple_GetItem(value, 2);
120 if (!offset) {
121 Py_DECREF(errstr);
122 return;
123 }
124 Py_DECREF(value);
125
126 loc = PyErr_ProgramTextObject(filename, lineno);
127 if (!loc) { 578 if (!loc) {
128 Py_INCREF(Py_None); 579 Py_INCREF(Py_None);
129 loc = Py_None; 580 loc = Py_None;
130 } 581 }
131 tmp = Py_BuildValue("(OlOO)", 582 tmp = Py_BuildValue("(OiiN)", c->c_filename, LINENO(n), n->n_col_offset, loc );
132 filename ? filename : Py_None, lineno, offset, loc); 583 if (!tmp)
133 Py_DECREF(loc); 584 return 0;
134 if (!tmp) { 585 errstr = PyUnicode_FromString(errmsg);
135 Py_DECREF(errstr); 586 if (!errstr) {
136 return; 587 Py_DECREF(tmp);
588 return 0;
137 } 589 }
138 value = PyTuple_Pack(2, errstr, tmp); 590 value = PyTuple_Pack(2, errstr, tmp);
139 Py_DECREF(errstr); 591 Py_DECREF(errstr);
140 Py_DECREF(tmp); 592 Py_DECREF(tmp);
141 if (!value) 593 if (value) {
142 return; 594 PyErr_SetObject(PyExc_SyntaxError, value);
143 PyErr_Restore(type, value, tback); 595 Py_DECREF(value);
596 }
597 return 0;
144 } 598 }
145 599
146 /* num_stmts() returns number of contained statements. 600 /* num_stmts() returns number of contained statements.
147 601
148 Use this routine to determine how big a sequence is needed for 602 Use this routine to determine how big a sequence is needed for
149 the statements in a parse tree. Its raison d'etre is this bit of 603 the statements in a parse tree. Its raison d'etre is this bit of
150 grammar: 604 grammar:
151 605
152 stmt: simple_stmt | compound_stmt 606 stmt: simple_stmt | compound_stmt
153 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 607 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after
209 663
210 mod_ty 664 mod_ty
211 PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, 665 PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
212 PyObject *filename, PyArena *arena) 666 PyObject *filename, PyArena *arena)
213 { 667 {
214 int i, j, k, num; 668 int i, j, k, num;
215 asdl_seq *stmts = NULL; 669 asdl_seq *stmts = NULL;
216 stmt_ty s; 670 stmt_ty s;
217 node *ch; 671 node *ch;
218 struct compiling c; 672 struct compiling c;
219 673 mod_ty res = NULL;
674
675 c.c_arena = arena;
676 /* borrowed reference */
677 c.c_filename = filename;
678 c.c_normalize = c.c_normalize_args = NULL;
220 if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { 679 if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) {
221 c.c_encoding = "utf-8"; 680 c.c_encoding = "utf-8";
222 if (TYPE(n) == encoding_decl) { 681 if (TYPE(n) == encoding_decl) {
223 #if 0 682 #if 0
224 ast_error(n, "encoding declaration in Unicode string"); 683 ast_error(c, n, "encoding declaration in Unicode string");
225 goto error; 684 goto out;
226 #endif 685 #endif
227 n = CHILD(n, 0); 686 n = CHILD(n, 0);
228 } 687 }
229 } else if (TYPE(n) == encoding_decl) { 688 } else if (TYPE(n) == encoding_decl) {
230 c.c_encoding = STR(n); 689 c.c_encoding = STR(n);
231 n = CHILD(n, 0); 690 n = CHILD(n, 0);
232 } else { 691 } else {
233 /* PEP 3120 */ 692 /* PEP 3120 */
234 c.c_encoding = "utf-8"; 693 c.c_encoding = "utf-8";
235 } 694 }
236 c.c_arena = arena;
237 695
238 k = 0; 696 k = 0;
239 switch (TYPE(n)) { 697 switch (TYPE(n)) {
240 case file_input: 698 case file_input:
241 stmts = asdl_seq_new(num_stmts(n), arena); 699 stmts = asdl_seq_new(num_stmts(n), arena);
242 if (!stmts) 700 if (!stmts)
243 return NULL; 701 goto out;
244 for (i = 0; i < NCH(n) - 1; i++) { 702 for (i = 0; i < NCH(n) - 1; i++) {
245 ch = CHILD(n, i); 703 ch = CHILD(n, i);
246 if (TYPE(ch) == NEWLINE) 704 if (TYPE(ch) == NEWLINE)
247 continue; 705 continue;
248 REQ(ch, stmt); 706 REQ(ch, stmt);
249 num = num_stmts(ch); 707 num = num_stmts(ch);
250 if (num == 1) { 708 if (num == 1) {
251 s = ast_for_stmt(&c, ch); 709 s = ast_for_stmt(&c, ch);
252 if (!s) 710 if (!s)
253 goto error; 711 goto out;
254 asdl_seq_SET(stmts, k++, s); 712 asdl_seq_SET(stmts, k++, s);
255 } 713 }
256 else { 714 else {
257 ch = CHILD(ch, 0); 715 ch = CHILD(ch, 0);
258 REQ(ch, simple_stmt); 716 REQ(ch, simple_stmt);
259 for (j = 0; j < num; j++) { 717 for (j = 0; j < num; j++) {
260 s = ast_for_stmt(&c, CHILD(ch, j * 2)); 718 s = ast_for_stmt(&c, CHILD(ch, j * 2));
261 if (!s) 719 if (!s)
262 goto error; 720 goto out;
263 asdl_seq_SET(stmts, k++, s); 721 asdl_seq_SET(stmts, k++, s);
264 } 722 }
265 } 723 }
266 } 724 }
267 return Module(stmts, arena); 725 res = Module(stmts, arena);
726 break;
268 case eval_input: { 727 case eval_input: {
269 expr_ty testlist_ast; 728 expr_ty testlist_ast;
270 729
271 /* XXX Why not comp_for here? */ 730 /* XXX Why not comp_for here? */
272 testlist_ast = ast_for_testlist(&c, CHILD(n, 0)); 731 testlist_ast = ast_for_testlist(&c, CHILD(n, 0));
273 if (!testlist_ast) 732 if (!testlist_ast)
274 goto error; 733 goto out;
275 return Expression(testlist_ast, arena); 734 res = Expression(testlist_ast, arena);
735 break;
276 } 736 }
277 case single_input: 737 case single_input:
278 if (TYPE(CHILD(n, 0)) == NEWLINE) { 738 if (TYPE(CHILD(n, 0)) == NEWLINE) {
279 stmts = asdl_seq_new(1, arena); 739 stmts = asdl_seq_new(1, arena);
280 if (!stmts) 740 if (!stmts)
281 goto error; 741 goto out;
282 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset, 742 asdl_seq_SET(stmts, 0, Pass(n->n_lineno, n->n_col_offset,
283 arena)); 743 arena));
284 if (!asdl_seq_GET(stmts, 0)) 744 if (!asdl_seq_GET(stmts, 0))
285 goto error; 745 goto out;
286 return Interactive(stmts, arena); 746 res = Interactive(stmts, arena);
287 } 747 }
288 else { 748 else {
289 n = CHILD(n, 0); 749 n = CHILD(n, 0);
290 num = num_stmts(n); 750 num = num_stmts(n);
291 stmts = asdl_seq_new(num, arena); 751 stmts = asdl_seq_new(num, arena);
292 if (!stmts) 752 if (!stmts)
293 goto error; 753 goto out;
294 if (num == 1) { 754 if (num == 1) {
295 s = ast_for_stmt(&c, n); 755 s = ast_for_stmt(&c, n);
296 if (!s) 756 if (!s)
297 goto error; 757 goto out;
298 asdl_seq_SET(stmts, 0, s); 758 asdl_seq_SET(stmts, 0, s);
299 } 759 }
300 else { 760 else {
301 /* Only a simple_stmt can contain multiple statements. */ 761 /* Only a simple_stmt can contain multiple statements. */
302 REQ(n, simple_stmt); 762 REQ(n, simple_stmt);
303 for (i = 0; i < NCH(n); i += 2) { 763 for (i = 0; i < NCH(n); i += 2) {
304 if (TYPE(CHILD(n, i)) == NEWLINE) 764 if (TYPE(CHILD(n, i)) == NEWLINE)
305 break; 765 break;
306 s = ast_for_stmt(&c, CHILD(n, i)); 766 s = ast_for_stmt(&c, CHILD(n, i));
307 if (!s) 767 if (!s)
308 goto error; 768 goto out;
309 asdl_seq_SET(stmts, i / 2, s); 769 asdl_seq_SET(stmts, i / 2, s);
310 } 770 }
311 } 771 }
312 772
313 return Interactive(stmts, arena); 773 res = Interactive(stmts, arena);
314 } 774 }
775 break;
315 default: 776 default:
316 PyErr_Format(PyExc_SystemError, 777 PyErr_Format(PyExc_SystemError,
317 "invalid node %d for PyAST_FromNodeObject", TYPE(n)); 778 "invalid node %d for PyAST_FromNode", TYPE(n));
318 goto error; 779 goto out;
319 } 780 }
320 error: 781 out:
321 ast_error_finish(filename); 782 if (c.c_normalize) {
322 return NULL; 783 Py_DECREF(c.c_normalize);
784 PyTuple_SET_ITEM(c.c_normalize_args, 1, NULL);
785 Py_DECREF(c.c_normalize_args);
786 }
787 return res;
323 } 788 }
324 789
325 mod_ty 790 mod_ty
326 PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename_str, 791 PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename_str,
327 PyArena *arena) 792 PyArena *arena)
328 { 793 {
329 mod_ty mod; 794 mod_ty mod;
330 PyObject *filename; 795 PyObject *filename;
331 filename = PyUnicode_DecodeFSDefault(filename_str); 796 filename = PyUnicode_DecodeFSDefault(filename_str);
332 if (filename == NULL) 797 if (filename == NULL)
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after
372 } 837 }
373 838
374 static const char* FORBIDDEN[] = { 839 static const char* FORBIDDEN[] = {
375 "None", 840 "None",
376 "True", 841 "True",
377 "False", 842 "False",
378 NULL, 843 NULL,
379 }; 844 };
380 845
381 static int 846 static int
382 forbidden_name(identifier name, const node *n, int full_checks) 847 forbidden_name(struct compiling *c, identifier name, const node *n, int full_che cks)
383 { 848 {
384 assert(PyUnicode_Check(name)); 849 assert(PyUnicode_Check(name));
385 if (PyUnicode_CompareWithASCIIString(name, "__debug__") == 0) { 850 if (PyUnicode_CompareWithASCIIString(name, "__debug__") == 0) {
386 ast_error(n, "assignment to keyword"); 851 ast_error(c, n, "assignment to keyword");
387 return 1; 852 return 1;
388 } 853 }
389 if (full_checks) { 854 if (full_checks) {
390 const char **p; 855 const char **p;
391 for (p = FORBIDDEN; *p; p++) { 856 for (p = FORBIDDEN; *p; p++) {
392 if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { 857 if (PyUnicode_CompareWithASCIIString(name, *p) == 0) {
393 ast_error(n, "assignment to keyword"); 858 ast_error(c, n, "assignment to keyword");
394 return 1; 859 return 1;
395 } 860 }
396 } 861 }
397 } 862 }
398 return 0; 863 return 0;
399 } 864 }
400 865
401 /* Set the context ctx for expr_ty e, recursively traversing e. 866 /* Set the context ctx for expr_ty e, recursively traversing e.
402 867
403 Only sets context for expr kinds that "can appear in assignment context" 868 Only sets context for expr kinds that "can appear in assignment context"
(...skipping 15 matching lines...) Expand all
419 a little more complex than necessary as a result. It also means 884 a little more complex than necessary as a result. It also means
420 that expressions in an augmented assignment have a Store context. 885 that expressions in an augmented assignment have a Store context.
421 Consider restructuring so that augmented assignment uses 886 Consider restructuring so that augmented assignment uses
422 set_context(), too. 887 set_context(), too.
423 */ 888 */
424 assert(ctx != AugStore && ctx != AugLoad); 889 assert(ctx != AugStore && ctx != AugLoad);
425 890
426 switch (e->kind) { 891 switch (e->kind) {
427 case Attribute_kind: 892 case Attribute_kind:
428 e->v.Attribute.ctx = ctx; 893 e->v.Attribute.ctx = ctx;
429 if (ctx == Store && forbidden_name(e->v.Attribute.attr, n, 1)) 894 if (ctx == Store && forbidden_name(c, e->v.Attribute.attr, n, 1))
430 return 0; 895 return 0;
431 break; 896 break;
432 case Subscript_kind: 897 case Subscript_kind:
433 e->v.Subscript.ctx = ctx; 898 e->v.Subscript.ctx = ctx;
434 break; 899 break;
435 case Starred_kind: 900 case Starred_kind:
436 e->v.Starred.ctx = ctx; 901 e->v.Starred.ctx = ctx;
437 if (!set_context(c, e->v.Starred.value, ctx, n)) 902 if (!set_context(c, e->v.Starred.value, ctx, n))
438 return 0; 903 return 0;
439 break; 904 break;
440 case Name_kind: 905 case Name_kind:
441 if (ctx == Store) { 906 if (ctx == Store) {
442 if (forbidden_name(e->v.Name.id, n, 1)) 907 if (forbidden_name(c, e->v.Name.id, n, 0))
443 return 0; /* forbidden_name() calls ast_error() */ 908 return 0; /* forbidden_name() calls ast_error() */
444 } 909 }
445 e->v.Name.ctx = ctx; 910 e->v.Name.ctx = ctx;
446 break; 911 break;
447 case List_kind: 912 case List_kind:
448 e->v.List.ctx = ctx; 913 e->v.List.ctx = ctx;
449 s = e->v.List.elts; 914 s = e->v.List.elts;
450 break; 915 break;
451 case Tuple_kind: 916 case Tuple_kind:
452 if (asdl_seq_LEN(e->v.Tuple.elts)) { 917 if (asdl_seq_LEN(e->v.Tuple.elts)) {
(...skipping 12 matching lines...) Expand all
465 break; 930 break;
466 case BoolOp_kind: 931 case BoolOp_kind:
467 case BinOp_kind: 932 case BinOp_kind:
468 case UnaryOp_kind: 933 case UnaryOp_kind:
469 expr_name = "operator"; 934 expr_name = "operator";
470 break; 935 break;
471 case GeneratorExp_kind: 936 case GeneratorExp_kind:
472 expr_name = "generator expression"; 937 expr_name = "generator expression";
473 break; 938 break;
474 case Yield_kind: 939 case Yield_kind:
940 case YieldFrom_kind:
475 expr_name = "yield expression"; 941 expr_name = "yield expression";
476 break; 942 break;
477 case ListComp_kind: 943 case ListComp_kind:
478 expr_name = "list comprehension"; 944 expr_name = "list comprehension";
479 break; 945 break;
480 case SetComp_kind: 946 case SetComp_kind:
481 expr_name = "set comprehension"; 947 expr_name = "set comprehension";
482 break; 948 break;
483 case DictComp_kind: 949 case DictComp_kind:
484 expr_name = "dict comprehension"; 950 expr_name = "dict comprehension";
485 break; 951 break;
486 case Dict_kind: 952 case Dict_kind:
487 case Set_kind: 953 case Set_kind:
488 case Num_kind: 954 case Num_kind:
489 case Str_kind: 955 case Str_kind:
490 case Bytes_kind: 956 case Bytes_kind:
491 expr_name = "literal"; 957 expr_name = "literal";
492 break; 958 break;
959 case NameConstant_kind:
960 expr_name = "keyword";
961 break;
493 case Ellipsis_kind: 962 case Ellipsis_kind:
494 expr_name = "Ellipsis"; 963 expr_name = "Ellipsis";
495 break; 964 break;
496 case Compare_kind: 965 case Compare_kind:
497 expr_name = "comparison"; 966 expr_name = "comparison";
498 break; 967 break;
499 case IfExp_kind: 968 case IfExp_kind:
500 expr_name = "conditional expression"; 969 expr_name = "conditional expression";
501 break; 970 break;
502 default: 971 default:
503 PyErr_Format(PyExc_SystemError, 972 PyErr_Format(PyExc_SystemError,
504 "unexpected expression in assignment %d (line %d)", 973 "unexpected expression in assignment %d (line %d)",
505 e->kind, e->lineno); 974 e->kind, e->lineno);
506 return 0; 975 return 0;
507 } 976 }
508 /* Check for error string set by switch */ 977 /* Check for error string set by switch */
509 if (expr_name) { 978 if (expr_name) {
510 char buf[300]; 979 char buf[300];
511 PyOS_snprintf(buf, sizeof(buf), 980 PyOS_snprintf(buf, sizeof(buf),
512 "can't %s %s", 981 "can't %s %s",
513 ctx == Store ? "assign to" : "delete", 982 ctx == Store ? "assign to" : "delete",
514 expr_name); 983 expr_name);
515 return ast_error(n, buf); 984 return ast_error(c, n, buf);
516 } 985 }
517 986
518 /* If the LHS is a list or tuple, we need to set the assignment 987 /* If the LHS is a list or tuple, we need to set the assignment
519 context for all the contained elements. 988 context for all the contained elements.
520 */ 989 */
521 if (s) { 990 if (s) {
522 int i; 991 int i;
523 992
524 for (i = 0; i < asdl_seq_LEN(s); i++) { 993 for (i = 0; i < asdl_seq_LEN(s); i++) {
525 if (!set_context(c, (expr_ty)asdl_seq_GET(s, i), ctx, n)) 994 if (!set_context(c, (expr_ty)asdl_seq_GET(s, i), ctx, n))
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after
642 if (!expression) 1111 if (!expression)
643 return NULL; 1112 return NULL;
644 1113
645 assert(i / 2 < seq->size); 1114 assert(i / 2 < seq->size);
646 asdl_seq_SET(seq, i / 2, expression); 1115 asdl_seq_SET(seq, i / 2, expression);
647 } 1116 }
648 return seq; 1117 return seq;
649 } 1118 }
650 1119
651 static arg_ty 1120 static arg_ty
652 compiler_arg(struct compiling *c, const node *n) 1121 ast_for_arg(struct compiling *c, const node *n)
653 { 1122 {
654 identifier name; 1123 identifier name;
655 expr_ty annotation = NULL; 1124 expr_ty annotation = NULL;
656 node *ch; 1125 node *ch;
1126 arg_ty tmp;
657 1127
658 assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef); 1128 assert(TYPE(n) == tfpdef || TYPE(n) == vfpdef);
659 ch = CHILD(n, 0); 1129 ch = CHILD(n, 0);
660 name = NEW_IDENTIFIER(ch); 1130 name = NEW_IDENTIFIER(ch);
661 if (!name) 1131 if (!name)
662 return NULL; 1132 return NULL;
663 if (forbidden_name(name, ch, 0)) 1133 if (forbidden_name(c, name, ch, 0))
664 return NULL; 1134 return NULL;
665 1135
666 if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) { 1136 if (NCH(n) == 3 && TYPE(CHILD(n, 1)) == COLON) {
667 annotation = ast_for_expr(c, CHILD(n, 2)); 1137 annotation = ast_for_expr(c, CHILD(n, 2));
668 if (!annotation) 1138 if (!annotation)
669 return NULL; 1139 return NULL;
670 } 1140 }
671 1141
672 return arg(name, annotation, c->c_arena); 1142 tmp = arg(name, annotation, c->c_arena);
673 #if 0 1143 if (!tmp)
674 result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena); 1144 return NULL;
675 if (!set_context(c, result, Store, n)) 1145
676 return NULL; 1146 tmp->lineno = LINENO(n);
677 return result; 1147 tmp->col_offset = n->n_col_offset;
678 #endif 1148 return tmp;
679 } 1149 }
680 1150
681 /* returns -1 if failed to handle keyword only arguments 1151 /* returns -1 if failed to handle keyword only arguments
682 returns new position to keep processing if successful 1152 returns new position to keep processing if successful
683 (',' tfpdef ['=' test])* 1153 (',' tfpdef ['=' test])*
684 ^^^ 1154 ^^^
685 start pointing here 1155 start pointing here
686 */ 1156 */
687 static int 1157 static int
688 handle_keywordonly_args(struct compiling *c, const node *n, int start, 1158 handle_keywordonly_args(struct compiling *c, const node *n, int start,
689 asdl_seq *kwonlyargs, asdl_seq *kwdefaults) 1159 asdl_seq *kwonlyargs, asdl_seq *kwdefaults)
690 { 1160 {
691 PyObject *argname; 1161 PyObject *argname;
692 node *ch; 1162 node *ch;
693 expr_ty expression, annotation; 1163 expr_ty expression, annotation;
694 arg_ty arg; 1164 arg_ty arg;
695 int i = start; 1165 int i = start;
696 int j = 0; /* index for kwdefaults and kwonlyargs */ 1166 int j = 0; /* index for kwdefaults and kwonlyargs */
697 1167
698 if (kwonlyargs == NULL) { 1168 if (kwonlyargs == NULL) {
699 ast_error(CHILD(n, start), "named arguments must follow bare *"); 1169 ast_error(c, CHILD(n, start), "named arguments must follow bare *");
700 return -1; 1170 return -1;
701 } 1171 }
702 assert(kwdefaults != NULL); 1172 assert(kwdefaults != NULL);
703 while (i < NCH(n)) { 1173 while (i < NCH(n)) {
704 ch = CHILD(n, i); 1174 ch = CHILD(n, i);
705 switch (TYPE(ch)) { 1175 switch (TYPE(ch)) {
706 case vfpdef: 1176 case vfpdef:
707 case tfpdef: 1177 case tfpdef:
708 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { 1178 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
709 expression = ast_for_expr(c, CHILD(n, i + 2)); 1179 expression = ast_for_expr(c, CHILD(n, i + 2));
(...skipping 11 matching lines...) Expand all
721 if (!annotation) 1191 if (!annotation)
722 goto error; 1192 goto error;
723 } 1193 }
724 else { 1194 else {
725 annotation = NULL; 1195 annotation = NULL;
726 } 1196 }
727 ch = CHILD(ch, 0); 1197 ch = CHILD(ch, 0);
728 argname = NEW_IDENTIFIER(ch); 1198 argname = NEW_IDENTIFIER(ch);
729 if (!argname) 1199 if (!argname)
730 goto error; 1200 goto error;
731 if (forbidden_name(argname, ch, 0)) 1201 if (forbidden_name(c, argname, ch, 0))
732 goto error; 1202 goto error;
733 arg = arg(argname, annotation, c->c_arena); 1203 arg = arg(argname, annotation, c->c_arena);
734 if (!arg) 1204 if (!arg)
735 goto error; 1205 goto error;
736 asdl_seq_SET(kwonlyargs, j++, arg); 1206 asdl_seq_SET(kwonlyargs, j++, arg);
737 i += 2; /* the name and the comma */ 1207 i += 2; /* the name and the comma */
738 break; 1208 break;
739 case DOUBLESTAR: 1209 case DOUBLESTAR:
740 return i; 1210 return i;
741 default: 1211 default:
742 ast_error(ch, "unexpected node"); 1212 ast_error(c, ch, "unexpected node");
743 goto error; 1213 goto error;
744 } 1214 }
745 } 1215 }
746 return i; 1216 return i;
747 error: 1217 error:
748 return -1; 1218 return -1;
749 } 1219 }
750 1220
751 /* Create AST for argument list. */ 1221 /* Create AST for argument list. */
752 1222
(...skipping 11 matching lines...) Expand all
764 tfpdef: NAME [':' test] 1234 tfpdef: NAME [':' test]
765 varargslist: ((vfpdef ['=' test] ',')* 1235 varargslist: ((vfpdef ['=' test] ',')*
766 ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] 1236 ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef]
767 | '**' vfpdef) 1237 | '**' vfpdef)
768 | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) 1238 | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
769 vfpdef: NAME 1239 vfpdef: NAME
770 */ 1240 */
771 int i, j, k, nposargs = 0, nkwonlyargs = 0; 1241 int i, j, k, nposargs = 0, nkwonlyargs = 0;
772 int nposdefaults = 0, found_default = 0; 1242 int nposdefaults = 0, found_default = 0;
773 asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults; 1243 asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults;
774 identifier vararg = NULL, kwarg = NULL; 1244 arg_ty vararg = NULL, kwarg = NULL;
775 arg_ty arg; 1245 arg_ty arg;
776 expr_ty varargannotation = NULL, kwargannotation = NULL;
777 node *ch; 1246 node *ch;
778 1247
779 if (TYPE(n) == parameters) { 1248 if (TYPE(n) == parameters) {
780 if (NCH(n) == 2) /* () as argument list */ 1249 if (NCH(n) == 2) /* () as argument list */
781 return arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1250 return arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena);
782 NULL, c->c_arena);
783 n = CHILD(n, 1); 1251 n = CHILD(n, 1);
784 } 1252 }
785 assert(TYPE(n) == typedargslist || TYPE(n) == varargslist); 1253 assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);
786 1254
787 /* First count the number of positional args & defaults. The 1255 /* First count the number of positional args & defaults. The
788 variable i is the loop index for this for loop and the next. 1256 variable i is the loop index for this for loop and the next.
789 The next loop picks up where the first leaves off. 1257 The next loop picks up where the first leaves off.
790 */ 1258 */
791 for (i = 0; i < NCH(n); i++) { 1259 for (i = 0; i < NCH(n); i++) {
792 ch = CHILD(n, i); 1260 ch = CHILD(n, i);
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
824 return NULL; 1292 return NULL;
825 /* The length of kwonlyargs and kwdefaults are same 1293 /* The length of kwonlyargs and kwdefaults are same
826 since we set NULL as default for keyword only argument w/o default 1294 since we set NULL as default for keyword only argument w/o default
827 - we have sequence data structure, but no dictionary */ 1295 - we have sequence data structure, but no dictionary */
828 kwdefaults = (nkwonlyargs ? 1296 kwdefaults = (nkwonlyargs ?
829 asdl_seq_new(nkwonlyargs, c->c_arena) : NULL); 1297 asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
830 if (!kwdefaults && nkwonlyargs) 1298 if (!kwdefaults && nkwonlyargs)
831 return NULL; 1299 return NULL;
832 1300
833 if (nposargs + nkwonlyargs > 255) { 1301 if (nposargs + nkwonlyargs > 255) {
834 ast_error(n, "more than 255 arguments"); 1302 ast_error(c, n, "more than 255 arguments");
835 return NULL; 1303 return NULL;
836 } 1304 }
837 1305
838 /* tfpdef: NAME [':' test] 1306 /* tfpdef: NAME [':' test]
839 vfpdef: NAME 1307 vfpdef: NAME
840 */ 1308 */
841 i = 0; 1309 i = 0;
842 j = 0; /* index for defaults */ 1310 j = 0; /* index for defaults */
843 k = 0; /* index for args */ 1311 k = 0; /* index for args */
844 while (i < NCH(n)) { 1312 while (i < NCH(n)) {
845 ch = CHILD(n, i); 1313 ch = CHILD(n, i);
846 switch (TYPE(ch)) { 1314 switch (TYPE(ch)) {
847 case tfpdef: 1315 case tfpdef:
848 case vfpdef: 1316 case vfpdef:
849 /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is 1317 /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
850 anything other than EQUAL or a comma? */ 1318 anything other than EQUAL or a comma? */
851 /* XXX Should NCH(n) check be made a separate check? */ 1319 /* XXX Should NCH(n) check be made a separate check? */
852 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) { 1320 if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
853 expr_ty expression = ast_for_expr(c, CHILD(n, i + 2)); 1321 expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
854 if (!expression) 1322 if (!expression)
855 return NULL; 1323 return NULL;
856 assert(posdefaults != NULL); 1324 assert(posdefaults != NULL);
857 asdl_seq_SET(posdefaults, j++, expression); 1325 asdl_seq_SET(posdefaults, j++, expression);
858 i += 2; 1326 i += 2;
859 found_default = 1; 1327 found_default = 1;
860 } 1328 }
861 else if (found_default) { 1329 else if (found_default) {
862 ast_error(n, 1330 ast_error(c, n,
863 "non-default argument follows default argument"); 1331 "non-default argument follows default argument");
864 return NULL; 1332 return NULL;
865 } 1333 }
866 arg = compiler_arg(c, ch); 1334 arg = ast_for_arg(c, ch);
867 if (!arg) 1335 if (!arg)
868 return NULL; 1336 return NULL;
869 asdl_seq_SET(posargs, k++, arg); 1337 asdl_seq_SET(posargs, k++, arg);
870 i += 2; /* the name and the comma */ 1338 i += 2; /* the name and the comma */
871 break; 1339 break;
872 case STAR: 1340 case STAR:
873 if (i+1 >= NCH(n)) { 1341 if (i+1 >= NCH(n)) {
874 ast_error(CHILD(n, i), 1342 ast_error(c, CHILD(n, i),
875 "named arguments must follow bare *"); 1343 "named arguments must follow bare *");
876 return NULL; 1344 return NULL;
877 } 1345 }
878 ch = CHILD(n, i+1); /* tfpdef or COMMA */ 1346 ch = CHILD(n, i+1); /* tfpdef or COMMA */
879 if (TYPE(ch) == COMMA) { 1347 if (TYPE(ch) == COMMA) {
880 int res = 0; 1348 int res = 0;
881 i += 2; /* now follows keyword only arguments */ 1349 i += 2; /* now follows keyword only arguments */
882 res = handle_keywordonly_args(c, n, i, 1350 res = handle_keywordonly_args(c, n, i,
883 kwonlyargs, kwdefaults); 1351 kwonlyargs, kwdefaults);
884 if (res == -1) return NULL; 1352 if (res == -1) return NULL;
885 i = res; /* res has new position to process */ 1353 i = res; /* res has new position to process */
886 } 1354 }
887 else { 1355 else {
888 vararg = NEW_IDENTIFIER(CHILD(ch, 0)); 1356 vararg = ast_for_arg(c, ch);
889 if (!vararg) 1357 if (!vararg)
890 return NULL; 1358 return NULL;
891 if (forbidden_name(vararg, CHILD(ch, 0), 0)) 1359
892 return NULL;
893 if (NCH(ch) > 1) {
894 /* there is an annotation on the vararg */
895 varargannotation = ast_for_expr(c, CHILD(ch, 2));
896 if (!varargannotation)
897 return NULL;
898 }
899 i += 3; 1360 i += 3;
900 if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef 1361 if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef
901 || TYPE(CHILD(n, i)) == vfpdef)) { 1362 || TYPE(CHILD(n, i)) == vfpdef)) {
902 int res = 0; 1363 int res = 0;
903 res = handle_keywordonly_args(c, n, i, 1364 res = handle_keywordonly_args(c, n, i,
904 kwonlyargs, kwdefaults); 1365 kwonlyargs, kwdefaults);
905 if (res == -1) return NULL; 1366 if (res == -1) return NULL;
906 i = res; /* res has new position to process */ 1367 i = res; /* res has new position to process */
907 } 1368 }
908 } 1369 }
909 break; 1370 break;
910 case DOUBLESTAR: 1371 case DOUBLESTAR:
911 ch = CHILD(n, i+1); /* tfpdef */ 1372 ch = CHILD(n, i+1); /* tfpdef */
912 assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef); 1373 assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
913 kwarg = NEW_IDENTIFIER(CHILD(ch, 0)); 1374 kwarg = ast_for_arg(c, ch);
914 if (!kwarg) 1375 if (!kwarg)
915 return NULL;
916 if (NCH(ch) > 1) {
917 /* there is an annotation on the kwarg */
918 kwargannotation = ast_for_expr(c, CHILD(ch, 2));
919 if (!kwargannotation)
920 return NULL;
921 }
922 if (forbidden_name(kwarg, CHILD(ch, 0), 0))
923 return NULL; 1376 return NULL;
924 i += 3; 1377 i += 3;
925 break; 1378 break;
926 default: 1379 default:
927 PyErr_Format(PyExc_SystemError, 1380 PyErr_Format(PyExc_SystemError,
928 "unexpected node in varargslist: %d @ %d", 1381 "unexpected node in varargslist: %d @ %d",
929 TYPE(ch), i); 1382 TYPE(ch), i);
930 return NULL; 1383 return NULL;
931 } 1384 }
932 } 1385 }
933 return arguments(posargs, vararg, varargannotation, kwonlyargs, kwarg, 1386 return arguments(posargs, vararg, kwonlyargs, kwdefaults, kwarg, posdefaults , c->c_arena);
934 kwargannotation, posdefaults, kwdefaults, c->c_arena);
935 } 1387 }
936 1388
937 static expr_ty 1389 static expr_ty
938 ast_for_dotted_name(struct compiling *c, const node *n) 1390 ast_for_dotted_name(struct compiling *c, const node *n)
939 { 1391 {
940 expr_ty e; 1392 expr_ty e;
941 identifier id; 1393 identifier id;
942 int lineno, col_offset; 1394 int lineno, col_offset;
943 int i; 1395 int i;
944 1396
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
1031 arguments_ty args; 1483 arguments_ty args;
1032 asdl_seq *body; 1484 asdl_seq *body;
1033 expr_ty returns = NULL; 1485 expr_ty returns = NULL;
1034 int name_i = 1; 1486 int name_i = 1;
1035 1487
1036 REQ(n, funcdef); 1488 REQ(n, funcdef);
1037 1489
1038 name = NEW_IDENTIFIER(CHILD(n, name_i)); 1490 name = NEW_IDENTIFIER(CHILD(n, name_i));
1039 if (!name) 1491 if (!name)
1040 return NULL; 1492 return NULL;
1041 if (forbidden_name(name, CHILD(n, name_i), 0)) 1493 if (forbidden_name(c, name, CHILD(n, name_i), 0))
1042 return NULL; 1494 return NULL;
1043 args = ast_for_arguments(c, CHILD(n, name_i + 1)); 1495 args = ast_for_arguments(c, CHILD(n, name_i + 1));
1044 if (!args) 1496 if (!args)
1045 return NULL; 1497 return NULL;
1046 if (TYPE(CHILD(n, name_i+2)) == RARROW) { 1498 if (TYPE(CHILD(n, name_i+2)) == RARROW) {
1047 returns = ast_for_expr(c, CHILD(n, name_i + 3)); 1499 returns = ast_for_expr(c, CHILD(n, name_i + 3));
1048 if (!returns) 1500 if (!returns)
1049 return NULL; 1501 return NULL;
1050 name_i += 2; 1502 name_i += 2;
1051 } 1503 }
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
1089 1541
1090 static expr_ty 1542 static expr_ty
1091 ast_for_lambdef(struct compiling *c, const node *n) 1543 ast_for_lambdef(struct compiling *c, const node *n)
1092 { 1544 {
1093 /* lambdef: 'lambda' [varargslist] ':' test 1545 /* lambdef: 'lambda' [varargslist] ':' test
1094 lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */ 1546 lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */
1095 arguments_ty args; 1547 arguments_ty args;
1096 expr_ty expression; 1548 expr_ty expression;
1097 1549
1098 if (NCH(n) == 3) { 1550 if (NCH(n) == 3) {
1099 args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1551 args = arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena);
1100 NULL, c->c_arena);
1101 if (!args) 1552 if (!args)
1102 return NULL; 1553 return NULL;
1103 expression = ast_for_expr(c, CHILD(n, 2)); 1554 expression = ast_for_expr(c, CHILD(n, 2));
1104 if (!expression) 1555 if (!expression)
1105 return NULL; 1556 return NULL;
1106 } 1557 }
1107 else { 1558 else {
1108 args = ast_for_arguments(c, CHILD(n, 1)); 1559 args = ast_for_arguments(c, CHILD(n, 1));
1109 if (!args) 1560 if (!args)
1110 return NULL; 1561 return NULL;
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after
1353 { 1804 {
1354 /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' 1805 /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']'
1355 | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+ 1806 | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+
1356 | '...' | 'None' | 'True' | 'False' 1807 | '...' | 'None' | 'True' | 'False'
1357 */ 1808 */
1358 node *ch = CHILD(n, 0); 1809 node *ch = CHILD(n, 0);
1359 int bytesmode = 0; 1810 int bytesmode = 0;
1360 1811
1361 switch (TYPE(ch)) { 1812 switch (TYPE(ch)) {
1362 case NAME: { 1813 case NAME: {
1363 /* All names start in Load context, but may later be 1814 PyObject *name;
1364 changed. */ 1815 const char *s = STR(ch);
1365 PyObject *name = NEW_IDENTIFIER(ch); 1816 size_t len = strlen(s);
1817 if (len >= 4 && len <= 5) {
1818 if (!strcmp(s, "None"))
1819 return NameConstant(Py_None, LINENO(n), n->n_col_offset, c->c_ar ena);
1820 if (!strcmp(s, "True"))
1821 return NameConstant(Py_True, LINENO(n), n->n_col_offset, c->c_ar ena);
1822 if (!strcmp(s, "False"))
1823 return NameConstant(Py_False, LINENO(n), n->n_col_offset, c->c_a rena);
1824 }
1825 name = new_identifier(s, c);
1366 if (!name) 1826 if (!name)
1367 return NULL; 1827 return NULL;
1828 /* All names start in Load context, but may later be changed. */
1368 return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); 1829 return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
1369 } 1830 }
1370 case STRING: { 1831 case STRING: {
1371 PyObject *str = parsestrplus(c, n, &bytesmode); 1832 PyObject *str = parsestrplus(c, n, &bytesmode);
1372 if (!str) { 1833 if (!str) {
1373 if (PyErr_ExceptionMatches(PyExc_UnicodeError)) { 1834 const char *errtype = NULL;
1835 if (PyErr_ExceptionMatches(PyExc_UnicodeError))
1836 errtype = "unicode error";
1837 else if (PyErr_ExceptionMatches(PyExc_ValueError))
1838 errtype = "value error";
1839 if (errtype) {
1840 char buf[128];
1374 PyObject *type, *value, *tback, *errstr; 1841 PyObject *type, *value, *tback, *errstr;
1375 PyErr_Fetch(&type, &value, &tback); 1842 PyErr_Fetch(&type, &value, &tback);
1376 errstr = PyObject_Str(value); 1843 errstr = PyObject_Str(value);
1377 if (errstr) { 1844 if (errstr) {
1378 char *s = ""; 1845 char *s = _PyUnicode_AsString(errstr);
1379 char buf[128]; 1846 PyOS_snprintf(buf, sizeof(buf), "(%s) %s", errtype, s);
1380 s = _PyUnicode_AsString(errstr);
1381 PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s);
1382 ast_error(n, buf);
1383 Py_DECREF(errstr); 1847 Py_DECREF(errstr);
1384 } else { 1848 } else {
1385 ast_error(n, "(unicode error) unknown error"); 1849 PyOS_snprintf(buf, sizeof(buf), "(%s) unknown error", errtyp e);
1386 } 1850 }
1851 ast_error(c, n, buf);
1387 Py_DECREF(type); 1852 Py_DECREF(type);
1388 Py_DECREF(value); 1853 Py_XDECREF(value);
1389 Py_XDECREF(tback); 1854 Py_XDECREF(tback);
1390 } 1855 }
1391 return NULL; 1856 return NULL;
1392 } 1857 }
1393 PyArena_AddPyObject(c->c_arena, str); 1858 if (PyArena_AddPyObject(c->c_arena, str) < 0) {
1859 Py_DECREF(str);
1860 return NULL;
1861 }
1394 if (bytesmode) 1862 if (bytesmode)
1395 return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); 1863 return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena);
1396 else 1864 else
1397 return Str(str, LINENO(n), n->n_col_offset, c->c_arena); 1865 return Str(str, LINENO(n), n->n_col_offset, c->c_arena);
1398 } 1866 }
1399 case NUMBER: { 1867 case NUMBER: {
1400 PyObject *pynum = parsenumber(c, STR(ch)); 1868 PyObject *pynum = parsenumber(c, STR(ch));
1401 if (!pynum) 1869 if (!pynum)
1402 return NULL; 1870 return NULL;
1403 1871
1404 PyArena_AddPyObject(c->c_arena, pynum); 1872 if (PyArena_AddPyObject(c->c_arena, pynum) < 0) {
1873 Py_DECREF(pynum);
1874 return NULL;
1875 }
1405 return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); 1876 return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
1406 } 1877 }
1407 case ELLIPSIS: /* Ellipsis */ 1878 case ELLIPSIS: /* Ellipsis */
1408 return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena); 1879 return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena);
1409 case LPAR: /* some parenthesized expressions */ 1880 case LPAR: /* some parenthesized expressions */
1410 ch = CHILD(n, 1); 1881 ch = CHILD(n, 1);
1411 1882
1412 if (TYPE(ch) == RPAR) 1883 if (TYPE(ch) == RPAR)
1413 return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); 1884 return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);
1414 1885
(...skipping 208 matching lines...) Expand 10 before | Expand all | Expand 10 after
1623 { 2094 {
1624 /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 2095 /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
1625 subscriptlist: subscript (',' subscript)* [','] 2096 subscriptlist: subscript (',' subscript)* [',']
1626 subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] 2097 subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
1627 */ 2098 */
1628 REQ(n, trailer); 2099 REQ(n, trailer);
1629 if (TYPE(CHILD(n, 0)) == LPAR) { 2100 if (TYPE(CHILD(n, 0)) == LPAR) {
1630 if (NCH(n) == 2) 2101 if (NCH(n) == 2)
1631 return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n), 2102 return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n),
1632 n->n_col_offset, c->c_arena); 2103 n->n_col_offset, c->c_arena);
1633 else 2104 else {
1634 return ast_for_call(c, CHILD(n, 1), left_expr); 2105 expr_ty tmp = ast_for_call(c, CHILD(n, 1), left_expr);
2106 if (!tmp)
2107 return NULL;
2108
2109 tmp->lineno = LINENO(n);
2110 tmp->col_offset = n->n_col_offset;
2111 return tmp;
2112 }
1635 } 2113 }
1636 else if (TYPE(CHILD(n, 0)) == DOT ) { 2114 else if (TYPE(CHILD(n, 0)) == DOT ) {
1637 PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1)); 2115 PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
1638 if (!attr_id) 2116 if (!attr_id)
1639 return NULL; 2117 return NULL;
1640 return Attribute(left_expr, attr_id, Load, 2118 return Attribute(left_expr, attr_id, Load,
1641 LINENO(n), n->n_col_offset, c->c_arena); 2119 LINENO(CHILD(n, 1)), CHILD(n, 1)->n_col_offset, c->c_ar ena);
1642 } 2120 }
1643 else { 2121 else {
1644 REQ(CHILD(n, 0), LSQB); 2122 REQ(CHILD(n, 0), LSQB);
1645 REQ(CHILD(n, 2), RSQB); 2123 REQ(CHILD(n, 2), RSQB);
1646 n = CHILD(n, 1); 2124 n = CHILD(n, 1);
1647 if (NCH(n) == 1) { 2125 if (NCH(n) == 1) {
1648 slice_ty slc = ast_for_slice(c, CHILD(n, 0)); 2126 slice_ty slc = ast_for_slice(c, CHILD(n, 0));
1649 if (!slc) 2127 if (!slc)
1650 return NULL; 2128 return NULL;
1651 return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, 2129 return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset,
(...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after
1732 return NULL; 2210 return NULL;
1733 if (NCH(n) == 1) 2211 if (NCH(n) == 1)
1734 return e; 2212 return e;
1735 for (i = 1; i < NCH(n); i++) { 2213 for (i = 1; i < NCH(n); i++) {
1736 node *ch = CHILD(n, i); 2214 node *ch = CHILD(n, i);
1737 if (TYPE(ch) != trailer) 2215 if (TYPE(ch) != trailer)
1738 break; 2216 break;
1739 tmp = ast_for_trailer(c, ch, e); 2217 tmp = ast_for_trailer(c, ch, e);
1740 if (!tmp) 2218 if (!tmp)
1741 return NULL; 2219 return NULL;
1742 tmp->lineno = e->lineno;
1743 tmp->col_offset = e->col_offset;
1744 e = tmp; 2220 e = tmp;
1745 } 2221 }
1746 if (TYPE(CHILD(n, NCH(n) - 1)) == factor) { 2222 if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
1747 expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1)); 2223 expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
1748 if (!f) 2224 if (!f)
1749 return NULL; 2225 return NULL;
1750 tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena); 2226 tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
1751 if (!tmp) 2227 if (!tmp)
1752 return NULL; 2228 return NULL;
1753 e = tmp; 2229 e = tmp;
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after
1892 case and_expr: 2368 case and_expr:
1893 case shift_expr: 2369 case shift_expr:
1894 case arith_expr: 2370 case arith_expr:
1895 case term: 2371 case term:
1896 if (NCH(n) == 1) { 2372 if (NCH(n) == 1) {
1897 n = CHILD(n, 0); 2373 n = CHILD(n, 0);
1898 goto loop; 2374 goto loop;
1899 } 2375 }
1900 return ast_for_binop(c, n); 2376 return ast_for_binop(c, n);
1901 case yield_expr: { 2377 case yield_expr: {
2378 node *an = NULL;
2379 node *en = NULL;
2380 int is_from = 0;
1902 expr_ty exp = NULL; 2381 expr_ty exp = NULL;
1903 if (NCH(n) == 2) { 2382 if (NCH(n) > 1)
1904 exp = ast_for_testlist(c, CHILD(n, 1)); 2383 an = CHILD(n, 1); /* yield_arg */
2384 if (an) {
2385 en = CHILD(an, NCH(an) - 1);
2386 if (NCH(an) == 2) {
2387 is_from = 1;
2388 exp = ast_for_expr(c, en);
2389 }
2390 else
2391 exp = ast_for_testlist(c, en);
1905 if (!exp) 2392 if (!exp)
1906 return NULL; 2393 return NULL;
1907 } 2394 }
2395 if (is_from)
2396 return YieldFrom(exp, LINENO(n), n->n_col_offset, c->c_arena);
1908 return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); 2397 return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
1909 } 2398 }
1910 case factor: 2399 case factor:
1911 if (NCH(n) == 1) { 2400 if (NCH(n) == 1) {
1912 n = CHILD(n, 0); 2401 n = CHILD(n, 0);
1913 goto loop; 2402 goto loop;
1914 } 2403 }
1915 return ast_for_factor(c, n); 2404 return ast_for_factor(c, n);
1916 case power: 2405 case power:
1917 return ast_for_power(c, n); 2406 return ast_for_power(c, n);
1918 default: 2407 default:
1919 PyErr_Format(PyExc_SystemError, "unhandled expr: %d", TYPE(n)); 2408 PyErr_Format(PyExc_SystemError, "unhandled expr: %d", TYPE(n));
1920 return NULL; 2409 return NULL;
1921 } 2410 }
1922 /* should never get here unless if error is set */ 2411 /* should never get here unless if error is set */
1923 return NULL; 2412 return NULL;
1924 } 2413 }
1925 2414
1926 static expr_ty 2415 static expr_ty
1927 ast_for_call(struct compiling *c, const node *n, expr_ty func) 2416 ast_for_call(struct compiling *c, const node *n, expr_ty func)
1928 { 2417 {
1929 /* 2418 /*
1930 arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] 2419 arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
1931 | '**' test) 2420 | '**' test)
1932 argument: [test '='] test [comp_for] # Really [keyword '='] test 2421 argument: [test '='] (test) [comp_for] # Really [keyword '='] test
1933 */ 2422 */
1934 2423
1935 int i, nargs, nkeywords, ngens; 2424 int i, nargs, nkeywords, ngens;
1936 asdl_seq *args; 2425 asdl_seq *args;
1937 asdl_seq *keywords; 2426 asdl_seq *keywords;
1938 expr_ty vararg = NULL, kwarg = NULL; 2427 expr_ty vararg = NULL, kwarg = NULL;
1939 2428
1940 REQ(n, arglist); 2429 REQ(n, arglist);
1941 2430
1942 nargs = 0; 2431 nargs = 0;
1943 nkeywords = 0; 2432 nkeywords = 0;
1944 ngens = 0; 2433 ngens = 0;
1945 for (i = 0; i < NCH(n); i++) { 2434 for (i = 0; i < NCH(n); i++) {
1946 node *ch = CHILD(n, i); 2435 node *ch = CHILD(n, i);
1947 if (TYPE(ch) == argument) { 2436 if (TYPE(ch) == argument) {
1948 if (NCH(ch) == 1) 2437 if (NCH(ch) == 1)
1949 nargs++; 2438 nargs++;
1950 else if (TYPE(CHILD(ch, 1)) == comp_for) 2439 else if (TYPE(CHILD(ch, 1)) == comp_for)
1951 ngens++; 2440 ngens++;
1952 else 2441 else
1953 nkeywords++; 2442 nkeywords++;
1954 } 2443 }
1955 } 2444 }
1956 if (ngens > 1 || (ngens && (nargs || nkeywords))) { 2445 if (ngens > 1 || (ngens && (nargs || nkeywords))) {
1957 ast_error(n, "Generator expression must be parenthesized " 2446 ast_error(c, n, "Generator expression must be parenthesized "
1958 "if not sole argument"); 2447 "if not sole argument");
1959 return NULL; 2448 return NULL;
1960 } 2449 }
1961 2450
1962 if (nargs + nkeywords + ngens > 255) { 2451 if (nargs + nkeywords + ngens > 255) {
1963 ast_error(n, "more than 255 arguments"); 2452 ast_error(c, n, "more than 255 arguments");
1964 return NULL; 2453 return NULL;
1965 } 2454 }
1966 2455
1967 args = asdl_seq_new(nargs + ngens, c->c_arena); 2456 args = asdl_seq_new(nargs + ngens, c->c_arena);
1968 if (!args) 2457 if (!args)
1969 return NULL; 2458 return NULL;
1970 keywords = asdl_seq_new(nkeywords, c->c_arena); 2459 keywords = asdl_seq_new(nkeywords, c->c_arena);
1971 if (!keywords) 2460 if (!keywords)
1972 return NULL; 2461 return NULL;
1973 nargs = 0; 2462 nargs = 0;
1974 nkeywords = 0; 2463 nkeywords = 0;
1975 for (i = 0; i < NCH(n); i++) { 2464 for (i = 0; i < NCH(n); i++) {
1976 node *ch = CHILD(n, i); 2465 node *ch = CHILD(n, i);
1977 if (TYPE(ch) == argument) { 2466 if (TYPE(ch) == argument) {
1978 expr_ty e; 2467 expr_ty e;
1979 if (NCH(ch) == 1) { 2468 if (NCH(ch) == 1) {
1980 if (nkeywords) { 2469 if (nkeywords) {
1981 ast_error(CHILD(ch, 0), 2470 ast_error(c, CHILD(ch, 0),
1982 "non-keyword arg after keyword arg"); 2471 "non-keyword arg after keyword arg");
1983 return NULL; 2472 return NULL;
1984 } 2473 }
1985 if (vararg) { 2474 if (vararg) {
1986 ast_error(CHILD(ch, 0), 2475 ast_error(c, CHILD(ch, 0),
1987 "only named arguments may follow *expression"); 2476 "only named arguments may follow *expression");
1988 return NULL; 2477 return NULL;
1989 } 2478 }
1990 e = ast_for_expr(c, CHILD(ch, 0)); 2479 e = ast_for_expr(c, CHILD(ch, 0));
1991 if (!e) 2480 if (!e)
1992 return NULL; 2481 return NULL;
1993 asdl_seq_SET(args, nargs++, e); 2482 asdl_seq_SET(args, nargs++, e);
1994 } 2483 }
1995 else if (TYPE(CHILD(ch, 1)) == comp_for) { 2484 else if (TYPE(CHILD(ch, 1)) == comp_for) {
1996 e = ast_for_genexp(c, ch); 2485 e = ast_for_genexp(c, ch);
1997 if (!e) 2486 if (!e)
1998 return NULL; 2487 return NULL;
1999 asdl_seq_SET(args, nargs++, e); 2488 asdl_seq_SET(args, nargs++, e);
2000 } 2489 }
2001 else { 2490 else {
2002 keyword_ty kw; 2491 keyword_ty kw;
2003 identifier key, tmp; 2492 identifier key, tmp;
2004 int k; 2493 int k;
2005 2494
2006 /* CHILD(ch, 0) is test, but must be an identifier? */ 2495 /* CHILD(ch, 0) is test, but must be an identifier? */
2007 e = ast_for_expr(c, CHILD(ch, 0)); 2496 e = ast_for_expr(c, CHILD(ch, 0));
2008 if (!e) 2497 if (!e)
2009 return NULL; 2498 return NULL;
2010 /* f(lambda x: x[0] = 3) ends up getting parsed with 2499 /* f(lambda x: x[0] = 3) ends up getting parsed with
2011 * LHS test = lambda x: x[0], and RHS test = 3. 2500 * LHS test = lambda x: x[0], and RHS test = 3.
2012 * SF bug 132313 points out that complaining about a keyword 2501 * SF bug 132313 points out that complaining about a keyword
2013 * then is very confusing. 2502 * then is very confusing.
2014 */ 2503 */
2015 if (e->kind == Lambda_kind) { 2504 if (e->kind == Lambda_kind) {
2016 ast_error(CHILD(ch, 0), "lambda cannot contain assignment"); 2505 ast_error(c, CHILD(ch, 0), "lambda cannot contain assignment ");
2017 return NULL; 2506 return NULL;
2018 } else if (e->kind != Name_kind) { 2507 } else if (e->kind != Name_kind) {
2019 ast_error(CHILD(ch, 0), "keyword can't be an expression"); 2508 ast_error(c, CHILD(ch, 0), "keyword can't be an expression") ;
2020 return NULL; 2509 return NULL;
2021 } else if (forbidden_name(e->v.Name.id, ch, 1)) { 2510 } else if (forbidden_name(c, e->v.Name.id, ch, 1)) {
2022 return NULL; 2511 return NULL;
2023 } 2512 }
2024 key = e->v.Name.id; 2513 key = e->v.Name.id;
2025 for (k = 0; k < nkeywords; k++) { 2514 for (k = 0; k < nkeywords; k++) {
2026 tmp = ((keyword_ty)asdl_seq_GET(keywords, k))->arg; 2515 tmp = ((keyword_ty)asdl_seq_GET(keywords, k))->arg;
2027 if (!PyUnicode_Compare(tmp, key)) { 2516 if (!PyUnicode_Compare(tmp, key)) {
2028 ast_error(CHILD(ch, 0), "keyword argument repeated"); 2517 ast_error(c, CHILD(ch, 0), "keyword argument repeated");
2029 return NULL; 2518 return NULL;
2030 } 2519 }
2031 } 2520 }
2032 e = ast_for_expr(c, CHILD(ch, 2)); 2521 e = ast_for_expr(c, CHILD(ch, 2));
2033 if (!e) 2522 if (!e)
2034 return NULL; 2523 return NULL;
2035 kw = keyword(key, e, c->c_arena); 2524 kw = keyword(key, e, c->c_arena);
2036 if (!kw) 2525 if (!kw)
2037 return NULL; 2526 return NULL;
2038 asdl_seq_SET(keywords, nkeywords++, kw); 2527 asdl_seq_SET(keywords, nkeywords++, kw);
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after
2111 /* set_context checks that most expressions are not the left side. 2600 /* set_context checks that most expressions are not the left side.
2112 Augmented assignments can only have a name, a subscript, or an 2601 Augmented assignments can only have a name, a subscript, or an
2113 attribute on the left, though, so we have to explicitly check for 2602 attribute on the left, though, so we have to explicitly check for
2114 those. */ 2603 those. */
2115 switch (expr1->kind) { 2604 switch (expr1->kind) {
2116 case Name_kind: 2605 case Name_kind:
2117 case Attribute_kind: 2606 case Attribute_kind:
2118 case Subscript_kind: 2607 case Subscript_kind:
2119 break; 2608 break;
2120 default: 2609 default:
2121 ast_error(ch, "illegal expression for augmented assignment"); 2610 ast_error(c, ch, "illegal expression for augmented assignment");
2122 return NULL; 2611 return NULL;
2123 } 2612 }
2124 2613
2125 ch = CHILD(n, 2); 2614 ch = CHILD(n, 2);
2126 if (TYPE(ch) == testlist) 2615 if (TYPE(ch) == testlist)
2127 expr2 = ast_for_testlist(c, ch); 2616 expr2 = ast_for_testlist(c, ch);
2128 else 2617 else
2129 expr2 = ast_for_expr(c, ch); 2618 expr2 = ast_for_expr(c, ch);
2130 if (!expr2) 2619 if (!expr2)
2131 return NULL; 2620 return NULL;
(...skipping 12 matching lines...) Expand all
2144 2633
2145 /* a normal assignment */ 2634 /* a normal assignment */
2146 REQ(CHILD(n, 1), EQUAL); 2635 REQ(CHILD(n, 1), EQUAL);
2147 targets = asdl_seq_new(NCH(n) / 2, c->c_arena); 2636 targets = asdl_seq_new(NCH(n) / 2, c->c_arena);
2148 if (!targets) 2637 if (!targets)
2149 return NULL; 2638 return NULL;
2150 for (i = 0; i < NCH(n) - 2; i += 2) { 2639 for (i = 0; i < NCH(n) - 2; i += 2) {
2151 expr_ty e; 2640 expr_ty e;
2152 node *ch = CHILD(n, i); 2641 node *ch = CHILD(n, i);
2153 if (TYPE(ch) == yield_expr) { 2642 if (TYPE(ch) == yield_expr) {
2154 ast_error(ch, "assignment to yield expression not possible"); 2643 ast_error(c, ch, "assignment to yield expression not possible");
2155 return NULL; 2644 return NULL;
2156 } 2645 }
2157 e = ast_for_testlist(c, ch); 2646 e = ast_for_testlist(c, ch);
2158 if (!e) 2647 if (!e)
2159 return NULL; 2648 return NULL;
2160 2649
2161 /* set context to assign */ 2650 /* set context to assign */
2162 if (!set_context(c, e, Store, CHILD(n, i))) 2651 if (!set_context(c, e, Store, CHILD(n, i)))
2163 return NULL; 2652 return NULL;
2164 2653
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
2216 static stmt_ty 2705 static stmt_ty
2217 ast_for_flow_stmt(struct compiling *c, const node *n) 2706 ast_for_flow_stmt(struct compiling *c, const node *n)
2218 { 2707 {
2219 /* 2708 /*
2220 flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt 2709 flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
2221 | yield_stmt 2710 | yield_stmt
2222 break_stmt: 'break' 2711 break_stmt: 'break'
2223 continue_stmt: 'continue' 2712 continue_stmt: 'continue'
2224 return_stmt: 'return' [testlist] 2713 return_stmt: 'return' [testlist]
2225 yield_stmt: yield_expr 2714 yield_stmt: yield_expr
2226 yield_expr: 'yield' testlist 2715 yield_expr: 'yield' testlist | 'yield' 'from' test
2227 raise_stmt: 'raise' [test [',' test [',' test]]] 2716 raise_stmt: 'raise' [test [',' test [',' test]]]
2228 */ 2717 */
2229 node *ch; 2718 node *ch;
2230 2719
2231 REQ(n, flow_stmt); 2720 REQ(n, flow_stmt);
2232 ch = CHILD(n, 0); 2721 ch = CHILD(n, 0);
2233 switch (TYPE(ch)) { 2722 switch (TYPE(ch)) {
2234 case break_stmt: 2723 case break_stmt:
2235 return Break(LINENO(n), n->n_col_offset, c->c_arena); 2724 return Break(LINENO(n), n->n_col_offset, c->c_arena);
2236 case continue_stmt: 2725 case continue_stmt:
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
2280 { 2769 {
2281 /* 2770 /*
2282 import_as_name: NAME ['as' NAME] 2771 import_as_name: NAME ['as' NAME]
2283 dotted_as_name: dotted_name ['as' NAME] 2772 dotted_as_name: dotted_name ['as' NAME]
2284 dotted_name: NAME ('.' NAME)* 2773 dotted_name: NAME ('.' NAME)*
2285 */ 2774 */
2286 identifier str, name; 2775 identifier str, name;
2287 2776
2288 loop: 2777 loop:
2289 switch (TYPE(n)) { 2778 switch (TYPE(n)) {
2290 case import_as_name: { 2779 case import_as_name: {
2291 node *name_node = CHILD(n, 0); 2780 node *name_node = CHILD(n, 0);
2292 str = NULL; 2781 str = NULL;
2293 name = NEW_IDENTIFIER(name_node); 2782 name = NEW_IDENTIFIER(name_node);
2294 if (!name) 2783 if (!name)
2295 return NULL; 2784 return NULL;
2296 if (NCH(n) == 3) { 2785 if (NCH(n) == 3) {
2297 node *str_node = CHILD(n, 2); 2786 node *str_node = CHILD(n, 2);
2298 str = NEW_IDENTIFIER(str_node); 2787 str = NEW_IDENTIFIER(str_node);
2299 if (!str) 2788 if (!str)
2300 return NULL; 2789 return NULL;
2301 if (store && forbidden_name(str, str_node, 0)) 2790 if (store && forbidden_name(c, str, str_node, 0))
2302 return NULL; 2791 return NULL;
2303 } 2792 }
2304 else { 2793 else {
2305 if (forbidden_name(name, name_node, 0)) 2794 if (forbidden_name(c, name, name_node, 0))
2306 return NULL; 2795 return NULL;
2307 } 2796 }
2308 return alias(name, str, c->c_arena); 2797 return alias(name, str, c->c_arena);
2309 } 2798 }
2310 case dotted_as_name: 2799 case dotted_as_name:
2311 if (NCH(n) == 1) { 2800 if (NCH(n) == 1) {
2312 n = CHILD(n, 0); 2801 n = CHILD(n, 0);
2313 goto loop; 2802 goto loop;
2314 } 2803 }
2315 else { 2804 else {
2316 node *asname_node = CHILD(n, 2); 2805 node *asname_node = CHILD(n, 2);
2317 alias_ty a = alias_for_import_name(c, CHILD(n, 0), 0); 2806 alias_ty a = alias_for_import_name(c, CHILD(n, 0), 0);
2318 if (!a) 2807 if (!a)
2319 return NULL; 2808 return NULL;
2320 assert(!a->asname); 2809 assert(!a->asname);
2321 a->asname = NEW_IDENTIFIER(asname_node); 2810 a->asname = NEW_IDENTIFIER(asname_node);
2322 if (!a->asname) 2811 if (!a->asname)
2323 return NULL; 2812 return NULL;
2324 if (forbidden_name(a->asname, asname_node, 0)) 2813 if (forbidden_name(c, a->asname, asname_node, 0))
2325 return NULL; 2814 return NULL;
2326 return a; 2815 return a;
2327 } 2816 }
2328 break; 2817 break;
2329 case dotted_name: 2818 case dotted_name:
2330 if (NCH(n) == 1) { 2819 if (NCH(n) == 1) {
2331 node *name_node = CHILD(n, 0); 2820 node *name_node = CHILD(n, 0);
2332 name = NEW_IDENTIFIER(name_node); 2821 name = NEW_IDENTIFIER(name_node);
2333 if (!name) 2822 if (!name)
2334 return NULL; 2823 return NULL;
2335 if (store && forbidden_name(name, name_node, 0)) 2824 if (store && forbidden_name(c, name, name_node, 0))
2336 return NULL; 2825 return NULL;
2337 return alias(name, NULL, c->c_arena); 2826 return alias(name, NULL, c->c_arena);
2338 } 2827 }
2339 else { 2828 else {
2340 /* Create a string of the form "a.b.c" */ 2829 /* Create a string of the form "a.b.c" */
2341 int i; 2830 int i;
2342 size_t len; 2831 size_t len;
2343 char *s; 2832 char *s;
2344 PyObject *uni; 2833 PyObject *uni;
2345 2834
(...skipping 17 matching lines...) Expand all
2363 --s; 2852 --s;
2364 *s = '\0'; 2853 *s = '\0';
2365 uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), 2854 uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str),
2366 PyBytes_GET_SIZE(str), 2855 PyBytes_GET_SIZE(str),
2367 NULL); 2856 NULL);
2368 Py_DECREF(str); 2857 Py_DECREF(str);
2369 if (!uni) 2858 if (!uni)
2370 return NULL; 2859 return NULL;
2371 str = uni; 2860 str = uni;
2372 PyUnicode_InternInPlace(&str); 2861 PyUnicode_InternInPlace(&str);
2373 PyArena_AddPyObject(c->c_arena, str); 2862 if (PyArena_AddPyObject(c->c_arena, str) < 0) {
2863 Py_DECREF(str);
2864 return NULL;
2865 }
2374 return alias(str, NULL, c->c_arena); 2866 return alias(str, NULL, c->c_arena);
2375 } 2867 }
2376 break; 2868 break;
2377 case STAR: 2869 case STAR:
2378 str = PyUnicode_InternFromString("*"); 2870 str = PyUnicode_InternFromString("*");
2379 PyArena_AddPyObject(c->c_arena, str); 2871 if (PyArena_AddPyObject(c->c_arena, str) < 0) {
2872 Py_DECREF(str);
2873 return NULL;
2874 }
2380 return alias(str, NULL, c->c_arena); 2875 return alias(str, NULL, c->c_arena);
2381 default: 2876 default:
2382 PyErr_Format(PyExc_SystemError, 2877 PyErr_Format(PyExc_SystemError,
2383 "unexpected import name: %d", TYPE(n)); 2878 "unexpected import name: %d", TYPE(n));
2384 return NULL; 2879 return NULL;
2385 } 2880 }
2386 2881
2387 PyErr_SetString(PyExc_SystemError, "unhandled import name condition"); 2882 PyErr_SetString(PyExc_SystemError, "unhandled import name condition");
2388 return NULL; 2883 return NULL;
2389 } 2884 }
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
2454 case LPAR: 2949 case LPAR:
2455 /* from ... import (x, y, z) */ 2950 /* from ... import (x, y, z) */
2456 n = CHILD(n, idx + 1); 2951 n = CHILD(n, idx + 1);
2457 n_children = NCH(n); 2952 n_children = NCH(n);
2458 break; 2953 break;
2459 case import_as_names: 2954 case import_as_names:
2460 /* from ... import x, y, z */ 2955 /* from ... import x, y, z */
2461 n = CHILD(n, idx); 2956 n = CHILD(n, idx);
2462 n_children = NCH(n); 2957 n_children = NCH(n);
2463 if (n_children % 2 == 0) { 2958 if (n_children % 2 == 0) {
2464 ast_error(n, "trailing comma not allowed without" 2959 ast_error(c, n, "trailing comma not allowed without"
2465 " surrounding parentheses"); 2960 " surrounding parentheses");
2466 return NULL; 2961 return NULL;
2467 } 2962 }
2468 break; 2963 break;
2469 default: 2964 default:
2470 ast_error(n, "Unexpected node-type in from-import"); 2965 ast_error(c, n, "Unexpected node-type in from-import");
2471 return NULL; 2966 return NULL;
2472 } 2967 }
2473 2968
2474 aliases = asdl_seq_new((n_children + 1) / 2, c->c_arena); 2969 aliases = asdl_seq_new((n_children + 1) / 2, c->c_arena);
2475 if (!aliases) 2970 if (!aliases)
2476 return NULL; 2971 return NULL;
2477 2972
2478 /* handle "from ... import *" special b/c there's no children */ 2973 /* handle "from ... import *" special b/c there's no children */
2479 if (TYPE(n) == STAR) { 2974 if (TYPE(n) == STAR) {
2480 alias_ty import_alias = alias_for_import_name(c, n, 1); 2975 alias_ty import_alias = alias_for_import_name(c, n, 1);
(...skipping 385 matching lines...) Expand 10 before | Expand all | Expand 10 after
2866 3361
2867 return ExceptHandler(expression, NULL, suite_seq, LINENO(exc), 3362 return ExceptHandler(expression, NULL, suite_seq, LINENO(exc),
2868 exc->n_col_offset, c->c_arena); 3363 exc->n_col_offset, c->c_arena);
2869 } 3364 }
2870 else if (NCH(exc) == 4) { 3365 else if (NCH(exc) == 4) {
2871 asdl_seq *suite_seq; 3366 asdl_seq *suite_seq;
2872 expr_ty expression; 3367 expr_ty expression;
2873 identifier e = NEW_IDENTIFIER(CHILD(exc, 3)); 3368 identifier e = NEW_IDENTIFIER(CHILD(exc, 3));
2874 if (!e) 3369 if (!e)
2875 return NULL; 3370 return NULL;
2876 if (forbidden_name(e, CHILD(exc, 3), 0)) 3371 if (forbidden_name(c, e, CHILD(exc, 3), 0))
2877 return NULL; 3372 return NULL;
2878 expression = ast_for_expr(c, CHILD(exc, 1)); 3373 expression = ast_for_expr(c, CHILD(exc, 1));
2879 if (!expression) 3374 if (!expression)
2880 return NULL; 3375 return NULL;
2881 suite_seq = ast_for_suite(c, body); 3376 suite_seq = ast_for_suite(c, body);
2882 if (!suite_seq) 3377 if (!suite_seq)
2883 return NULL; 3378 return NULL;
2884 3379
2885 return ExceptHandler(expression, e, suite_seq, LINENO(exc), 3380 return ExceptHandler(expression, e, suite_seq, LINENO(exc),
2886 exc->n_col_offset, c->c_arena); 3381 exc->n_col_offset, c->c_arena);
2887 } 3382 }
2888 3383
2889 PyErr_Format(PyExc_SystemError, 3384 PyErr_Format(PyExc_SystemError,
2890 "wrong number of children for 'except' clause: %d", 3385 "wrong number of children for 'except' clause: %d",
2891 NCH(exc)); 3386 NCH(exc));
2892 return NULL; 3387 return NULL;
2893 } 3388 }
2894 3389
2895 static stmt_ty 3390 static stmt_ty
2896 ast_for_try_stmt(struct compiling *c, const node *n) 3391 ast_for_try_stmt(struct compiling *c, const node *n)
2897 { 3392 {
2898 const int nch = NCH(n); 3393 const int nch = NCH(n);
2899 int n_except = (nch - 3)/3; 3394 int n_except = (nch - 3)/3;
2900 asdl_seq *body, *orelse = NULL, *finally = NULL; 3395 asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL;
2901 3396
2902 REQ(n, try_stmt); 3397 REQ(n, try_stmt);
2903 3398
2904 body = ast_for_suite(c, CHILD(n, 2)); 3399 body = ast_for_suite(c, CHILD(n, 2));
2905 if (body == NULL) 3400 if (body == NULL)
2906 return NULL; 3401 return NULL;
2907 3402
2908 if (TYPE(CHILD(n, nch - 3)) == NAME) { 3403 if (TYPE(CHILD(n, nch - 3)) == NAME) {
2909 if (strcmp(STR(CHILD(n, nch - 3)), "finally") == 0) { 3404 if (strcmp(STR(CHILD(n, nch - 3)), "finally") == 0) {
2910 if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == NAME) { 3405 if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == NAME) {
(...skipping 14 matching lines...) Expand all
2925 else { 3420 else {
2926 /* we can assume it's an "else", 3421 /* we can assume it's an "else",
2927 otherwise it would have a type of except_clause */ 3422 otherwise it would have a type of except_clause */
2928 orelse = ast_for_suite(c, CHILD(n, nch - 1)); 3423 orelse = ast_for_suite(c, CHILD(n, nch - 1));
2929 if (orelse == NULL) 3424 if (orelse == NULL)
2930 return NULL; 3425 return NULL;
2931 n_except--; 3426 n_except--;
2932 } 3427 }
2933 } 3428 }
2934 else if (TYPE(CHILD(n, nch - 3)) != except_clause) { 3429 else if (TYPE(CHILD(n, nch - 3)) != except_clause) {
2935 ast_error(n, "malformed 'try' statement"); 3430 ast_error(c, n, "malformed 'try' statement");
2936 return NULL; 3431 return NULL;
2937 } 3432 }
2938 3433
2939 if (n_except > 0) { 3434 if (n_except > 0) {
2940 int i; 3435 int i;
2941 stmt_ty except_st;
2942 /* process except statements to create a try ... except */ 3436 /* process except statements to create a try ... except */
2943 asdl_seq *handlers = asdl_seq_new(n_except, c->c_arena); 3437 handlers = asdl_seq_new(n_except, c->c_arena);
2944 if (handlers == NULL) 3438 if (handlers == NULL)
2945 return NULL; 3439 return NULL;
2946 3440
2947 for (i = 0; i < n_except; i++) { 3441 for (i = 0; i < n_except; i++) {
2948 excepthandler_ty e = ast_for_except_clause(c, CHILD(n, 3 + i * 3), 3442 excepthandler_ty e = ast_for_except_clause(c, CHILD(n, 3 + i * 3),
2949 CHILD(n, 5 + i * 3)); 3443 CHILD(n, 5 + i * 3));
2950 if (!e) 3444 if (!e)
2951 return NULL; 3445 return NULL;
2952 asdl_seq_SET(handlers, i, e); 3446 asdl_seq_SET(handlers, i, e);
2953 } 3447 }
2954 3448 }
2955 except_st = TryExcept(body, handlers, orelse, LINENO(n), 3449
2956 n->n_col_offset, c->c_arena); 3450 assert(finally != NULL || asdl_seq_LEN(handlers));
2957 if (!finally) 3451 return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, c->c _arena);
2958 return except_st;
2959
2960 /* if a 'finally' is present too, we nest the TryExcept within a
2961 TryFinally to emulate try ... except ... finally */
2962 body = asdl_seq_new(1, c->c_arena);
2963 if (body == NULL)
2964 return NULL;
2965 asdl_seq_SET(body, 0, except_st);
2966 }
2967
2968 /* must be a try ... finally (except clauses are in body, if any exist) */
2969 assert(finally != NULL);
2970 return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena);
2971 } 3452 }
2972 3453
2973 /* with_item: test ['as' expr] */ 3454 /* with_item: test ['as' expr] */
2974 static stmt_ty 3455 static withitem_ty
2975 ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content) 3456 ast_for_with_item(struct compiling *c, const node *n)
2976 { 3457 {
2977 expr_ty context_expr, optional_vars = NULL; 3458 expr_ty context_expr, optional_vars = NULL;
2978 3459
2979 REQ(n, with_item); 3460 REQ(n, with_item);
2980 context_expr = ast_for_expr(c, CHILD(n, 0)); 3461 context_expr = ast_for_expr(c, CHILD(n, 0));
2981 if (!context_expr) 3462 if (!context_expr)
2982 return NULL; 3463 return NULL;
2983 if (NCH(n) == 3) { 3464 if (NCH(n) == 3) {
2984 optional_vars = ast_for_expr(c, CHILD(n, 2)); 3465 optional_vars = ast_for_expr(c, CHILD(n, 2));
2985 3466
2986 if (!optional_vars) { 3467 if (!optional_vars) {
2987 return NULL; 3468 return NULL;
2988 } 3469 }
2989 if (!set_context(c, optional_vars, Store, n)) { 3470 if (!set_context(c, optional_vars, Store, n)) {
2990 return NULL; 3471 return NULL;
2991 } 3472 }
2992 } 3473 }
2993 3474
2994 return With(context_expr, optional_vars, content, LINENO(n), 3475 return withitem(context_expr, optional_vars, c->c_arena);
2995 n->n_col_offset, c->c_arena);
2996 } 3476 }
2997 3477
2998 /* with_stmt: 'with' with_item (',' with_item)* ':' suite */ 3478 /* with_stmt: 'with' with_item (',' with_item)* ':' suite */
2999 static stmt_ty 3479 static stmt_ty
3000 ast_for_with_stmt(struct compiling *c, const node *n) 3480 ast_for_with_stmt(struct compiling *c, const node *n)
3001 { 3481 {
3002 int i; 3482 int i, n_items;
3003 stmt_ty ret; 3483 asdl_seq *items, *body;
3004 asdl_seq *inner;
3005 3484
3006 REQ(n, with_stmt); 3485 REQ(n, with_stmt);
3007 3486
3008 /* process the with items inside-out */ 3487 n_items = (NCH(n) - 2) / 2;
3009 i = NCH(n) - 1; 3488 items = asdl_seq_new(n_items, c->c_arena);
3010 /* the suite of the innermost with item is the suite of the with stmt */ 3489 if (!items)
3011 inner = ast_for_suite(c, CHILD(n, i)); 3490 return NULL;
3012 if (!inner) 3491 for (i = 1; i < NCH(n) - 2; i += 2) {
3013 return NULL; 3492 withitem_ty item = ast_for_with_item(c, CHILD(n, i));
3014 3493 if (!item)
3015 for (;;) { 3494 return NULL;
3016 i -= 2; 3495 asdl_seq_SET(items, (i - 1) / 2, item);
3017 ret = ast_for_with_item(c, CHILD(n, i), inner); 3496 }
3018 if (!ret) 3497
3019 return NULL; 3498 body = ast_for_suite(c, CHILD(n, NCH(n) - 1));
3020 /* was this the last item? */ 3499 if (!body)
3021 if (i == 1) 3500 return NULL;
3022 break; 3501
3023 /* if not, wrap the result so far in a new sequence */ 3502 return With(items, body, LINENO(n), n->n_col_offset, c->c_arena);
3024 inner = asdl_seq_new(1, c->c_arena);
3025 if (!inner)
3026 return NULL;
3027 asdl_seq_SET(inner, 0, ret);
3028 }
3029
3030 return ret;
3031 } 3503 }
3032 3504
3033 static stmt_ty 3505 static stmt_ty
3034 ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) 3506 ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
3035 { 3507 {
3036 /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */ 3508 /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */
3037 PyObject *classname; 3509 PyObject *classname;
3038 asdl_seq *s; 3510 asdl_seq *s;
3039 expr_ty call; 3511 expr_ty call;
3040 3512
3041 REQ(n, classdef); 3513 REQ(n, classdef);
3042 3514
3043 if (NCH(n) == 4) { /* class NAME ':' suite */ 3515 if (NCH(n) == 4) { /* class NAME ':' suite */
3044 s = ast_for_suite(c, CHILD(n, 3)); 3516 s = ast_for_suite(c, CHILD(n, 3));
3045 if (!s) 3517 if (!s)
3046 return NULL; 3518 return NULL;
3047 classname = NEW_IDENTIFIER(CHILD(n, 1)); 3519 classname = NEW_IDENTIFIER(CHILD(n, 1));
3048 if (!classname) 3520 if (!classname)
3049 return NULL; 3521 return NULL;
3050 if (forbidden_name(classname, CHILD(n, 3), 0)) 3522 if (forbidden_name(c, classname, CHILD(n, 3), 0))
3051 return NULL; 3523 return NULL;
3052 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, 3524 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
3053 LINENO(n), n->n_col_offset, c->c_arena); 3525 LINENO(n), n->n_col_offset, c->c_arena);
3054 } 3526 }
3055 3527
3056 if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ 3528 if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
3057 s = ast_for_suite(c, CHILD(n,5)); 3529 s = ast_for_suite(c, CHILD(n,5));
3058 if (!s) 3530 if (!s)
3059 return NULL; 3531 return NULL;
3060 classname = NEW_IDENTIFIER(CHILD(n, 1)); 3532 classname = NEW_IDENTIFIER(CHILD(n, 1));
3061 if (!classname) 3533 if (!classname)
3062 return NULL; 3534 return NULL;
3063 if (forbidden_name(classname, CHILD(n, 3), 0)) 3535 if (forbidden_name(c, classname, CHILD(n, 3), 0))
3064 return NULL; 3536 return NULL;
3065 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, 3537 return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
3066 LINENO(n), n->n_col_offset, c->c_arena); 3538 LINENO(n), n->n_col_offset, c->c_arena);
3067 } 3539 }
3068 3540
3069 /* class NAME '(' arglist ')' ':' suite */ 3541 /* class NAME '(' arglist ')' ':' suite */
3070 /* build up a fake Call node so we can extract its pieces */ 3542 /* build up a fake Call node so we can extract its pieces */
3071 { 3543 {
3072 PyObject *dummy_name; 3544 PyObject *dummy_name;
3073 expr_ty dummy; 3545 expr_ty dummy;
3074 dummy_name = NEW_IDENTIFIER(CHILD(n, 1)); 3546 dummy_name = NEW_IDENTIFIER(CHILD(n, 1));
3075 if (!dummy_name) 3547 if (!dummy_name)
3076 return NULL; 3548 return NULL;
3077 dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena); 3549 dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena);
3078 call = ast_for_call(c, CHILD(n, 3), dummy); 3550 call = ast_for_call(c, CHILD(n, 3), dummy);
3079 if (!call) 3551 if (!call)
3080 return NULL; 3552 return NULL;
3081 } 3553 }
3082 s = ast_for_suite(c, CHILD(n, 6)); 3554 s = ast_for_suite(c, CHILD(n, 6));
3083 if (!s) 3555 if (!s)
3084 return NULL; 3556 return NULL;
3085 classname = NEW_IDENTIFIER(CHILD(n, 1)); 3557 classname = NEW_IDENTIFIER(CHILD(n, 1));
3086 if (!classname) 3558 if (!classname)
3087 return NULL; 3559 return NULL;
3088 if (forbidden_name(classname, CHILD(n, 1), 0)) 3560 if (forbidden_name(c, classname, CHILD(n, 1), 0))
3089 return NULL; 3561 return NULL;
3090 3562
3091 return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, 3563 return ClassDef(classname, call->v.Call.args, call->v.Call.keywords,
3092 call->v.Call.starargs, call->v.Call.kwargs, s, 3564 call->v.Call.starargs, call->v.Call.kwargs, s,
3093 decorator_seq, LINENO(n), n->n_col_offset, c->c_arena); 3565 decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
3094 } 3566 }
3095 3567
3096 static stmt_ty 3568 static stmt_ty
3097 ast_for_stmt(struct compiling *c, const node *n) 3569 ast_for_stmt(struct compiling *c, const node *n)
3098 { 3570 {
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after
3204 else 3676 else
3205 { 3677 {
3206 dx = PyOS_string_to_double(s, NULL, NULL); 3678 dx = PyOS_string_to_double(s, NULL, NULL);
3207 if (dx == -1.0 && PyErr_Occurred()) 3679 if (dx == -1.0 && PyErr_Occurred())
3208 return NULL; 3680 return NULL;
3209 return PyFloat_FromDouble(dx); 3681 return PyFloat_FromDouble(dx);
3210 } 3682 }
3211 } 3683 }
3212 3684
3213 static PyObject * 3685 static PyObject *
3214 decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encod ing) 3686 decode_utf8(struct compiling *c, const char **sPtr, const char *end)
3215 { 3687 {
3216 PyObject *u, *v;
3217 char *s, *t; 3688 char *s, *t;
3218 t = s = (char *)*sPtr; 3689 t = s = (char *)*sPtr;
3219 /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */ 3690 /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */
3220 while (s < end && (*s & 0x80)) s++; 3691 while (s < end && (*s & 0x80)) s++;
3221 *sPtr = s; 3692 *sPtr = s;
3222 u = PyUnicode_DecodeUTF8(t, s - t, NULL); 3693 return PyUnicode_DecodeUTF8(t, s - t, NULL);
3223 if (u == NULL)
3224 return NULL;
3225 v = PyUnicode_AsEncodedString(u, encoding, NULL);
3226 Py_DECREF(u);
3227 return v;
3228 } 3694 }
3229 3695
3230 static PyObject * 3696 static PyObject *
3231 decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons t char *encoding) 3697 decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons t char *encoding)
3232 { 3698 {
3233 PyObject *v, *u; 3699 PyObject *v, *u;
3234 char *buf; 3700 char *buf;
3235 char *p; 3701 char *p;
3236 const char *end; 3702 const char *end;
3237 3703
(...skipping 13 matching lines...) Expand all
3251 while (s < end) { 3717 while (s < end) {
3252 if (*s == '\\') { 3718 if (*s == '\\') {
3253 *p++ = *s++; 3719 *p++ = *s++;
3254 if (*s & 0x80) { 3720 if (*s & 0x80) {
3255 strcpy(p, "u005c"); 3721 strcpy(p, "u005c");
3256 p += 5; 3722 p += 5;
3257 } 3723 }
3258 } 3724 }
3259 if (*s & 0x80) { /* XXX inefficient */ 3725 if (*s & 0x80) { /* XXX inefficient */
3260 PyObject *w; 3726 PyObject *w;
3261 char *r; 3727 int kind;
3262 Py_ssize_t rn, i; 3728 void *data;
3263 w = decode_utf8(c, &s, end, "utf-32-be"); 3729 Py_ssize_t len, i;
3730 w = decode_utf8(c, &s, end);
3264 if (w == NULL) { 3731 if (w == NULL) {
3265 Py_DECREF(u); 3732 Py_DECREF(u);
3266 return NULL; 3733 return NULL;
3267 } 3734 }
3268 r = PyBytes_AS_STRING(w); 3735 kind = PyUnicode_KIND(w);
3269 rn = Py_SIZE(w); 3736 data = PyUnicode_DATA(w);
3270 assert(rn % 4 == 0); 3737 len = PyUnicode_GET_LENGTH(w);
3271 for (i = 0; i < rn; i += 4) { 3738 for (i = 0; i < len; i++) {
3272 sprintf(p, "\\U%02x%02x%02x%02x", 3739 Py_UCS4 chr = PyUnicode_READ(kind, data, i);
3273 r[i + 0] & 0xFF, 3740 sprintf(p, "\\U%08x", chr);
3274 r[i + 1] & 0xFF,
3275 r[i + 2] & 0xFF,
3276 r[i + 3] & 0xFF);
3277 p += 10; 3741 p += 10;
3278 } 3742 }
3279 /* Should be impossible to overflow */ 3743 /* Should be impossible to overflow */
3280 assert(p - buf <= Py_SIZE(u)); 3744 assert(p - buf <= Py_SIZE(u));
3281 Py_DECREF(w); 3745 Py_DECREF(w);
3282 } else { 3746 } else {
3283 *p++ = *s++; 3747 *p++ = *s++;
3284 } 3748 }
3285 } 3749 }
3286 len = p - buf; 3750 len = p - buf;
(...skipping 12 matching lines...) Expand all
3299 * parsestr parses it, and returns the decoded Python string object. 3763 * parsestr parses it, and returns the decoded Python string object.
3300 */ 3764 */
3301 static PyObject * 3765 static PyObject *
3302 parsestr(struct compiling *c, const node *n, int *bytesmode) 3766 parsestr(struct compiling *c, const node *n, int *bytesmode)
3303 { 3767 {
3304 size_t len; 3768 size_t len;
3305 const char *s = STR(n); 3769 const char *s = STR(n);
3306 int quote = Py_CHARMASK(*s); 3770 int quote = Py_CHARMASK(*s);
3307 int rawmode = 0; 3771 int rawmode = 0;
3308 int need_encoding; 3772 int need_encoding;
3309 if (isalpha(quote)) { 3773 if (Py_ISALPHA(quote)) {
3310 if (quote == 'b' || quote == 'B') { 3774 while (!*bytesmode || !rawmode) {
3311 quote = *++s; 3775 if (quote == 'b' || quote == 'B') {
3312 *bytesmode = 1; 3776 quote = *++s;
3313 } 3777 *bytesmode = 1;
3314 if (quote == 'r' || quote == 'R') { 3778 }
3315 quote = *++s; 3779 else if (quote == 'u' || quote == 'U') {
3316 rawmode = 1; 3780 quote = *++s;
3781 }
3782 else if (quote == 'r' || quote == 'R') {
3783 quote = *++s;
3784 rawmode = 1;
3785 }
3786 else {
3787 break;
3788 }
3317 } 3789 }
3318 } 3790 }
3319 if (quote != '\'' && quote != '\"') { 3791 if (quote != '\'' && quote != '\"') {
3320 PyErr_BadInternalCall(); 3792 PyErr_BadInternalCall();
3321 return NULL; 3793 return NULL;
3322 } 3794 }
3323 s++; 3795 s++;
3324 len = strlen(s); 3796 len = strlen(s);
3325 if (len > INT_MAX) { 3797 if (len > INT_MAX) {
3326 PyErr_SetString(PyExc_OverflowError, 3798 PyErr_SetString(PyExc_OverflowError,
(...skipping 10 matching lines...) Expand all
3337 if (s[--len] != quote || s[--len] != quote) { 3809 if (s[--len] != quote || s[--len] != quote) {
3338 PyErr_BadInternalCall(); 3810 PyErr_BadInternalCall();
3339 return NULL; 3811 return NULL;
3340 } 3812 }
3341 } 3813 }
3342 if (!*bytesmode && !rawmode) { 3814 if (!*bytesmode && !rawmode) {
3343 return decode_unicode(c, s, len, rawmode, c->c_encoding); 3815 return decode_unicode(c, s, len, rawmode, c->c_encoding);
3344 } 3816 }
3345 if (*bytesmode) { 3817 if (*bytesmode) {
3346 /* Disallow non-ascii characters (but not escapes) */ 3818 /* Disallow non-ascii characters (but not escapes) */
3347 const char *c; 3819 const char *ch;
3348 for (c = s; *c; c++) { 3820 for (ch = s; *ch; ch++) {
3349 if (Py_CHARMASK(*c) >= 0x80) { 3821 if (Py_CHARMASK(*ch) >= 0x80) {
3350 ast_error(n, "bytes can only contain ASCII " 3822 ast_error(c, n, "bytes can only contain ASCII "
3351 "literal characters."); 3823 "literal characters.");
3352 return NULL; 3824 return NULL;
3353 } 3825 }
3354 } 3826 }
3355 } 3827 }
3356 need_encoding = (!*bytesmode && c->c_encoding != NULL && 3828 need_encoding = (!*bytesmode && c->c_encoding != NULL &&
3357 strcmp(c->c_encoding, "utf-8") != 0); 3829 strcmp(c->c_encoding, "utf-8") != 0);
3358 if (rawmode || strchr(s, '\\') == NULL) { 3830 if (rawmode || strchr(s, '\\') == NULL) {
3359 if (need_encoding) { 3831 if (need_encoding) {
3360 PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); 3832 PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
(...skipping 27 matching lines...) Expand all
3388 v = parsestr(c, CHILD(n, 0), bytesmode); 3860 v = parsestr(c, CHILD(n, 0), bytesmode);
3389 if (v != NULL) { 3861 if (v != NULL) {
3390 /* String literal concatenation */ 3862 /* String literal concatenation */
3391 for (i = 1; i < NCH(n); i++) { 3863 for (i = 1; i < NCH(n); i++) {
3392 PyObject *s; 3864 PyObject *s;
3393 int subbm = 0; 3865 int subbm = 0;
3394 s = parsestr(c, CHILD(n, i), &subbm); 3866 s = parsestr(c, CHILD(n, i), &subbm);
3395 if (s == NULL) 3867 if (s == NULL)
3396 goto onError; 3868 goto onError;
3397 if (*bytesmode != subbm) { 3869 if (*bytesmode != subbm) {
3398 ast_error(n, "cannot mix bytes and nonbytes literals"); 3870 ast_error(c, n, "cannot mix bytes and nonbytes literals");
3871 Py_DECREF(s);
3399 goto onError; 3872 goto onError;
3400 } 3873 }
3401 if (PyBytes_Check(v) && PyBytes_Check(s)) { 3874 if (PyBytes_Check(v) && PyBytes_Check(s)) {
3402 PyBytes_ConcatAndDel(&v, s); 3875 PyBytes_ConcatAndDel(&v, s);
3403 if (v == NULL) 3876 if (v == NULL)
3404 goto onError; 3877 goto onError;
3405 } 3878 }
3406 else { 3879 else {
3407 PyObject *temp = PyUnicode_Concat(v, s); 3880 PyObject *temp = PyUnicode_Concat(v, s);
3408 Py_DECREF(s); 3881 Py_DECREF(s);
3409 Py_DECREF(v); 3882 Py_DECREF(v);
3410 v = temp; 3883 v = temp;
3411 if (v == NULL) 3884 if (v == NULL)
3412 goto onError; 3885 goto onError;
3413 } 3886 }
3414 } 3887 }
3415 } 3888 }
3416 return v; 3889 return v;
3417 3890
3418 onError: 3891 onError:
3419 Py_XDECREF(v); 3892 Py_XDECREF(v);
3420 return NULL; 3893 return NULL;
3421 } 3894 }
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+