diff -r 3a0bc2efed26 Include/node.h --- a/Include/node.h Wed Mar 09 10:52:08 2016 +0200 +++ b/Include/node.h Wed Mar 09 11:44:21 2016 +0000 @@ -19,6 +19,7 @@ PyAPI_FUNC(node *) PyNode_New(int type); PyAPI_FUNC(int) PyNode_AddChild(node *n, int type, char *str, int lineno, int col_offset); +PyAPI_FUNC(void) PyNode_Compress(node *n); PyAPI_FUNC(void) PyNode_Free(node *n); #ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); diff -r 3a0bc2efed26 Parser/parser.c --- a/Parser/parser.c Wed Mar 09 10:52:08 2016 +0200 +++ b/Parser/parser.c Wed Mar 09 11:44:21 2016 +0000 @@ -56,12 +56,12 @@ { if (s_empty(s)) Py_FatalError("s_pop: parser stack underflow -- FATAL"); - s->s_top++; + PyNode_Compress(s->s_top++->s_parent); } #else /* !Py_DEBUG */ -#define s_pop(s) (s)->s_top++ +#define s_pop(s) PyNode_Compress((s)->s_top++->s_parent) #endif diff -r 3a0bc2efed26 Parser/pgenmain.c --- a/Parser/pgenmain.c Wed Mar 09 10:52:08 2016 +0200 +++ b/Parser/pgenmain.c Wed Mar 09 11:44:21 2016 +0000 @@ -177,3 +177,6 @@ vfprintf(stderr, format, va); va_end(va); } + +void PyNode_Compress(node* n) {} // no compression in pgen + diff -r 3a0bc2efed26 Python/ast.c --- a/Python/ast.c Wed Mar 09 10:52:08 2016 +0200 +++ b/Python/ast.c Wed Mar 09 11:44:21 2016 +0000 @@ -11,6 +11,13 @@ #include +#define case_EXPR case comparison: case lambdef: \ + case term: case factor: case power: \ + case test: case test_nocond: case and_test: case or_test: case not_test: \ + case expr: case xor_expr: case and_expr: case shift_expr: \ + case arith_expr: case star_expr: case atom_expr: case atom + + static int validate_stmts(asdl_seq *); static int validate_exprs(asdl_seq *, expr_context_ty, int); static int validate_nonempty_seq(asdl_seq *, const char *, const char *); @@ -712,7 +719,7 @@ l = 0; for (i = 0; i < NCH(n); i++) { ch = CHILD(n, i); - if (TYPE(ch) == stmt) + if (TYPE(ch) == stmt || TYPE(ch) == simple_stmt || TYPE(ch) == compound_stmt) l += num_stmts(ch); } return l; @@ -723,14 +730,11 @@ case simple_stmt: return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */ case suite: - if (NCH(n) == 1) - return num_stmts(CHILD(n, 0)); - else { - l = 0; - for (i = 2; i < (NCH(n) - 1); i++) - l += num_stmts(CHILD(n, i)); - return l; - } + assert(NCH(n) > 1); + l = 0; + for (i = 2; i < (NCH(n) - 1); i++) + l += num_stmts(CHILD(n, i)); + return l; default: { char buf[128]; @@ -776,7 +780,7 @@ ch = CHILD(n, i); if (TYPE(ch) == NEWLINE) continue; - REQ(ch, stmt); + assert(TYPE(ch) == stmt || TYPE(ch) == simple_stmt || TYPE(ch) == compound_stmt); num = num_stmts(ch); if (num == 1) { s = ast_for_stmt(&c, ch); @@ -785,7 +789,6 @@ asdl_seq_SET(stmts, k++, s); } else { - ch = CHILD(ch, 0); REQ(ch, simple_stmt); for (j = 0; j < num; j++) { s = ast_for_stmt(&c, CHILD(ch, j * 2)); @@ -1125,9 +1128,7 @@ /* comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is' |'is' 'not' */ - REQ(n, comp_op); - if (NCH(n) == 1) { - n = CHILD(n, 0); + if (NCH(n) == 0) { switch (TYPE(n)) { case LESS: return Lt; @@ -1152,7 +1153,8 @@ return (cmpop_ty)0; } } - else if (NCH(n) == 2) { + REQ(n, comp_op); + if (NCH(n) == 2) { /* handle "not in" and "is not" */ switch (TYPE(CHILD(n, 0))) { case NAME: @@ -1188,7 +1190,10 @@ for (i = 0; i < NCH(n); i += 2) { const node *ch = CHILD(n, i); - assert(TYPE(ch) == test || TYPE(ch) == test_nocond || TYPE(ch) == star_expr); + switch (TYPE(ch)) { + case_EXPR: break; + default: assert(0 && "unexpected type in testlist"); + } expression = ast_for_expr(c, ch); if (!expression) @@ -2197,47 +2202,51 @@ node *ch; expr_ty lower = NULL, upper = NULL, step = NULL; - REQ(n, subscript); - /* subscript: test | [test] ':' [test] [sliceop] sliceop: ':' [test] */ - ch = CHILD(n, 0); - if (NCH(n) == 1 && TYPE(ch) == test) { + switch (TYPE(n)) { + case_EXPR: /* 'step' variable hold no significance in terms of being used over other vars */ - step = ast_for_expr(c, ch); + step = ast_for_expr(c, n); if (!step) return NULL; return Index(step, c->c_arena); + case COLON: + return Slice(NULL, NULL, NULL, c->c_arena); } - if (TYPE(ch) == test) { + REQ(n, subscript); + assert(NCH(n) > 1); + ch = CHILD(n, 0); + if (TYPE(ch) != COLON) { lower = ast_for_expr(c, ch); if (!lower) return NULL; - } - - /* If there's an upper bound it's in the second or third position. */ - if (TYPE(ch) == COLON) { - if (NCH(n) > 1) { - node *n2 = CHILD(n, 1); - - if (TYPE(n2) == test) { + + /* If there's an upper bound it's in the second or third position. */ + if (NCH(n) > 2) { + node *n2 = CHILD(n, 2); + + if (TYPE(n2) != sliceop) { upper = ast_for_expr(c, n2); if (!upper) return NULL; } } - } else if (NCH(n) > 2) { - node *n2 = CHILD(n, 2); - - if (TYPE(n2) == test) { - upper = ast_for_expr(c, n2); - if (!upper) - return NULL; + } else { + REQ(ch, COLON); + if (NCH(n) > 1) { + node *n2 = CHILD(n, 1); + + if (TYPE(n2) != sliceop) { + upper = ast_for_expr(c, n2); + if (!upper) + return NULL; + } } } @@ -2245,11 +2254,9 @@ if (TYPE(ch) == sliceop) { if (NCH(ch) != 1) { ch = CHILD(ch, 1); - if (TYPE(ch) == test) { - step = ast_for_expr(c, ch); - if (!step) - return NULL; - } + step = ast_for_expr(c, ch); + if (!step) + return NULL; } } @@ -2417,17 +2424,14 @@ REQ(n, atom_expr); nch = NCH(n); - - if (TYPE(CHILD(n, 0)) == AWAIT) { + assert(nch > 1); + + if (TYPE(CHILD(n, 0)) == AWAIT) start = 1; - assert(nch > 1); - } e = ast_for_atom(c, CHILD(n, start)); if (!e) return NULL; - if (nch == 1) - return e; if (start && nch == 2) { return Await(e, LINENO(n), n->n_col_offset, c->c_arena); } @@ -2459,13 +2463,17 @@ /* power: atom trailer* ('**' factor)* */ expr_ty e; + node *ch; REQ(n, power); - e = ast_for_atom_expr(c, CHILD(n, 0)); + assert(NCH(n) > 1); + ch = CHILD(n, 0); + if (TYPE(ch) == atom) + e = ast_for_atom(c, ch); + else + e = ast_for_atom_expr(c, ch); if (!e) return NULL; - if (NCH(n) == 1) - return e; - if (TYPE(CHILD(n, NCH(n) - 1)) == factor) { + if (TYPE(CHILD(n, NCH(n) - 2)) == DOUBLESTAR) { expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1)); if (!f) return NULL; @@ -2514,25 +2522,22 @@ yield_expr: 'yield' [yield_arg] */ - asdl_seq *seq; + asdl_seq *seq, *cmps; + asdl_int_seq *ops; + expr_ty expression; int i; - loop: switch (TYPE(n)) { + case lambdef: + case lambdef_nocond: + return ast_for_lambdef(c, n); case test: case test_nocond: - if (TYPE(CHILD(n, 0)) == lambdef || - TYPE(CHILD(n, 0)) == lambdef_nocond) - return ast_for_lambdef(c, CHILD(n, 0)); - else if (NCH(n) > 1) - return ast_for_ifexpr(c, n); - /* Fallthrough */ + assert(NCH(n) > 1); + return ast_for_ifexpr(c, n); case or_test: case and_test: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } + assert(NCH(n) > 1); seq = _Py_asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); if (!seq) return NULL; @@ -2548,59 +2553,45 @@ assert(!strcmp(STR(CHILD(n, 1)), "or")); return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena); case not_test: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; + assert(NCH(n) > 1); + expression = ast_for_expr(c, CHILD(n, 1)); + if (!expression) + return NULL; + + return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, + c->c_arena); + case comparison: + assert(NCH(n) > 1); + ops = _Py_asdl_int_seq_new(NCH(n) / 2, c->c_arena); + if (!ops) + return NULL; + cmps = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); + if (!cmps) { + return NULL; } - else { - expr_ty expression = ast_for_expr(c, CHILD(n, 1)); - if (!expression) - return NULL; - - return UnaryOp(Not, expression, LINENO(n), n->n_col_offset, - c->c_arena); - } - case comparison: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } - else { - expr_ty expression; - asdl_int_seq *ops; - asdl_seq *cmps; - ops = _Py_asdl_int_seq_new(NCH(n) / 2, c->c_arena); - if (!ops) - return NULL; - cmps = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); - if (!cmps) { + for (i = 1; i < NCH(n); i += 2) { + cmpop_ty newoperator; + + newoperator = ast_for_comp_op(c, CHILD(n, i)); + if (!newoperator) { return NULL; } - for (i = 1; i < NCH(n); i += 2) { - cmpop_ty newoperator; - - newoperator = ast_for_comp_op(c, CHILD(n, i)); - if (!newoperator) { - return NULL; - } - - expression = ast_for_expr(c, CHILD(n, i + 1)); - if (!expression) { - return NULL; - } - - asdl_seq_SET(ops, i / 2, newoperator); - asdl_seq_SET(cmps, i / 2, expression); - } - expression = ast_for_expr(c, CHILD(n, 0)); + + expression = ast_for_expr(c, CHILD(n, i + 1)); if (!expression) { return NULL; } - return Compare(expression, ops, cmps, LINENO(n), - n->n_col_offset, c->c_arena); + asdl_seq_SET(ops, i / 2, newoperator); + asdl_seq_SET(cmps, i / 2, expression); } - break; + expression = ast_for_expr(c, CHILD(n, 0)); + if (!expression) { + return NULL; + } + + return Compare(expression, ops, cmps, LINENO(n), + n->n_col_offset, c->c_arena); case star_expr: return ast_for_starred(c, n); @@ -2614,10 +2605,7 @@ case shift_expr: case arith_expr: case term: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } + assert(NCH(n) > 1); return ast_for_binop(c, n); case yield_expr: { node *an = NULL; @@ -2642,13 +2630,14 @@ return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena); } case factor: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } + assert(NCH(n) > 1); return ast_for_factor(c, n); case power: return ast_for_power(c, n); + case atom: + return ast_for_atom(c, n); + case atom_expr: + return ast_for_atom_expr(c, n); default: PyErr_Format(PyExc_SystemError, "unhandled expr: %d", TYPE(n)); return NULL; @@ -2829,23 +2818,30 @@ { /* testlist_comp: test (comp_for | (',' test)* [',']) */ /* testlist: test (',' test)* [','] */ + asdl_seq *tmp; assert(NCH(n) > 0); - if (TYPE(n) == testlist_comp) { - if (NCH(n) > 1) + switch (TYPE(n)) { + case testlist_comp: + if (NCH(n) > 1) { assert(TYPE(CHILD(n, 1)) != comp_for); + break; + } + n = CHILD(n, 0); + /* Fallthrough */ + case_EXPR: + return ast_for_expr(c, n); + case testlist: + case testlist_star_expr: + assert(NCH(n) > 1); + break; + default: + PyErr_SetString(PyExc_SystemError, "unexpected testlist type"); + return NULL; } - else { - assert(TYPE(n) == testlist || - TYPE(n) == testlist_star_expr); - } - if (NCH(n) == 1) - return ast_for_expr(c, CHILD(n, 0)); - else { - asdl_seq *tmp = seq_for_testlist(c, n); - if (!tmp) - return NULL; - return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena); - } + tmp = seq_for_testlist(c, n); + if (!tmp) + return NULL; + return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -3053,8 +3049,9 @@ dotted_name: NAME ('.' NAME)* */ identifier str, name; - - loop: + node *asname_node; + alias_ty a; + switch (TYPE(n)) { case import_as_name: { node *name_node = CHILD(n, 0); @@ -3077,24 +3074,18 @@ return alias(name, str, c->c_arena); } case dotted_as_name: - if (NCH(n) == 1) { - n = CHILD(n, 0); - goto loop; - } - else { - node *asname_node = CHILD(n, 2); - alias_ty a = alias_for_import_name(c, CHILD(n, 0), 0); - if (!a) - return NULL; - assert(!a->asname); - a->asname = NEW_IDENTIFIER(asname_node); - if (!a->asname) - return NULL; - if (forbidden_name(c, a->asname, asname_node, 0)) - return NULL; - return a; - } - break; + assert(NCH(n) > 1); + asname_node = CHILD(n, 2); + a = alias_for_import_name(c, CHILD(n, 0), 0); + if (!a) + return NULL; + assert(!a->asname); + a->asname = NEW_IDENTIFIER(asname_node); + if (!a->asname) + return NULL; + if (forbidden_name(c, a->asname, asname_node, 0)) + return NULL; + return a; case dotted_name: if (NCH(n) == 1) { node *name_node = CHILD(n, 0); @@ -3356,14 +3347,13 @@ int i, total, num, end, pos = 0; node *ch; - REQ(n, suite); + assert(TYPE(n) == suite || TYPE(n) == simple_stmt); total = num_stmts(n); seq = _Py_asdl_seq_new(total, c->c_arena); if (!seq) return NULL; - if (TYPE(CHILD(n, 0)) == simple_stmt) { - n = CHILD(n, 0); + if (TYPE(n) == simple_stmt) { /* simple_stmt always ends with a NEWLINE, and may have a trailing SEMI */ @@ -3382,7 +3372,7 @@ else { for (i = 2; i < (NCH(n) - 1); i++) { ch = CHILD(n, i); - REQ(ch, stmt); + assert(TYPE(ch) == stmt || TYPE(ch) == simple_stmt || TYPE(ch) == compound_stmt); num = num_stmts(ch); if (num == 1) { /* small_stmt or compound_stmt with only one child */ @@ -3393,7 +3383,6 @@ } else { int j; - ch = CHILD(ch, 0); REQ(ch, simple_stmt); for (j = 0; j < NCH(ch); j += 2) { /* statement terminates with a semi-colon ';' */ @@ -3624,7 +3613,7 @@ { /* except_clause: 'except' [test ['as' test]] */ REQ(exc, except_clause); - REQ(body, suite); + assert(TYPE(body) == suite || TYPE(body) == simple_stmt); if (NCH(exc) == 1) { asdl_seq *suite_seq = ast_for_suite(c, body); @@ -3856,10 +3845,7 @@ static stmt_ty ast_for_stmt(struct compiling *c, const node *n) { - if (TYPE(n) == stmt) { - assert(NCH(n) == 1); - n = CHILD(n, 0); - } + assert(TYPE(n) != stmt); // must have been compressed if (TYPE(n) == simple_stmt) { assert(num_stmts(n) == 1); n = CHILD(n, 0); @@ -5065,3 +5051,39 @@ FstringParser_Dealloc(&state); return NULL; } + +void PyNode_Compress(node* n) { + if (NCH(n) == 1) { + node* ch; + switch (TYPE(n)) { + case suite: + case comp_op: + case subscript: + case atom_expr: + case power: + case factor: + case expr: + case xor_expr: + case and_expr: + case shift_expr: + case arith_expr: + case term: + case comparison: + case testlist_star_expr: + case testlist: + case test: + case test_nocond: + case or_test: + case and_test: + case not_test: + case dotted_as_name: + case stmt: + if (STR(n) != NULL) + PyObject_FREE(STR(n)); + ch = CHILD(n, 0); + *n = *ch; + // All grandchildren are now adopted; don't need to free them, so no need for PyNode_Free + PyObject_FREE(ch); + } + } +}