diff -r 73aa6d672b81 -r 5c078eb8da39 Doc/library/dis.rst --- a/Doc/library/dis.rst Sun Feb 23 18:01:08 2014 -0500 +++ b/Doc/library/dis.rst Mon Feb 24 01:27:06 2014 +0100 @@ -566,6 +566,16 @@ denoting nested loops, try statements, and such. +.. opcode:: POP_BLOCK_AND_PUSH_VALUE + + Removes one block from the block stack, while preserving the top value. + Per frame, there is a stack of blocks, denoting nested loops, try + statements, and such. The top value is pushed onto the stack after the + rest of the block's values are popped off. + + .. versionadded:: 3.5 + + .. opcode:: POP_EXCEPT Removes one block from the block stack. The popped block must be an exception @@ -574,6 +584,18 @@ last three popped values are used to restore the exception state. +.. opcode:: POP_EXCEPT_AND_PUSH_VALUE + + Removes one block from the block stack, while preserving the top value. + The popped block must be an exception handler block, as implicitly + created when entering an except handler. In addition to popping + extraneous values from the frame stack, the last three popped values are + used to restore the exception state. The previous top is pushed onto the + stack afterwards. + + .. versionadded:: 3.5 + + .. opcode:: END_FINALLY Terminates a :keyword:`finally` clause. The interpreter recalls whether the diff -r 73aa6d672b81 -r 5c078eb8da39 Grammar/Grammar --- a/Grammar/Grammar Sun Feb 23 18:01:08 2014 -0500 +++ b/Grammar/Grammar Mon Feb 24 01:27:06 2014 +0100 @@ -100,11 +100,14 @@ term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom trailer* ['**' factor] -atom: ('(' [yield_expr|testlist_comp] ')' | +atom: ('(' [yield_expr|testlist_except] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +testlist_except: (test|star_expr) ( except_trailer + | comp_for + | (',' (test|star_expr))* [','] ) trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] subscript: test | [test] ':' [test] [sliceop] @@ -121,10 +124,11 @@ |'**' test) # The reason that keywords are test nodes instead of NAME is that using NAME # results in an ambiguity. ast.c makes sure it's a NAME. -argument: test [comp_for] | test '=' test # Really [keyword '='] test +argument: test [except_trailer | comp_for] | test '=' test # Really [keyword '='] test comp_iter: comp_for | comp_if comp_for: 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' test_nocond [comp_iter] +except_trailer: 'except' test ':' test # not used in grammar, but may appear in "node" passed from Parser to Compiler encoding_decl: NAME diff -r 73aa6d672b81 -r 5c078eb8da39 Include/opcode.h --- a/Include/opcode.h Sun Feb 23 18:01:08 2014 -0500 +++ b/Include/opcode.h Mon Feb 24 01:27:06 2014 +0100 @@ -60,10 +60,10 @@ #define INPLACE_OR 79 #define BREAK_LOOP 80 #define WITH_CLEANUP 81 - +#define POP_BLOCK_AND_PUSH_VALUE 82 #define RETURN_VALUE 83 #define IMPORT_STAR 84 - +#define POP_EXCEPT_AND_PUSH_VALUE 85 #define YIELD_VALUE 86 #define POP_BLOCK 87 #define END_FINALLY 88 diff -r 73aa6d672b81 -r 5c078eb8da39 Lib/opcode.py --- a/Lib/opcode.py Sun Feb 23 18:01:08 2014 -0500 +++ b/Lib/opcode.py Mon Feb 24 01:27:06 2014 +0100 @@ -109,10 +109,10 @@ def_op('INPLACE_OR', 79) def_op('BREAK_LOOP', 80) def_op('WITH_CLEANUP', 81) - +def_op('POP_BLOCK_AND_PUSH_VALUE', 82) def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) - +def_op('POP_EXCEPT_AND_PUSH_VALUE', 85) def_op('YIELD_VALUE', 86) def_op('POP_BLOCK', 87) def_op('END_FINALLY', 88) diff -r 73aa6d672b81 -r 5c078eb8da39 Lib/test/test_grammar.py --- a/Lib/test/test_grammar.py Sun Feb 23 18:01:08 2014 -0500 +++ b/Lib/test/test_grammar.py Mon Feb 24 01:27:06 2014 +0100 @@ -985,6 +985,54 @@ self.assertFalse((False is 2) is 3) self.assertFalse(False is 2 is 3) + def test_except_expr(self): + self.assertEqual((1 except Exception: 2), 1) + # The exception type shouldn't be evaluated unless an exception is + # raised. + self.assertEqual((1 except NotThere: 2), 1) + # And neither should the default value. + self.assertEqual((1 except Exception: NotThere), 1) + # Order of evaluation test: strictly left to right. + it = iter([5, TypeError, 10]) + self.assertEqual((next(it) + "" except next(it): next(it)), 10) + # The exception type can be a tuple of exceptions, of course. + self.assertEqual((1 + "" except (RuntimeError, ZeroDivisionError, + TypeError, ValueError): 2), 2) + # ... and actually any kind of expression. + self.assertEqual((1 + "" except RuntimeError and TypeError: 2), 2) + + # Matching exceptions should be caught. + self.assertEqual(("".notthere except AttributeError: 2), 2) + # Even when they're baseclasses. + self.assertEqual(("".notthere except Exception: 2), 2) + # Nonmatching ones shouldn't. + with self.assertRaises(AttributeError): + ("".notthere except ValueError: 2) + + # Using an except expression in an except or finally block shouldn't + # wipe the previous exception or affect their execution. + tracer = [] + try: + with self.assertRaises(AttributeError): + try: + "".notthere + except AttributeError: + tracer.append(sys.exc_info()[1]) + tracer.append(1 / 0 except ZeroDivisionError: 1) + tracer.append(sys.exc_info()[1]) + raise + finally: + tracer.append(sys.exc_info()[1]) + tracer.append(1.0 & 1 except TypeError: 4) + tracer.append(sys.exc_info()[1]) + self.assertIs(tracer[0], tracer[2]) + self.assertEqual(tracer[1], 1) + self.assertIs(tracer[0], tracer[3]) + self.assertIs(tracer[3], tracer[5]) + self.assertEqual(tracer[4], 4) + finally: + del tracer + def test_main(): run_unittest(TokenTests, GrammarTests) diff -r 73aa6d672b81 -r 5c078eb8da39 Lib/test/test_tokenize.py --- a/Lib/test/test_tokenize.py Sun Feb 23 18:01:08 2014 -0500 +++ b/Lib/test/test_tokenize.py Mon Feb 24 01:27:06 2014 +0100 @@ -578,15 +578,9 @@ >>> tempdir = os.path.dirname(f) or os.curdir >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py")) -Tokenize is broken on test_pep3131.py because regular expressions are -broken on the obscure unicode identifiers in it. *sigh* -With roundtrip extended to test the 5-tuple mode of untokenize, -7 more testfiles fail. Remove them also until the failure is diagnosed. - +tokenize is broken on test_pep3131.py because regular expressions are broken on +the obscure unicode identifiers in it. *sigh* >>> testfiles.remove(os.path.join(tempdir, "test_pep3131.py")) - >>> for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'): - ... testfiles.remove(os.path.join(tempdir, "test_%s.py") % f) - ... >>> if not support.is_resource_enabled("cpu"): ... testfiles = random.sample(testfiles, 10) ... @@ -665,39 +659,21 @@ def roundtrip(f): """ Test roundtrip for `untokenize`. `f` is an open file or a string. - The source code in f is tokenized to both 5- and 2-tuples. - Both sequences are converted back to source code via - tokenize.untokenize(), and the latter tokenized again to 2-tuples. - The test fails if the 3 pair tokenizations do not match. - - When untokenize bugs are fixed, untokenize with 5-tuples should - reproduce code that does not contain a backslash continuation - following spaces. A proper test should test this. - - This function would be more useful for correcting bugs if it reported - the first point of failure, like assertEqual, rather than just - returning False -- or if it were only used in unittests and not - doctest and actually used assertEqual. + The source code in f is tokenized, converted back to source code via + tokenize.untokenize(), and tokenized again from the latter. The test + fails if the second tokenization doesn't match the first. """ - # Get source code and original tokenizations if isinstance(f, str): - code = f.encode('utf-8') - else: - code = f.read() + f = BytesIO(f.encode('utf-8')) + try: + token_list = list(tokenize(f.readline)) + finally: f.close() - readline = iter(code.splitlines(keepends=True)).__next__ - tokens5 = list(tokenize(readline)) - tokens2 = [tok[:2] for tok in tokens5] - # Reproduce tokens2 from pairs - bytes_from2 = untokenize(tokens2) - readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__ - tokens2_from2 = [tok[:2] for tok in tokenize(readline2)] - # Reproduce tokens2 from 5-tuples - bytes_from5 = untokenize(tokens5) - readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__ - tokens2_from5 = [tok[:2] for tok in tokenize(readline5)] - # Compare 3 versions - return tokens2 == tokens2_from2 == tokens2_from5 + tokens1 = [tok[:2] for tok in token_list] + new_bytes = untokenize(tokens1) + readline = (line for line in new_bytes.splitlines(keepends=True)).__next__ + tokens2 = [tok[:2] for tok in tokenize(readline)] + return tokens1 == tokens2 # This is an example from the docs, set up as a doctest. def decistmt(s): diff -r 73aa6d672b81 -r 5c078eb8da39 Parser/Python.asdl --- a/Parser/Python.asdl Sun Feb 23 18:01:08 2014 -0500 +++ b/Parser/Python.asdl Mon Feb 24 01:27:06 2014 +0100 @@ -58,6 +58,7 @@ | SetComp(expr elt, comprehension* generators) | DictComp(expr key, expr value, comprehension* generators) | GeneratorExp(expr elt, comprehension* generators) + | ExceptExp(expr body, expr etype, expr value) -- the grammar constrains where yield expressions can occur | Yield(expr? value) | YieldFrom(expr value) diff -r 73aa6d672b81 -r 5c078eb8da39 Python/ast.c --- a/Python/ast.c Sun Feb 23 18:01:08 2014 -0500 +++ b/Python/ast.c Mon Feb 24 01:27:06 2014 +0100 @@ -187,6 +187,10 @@ case Lambda_kind: return validate_arguments(exp->v.Lambda.args) && validate_expr(exp->v.Lambda.body, Load); + case ExceptExp_kind: + return validate_expr(exp->v.ExceptExp.body, Load) && + validate_expr(exp->v.ExceptExp.etype, Load) && + validate_expr(exp->v.ExceptExp.value, Load); case IfExp_kind: return validate_expr(exp->v.IfExp.test, Load) && validate_expr(exp->v.IfExp.body, Load) && @@ -1097,7 +1101,8 @@ asdl_seq *seq; expr_ty expression; int i; - assert(TYPE(n) == testlist || TYPE(n) == testlist_star_expr || TYPE(n) == testlist_comp); + assert(TYPE(n) == testlist || TYPE(n) == testlist_star_expr || + TYPE(n) == testlist_comp || TYPE(n) == testlist_except); seq = _Py_asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); if (!seq) @@ -1589,6 +1594,36 @@ c->c_arena); } +static expr_ty +ast_for_except_trailer(struct compiling *c, const node *n) +{ + /* testlist_except: (test|star_expr) ( except_trailer + | comp_for + | (',' (test|star_expr))* [','] ) + except_trailer: 'except' test ':' test + */ + expr_ty body, etype, value; + const node *subn; + + assert(TYPE(n) == testlist_except || TYPE(n) == argument); + assert(NCH(n) == 2); + subn = CHILD(n, 1); + REQ(subn, except_trailer); + assert(NCH(subn) == 4); + + body = ast_for_expr(c, CHILD(n, 0)); + if (!body) + return NULL; + etype = ast_for_expr(c, CHILD(subn, 1)); + if (!etype) + return NULL; + value = ast_for_expr(c, CHILD(subn, 3)); + if (!value) + return NULL; + return ExceptExp(body, etype, value, LINENO(n), n->n_col_offset, + c->c_arena); +} + /* Count the number of 'for' loops in a comprehension. @@ -1730,7 +1765,7 @@ static expr_ty ast_for_itercomp(struct compiling *c, const node *n, int type) { - /* testlist_comp: test ( comp_for | (',' test)* [','] ) + /* testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) argument: [test '='] test [comp_for] # Really [keyword '='] test */ expr_ty elt; asdl_seq *comps; @@ -1782,7 +1817,8 @@ static expr_ty ast_for_genexp(struct compiling *c, const node *n) { - assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument)); + assert(TYPE(n) == (testlist_comp) || TYPE(n) == testlist_except || + TYPE(n) == (argument)); return ast_for_itercomp(c, n, COMP_GENEXP); } @@ -1888,9 +1924,16 @@ if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); - /* testlist_comp: test ( comp_for | (',' test)* [','] ) */ - if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for)) - return ast_for_genexp(c, ch); + /* testlist_except: test ( except_trailer + | comp_for + | (',' test)* [','] ) */ + if (NCH(ch) > 1) { + if (TYPE(CHILD(ch, 1)) == except_trailer) + return ast_for_except_trailer(c, ch); + if (TYPE(CHILD(ch, 1)) == comp_for) + return ast_for_genexp(c, ch); + assert(TYPE(CHILD(ch, 1)) == COMMA); + } return ast_for_testlist(c, ch); case LSQB: /* list (or list comprehension) */ @@ -2423,7 +2466,7 @@ argument: [test '='] (test) [comp_for] # Really [keyword '='] test */ - int i, nargs, nkeywords, ngens; + int i, nargs, nkeywords, ngens, nexcepts; asdl_seq *args; asdl_seq *keywords; expr_ty vararg = NULL, kwarg = NULL; @@ -2433,11 +2476,16 @@ nargs = 0; nkeywords = 0; ngens = 0; + nexcepts = 0; for (i = 0; i < NCH(n); i++) { node *ch = CHILD(n, i); if (TYPE(ch) == argument) { if (NCH(ch) == 1) nargs++; + else if (TYPE(CHILD(ch, 1)) == except_trailer) { + nargs++; + nexcepts++; + } else if (TYPE(CHILD(ch, 1)) == comp_for) ngens++; else @@ -2450,6 +2498,12 @@ return NULL; } + if (nexcepts > 1 || (nexcepts && (nargs > 1 || ngens || nkeywords))) { + ast_error(c, n, "Except expression must be parenthesized " + "if not sole argument"); + return NULL; + } + if (nargs + nkeywords + ngens > 255) { ast_error(c, n, "more than 255 arguments"); return NULL; @@ -2483,6 +2537,12 @@ return NULL; asdl_seq_SET(args, nargs++, e); } + else if (TYPE(CHILD(ch, 1)) == except_trailer) { + e = ast_for_except_trailer(c, ch); + if (!e) + return NULL; + asdl_seq_SET(args, nargs++, e); + } else if (TYPE(CHILD(ch, 1)) == comp_for) { e = ast_for_genexp(c, ch); if (!e) @@ -2552,9 +2612,11 @@ /* testlist_comp: test (comp_for | (',' test)* [',']) */ /* testlist: test (',' test)* [','] */ assert(NCH(n) > 0); - if (TYPE(n) == testlist_comp) { - if (NCH(n) > 1) + if (TYPE(n) == testlist_comp || TYPE(n) == testlist_except) { + if (NCH(n) > 1) { assert(TYPE(CHILD(n, 1)) != comp_for); + assert(TYPE(CHILD(n, 1)) != except_trailer); + } } else { assert(TYPE(n) == testlist || diff -r 73aa6d672b81 -r 5c078eb8da39 Python/ceval.c --- a/Python/ceval.c Sun Feb 23 18:01:08 2014 -0500 +++ b/Python/ceval.c Mon Feb 24 01:27:06 2014 +0100 @@ -1943,12 +1943,35 @@ DISPATCH(); } + TARGET(POP_EXCEPT_AND_PUSH_VALUE) { + /* Like POP_EXCEPT, except pop the top off the stack first, then + push that after unwinding the handler. */ + PyObject *value = POP(); + PyTryBlock *b = PyFrame_BlockPop(f); + if (b->b_type != EXCEPT_HANDLER) { + PyErr_SetString(PyExc_SystemError, + "popped block is not an except handler"); + goto error; + } + UNWIND_EXCEPT_HANDLER(b); + PUSH(value); + DISPATCH(); + } + TARGET(POP_BLOCK) { PyTryBlock *b = PyFrame_BlockPop(f); UNWIND_BLOCK(b); DISPATCH(); } + TARGET(POP_BLOCK_AND_PUSH_VALUE) { + PyObject *value = POP(); + PyTryBlock *b = PyFrame_BlockPop(f); + UNWIND_BLOCK(b); + PUSH(value); + DISPATCH(); + } + PREDICTED(END_FINALLY); TARGET(END_FINALLY) { PyObject *status = POP(); diff -r 73aa6d672b81 -r 5c078eb8da39 Python/compile.c --- a/Python/compile.c Sun Feb 23 18:01:08 2014 -0500 +++ b/Python/compile.c Mon Feb 24 01:27:06 2014 +0100 @@ -941,8 +941,12 @@ return -1; case POP_BLOCK: return 0; + case POP_BLOCK_AND_PUSH_VALUE: + return 0; case POP_EXCEPT: return 0; /* -3 except if bad bytecode */ + case POP_EXCEPT_AND_PUSH_VALUE: + return 0; /* -3 except if bad bytecode */ case END_FINALLY: return -1; /* or -2 or -3 if exception occurred */ @@ -2297,6 +2301,74 @@ return compiler_try_except(c, s); } +/* Compile the except expression, a very limited form of try/except. */ +static int +compiler_exceptexp(struct compiler *c, expr_ty e) +{ + basicblock *body, *except, *value, *cleanup, *end; + + assert(e->kind == ExceptExp_kind); + body = compiler_new_block(c); + if (body == NULL) + return 0; + except = compiler_new_block(c); + if (except == NULL) + return 0; + value = compiler_new_block(c); + if (value== NULL) + return 0; + cleanup = compiler_new_block(c); + if (cleanup == NULL) + return 0; + end = compiler_new_block(c); + if (end == NULL) + return 0; + ADDOP_JREL(c, SETUP_EXCEPT, except); + compiler_use_next_block(c, body); + if (!compiler_push_fblock(c, EXCEPT, body)) + return 0; + VISIT(c, expr, e->v.ExceptExp.body); + ADDOP(c, POP_BLOCK_AND_PUSH_VALUE); + compiler_pop_fblock(c, EXCEPT, body); + ADDOP_JREL(c, JUMP_FORWARD, end); + compiler_use_next_block(c, except); + /* An exception occurred; tb, val and exc are pushed onto the stack (in + that order.) They need to stay on the stack as long as we may need to + re-raise the exception; DUP the exception type so we can COMPARE_OP + it. We don't have any need for tb or val, since the except expression + doesn't allow capturing either. + */ + ADDOP(c, DUP_TOP); + VISIT(c, expr, e->v.ExceptExp.etype); + ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, cleanup); + /* Exception matches; clear the error by poppping tb, val and exc from + the valuestack and pushing the replacement value, then jump to the + end of the expression.*/ + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + compiler_use_next_block(c, value); + if (!compiler_push_fblock(c, FINALLY_TRY, value)) + return 0; + VISIT(c, expr, e->v.ExceptExp.value); + /* When execution jumps to a handler, the previous exception triplet is + pushed onto the stack first (or Nones if there wasn't one.) Normally + when exiting a handler the previous exception is restored by popping + everything op to and including them off the stack, which is fine + since the except block can't return a value. We need to produce a + value, so we need to pop the value off the stack, then restore the + previous exception, then push the value onto the stack again. + */ + ADDOP(c, POP_EXCEPT_AND_PUSH_VALUE); + compiler_pop_fblock(c, FINALLY_TRY, value); + ADDOP_JREL(c, JUMP_FORWARD, end); + /* End the exception handling, re-raising the exception. */ + compiler_use_next_block(c, cleanup); + ADDOP(c, END_FINALLY); + compiler_use_next_block(c, end); + return 1; +} static int compiler_import_as(struct compiler *c, identifier name, identifier asname) @@ -3424,6 +3496,8 @@ return compiler_lambda(c, e); case IfExp_kind: return compiler_ifexp(c, e); + case ExceptExp_kind: + return compiler_exceptexp(c, e); case Dict_kind: n = asdl_seq_LEN(e->v.Dict.values); /* BUILD_MAP parameter is only used to preallocate the dictionary, diff -r 73aa6d672b81 -r 5c078eb8da39 Python/opcode_targets.h --- a/Python/opcode_targets.h Sun Feb 23 18:01:08 2014 -0500 +++ b/Python/opcode_targets.h Mon Feb 24 01:27:06 2014 +0100 @@ -81,10 +81,10 @@ &&TARGET_INPLACE_OR, &&TARGET_BREAK_LOOP, &&TARGET_WITH_CLEANUP, - &&_unknown_opcode, + &&TARGET_POP_BLOCK_AND_PUSH_VALUE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, - &&_unknown_opcode, + &&TARGET_POP_EXCEPT_AND_PUSH_VALUE, &&TARGET_YIELD_VALUE, &&TARGET_POP_BLOCK, &&TARGET_END_FINALLY, diff -r 73aa6d672b81 -r 5c078eb8da39 Python/symtable.c --- a/Python/symtable.c Sun Feb 23 18:01:08 2014 -0500 +++ b/Python/symtable.c Mon Feb 24 01:27:06 2014 +0100 @@ -1407,6 +1407,11 @@ VISIT_QUIT(st, 0); break; } + case ExceptExp_kind: + VISIT(st, expr, e->v.ExceptExp.body); + VISIT(st, expr, e->v.ExceptExp.etype); + VISIT(st, expr, e->v.ExceptExp.value); + break; case IfExp_kind: VISIT(st, expr, e->v.IfExp.test); VISIT(st, expr, e->v.IfExp.body);