Index: Python/import.c =================================================================== --- Python/import.c (revision 70362) +++ Python/import.c (working copy) @@ -67,16 +67,16 @@ Python 2.5b3: 62101 (fix wrong code: for x, in ...) Python 2.5b3: 62111 (fix wrong code: x += yield) Python 2.5c1: 62121 (fix wrong lnotab with for loops and - storing constants that should have been removed) + storing constants that should have been removed) Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) Python 2.6a1: 62161 (WITH_CLEANUP optimization) Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) Python 2.7a0: 62181 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -. + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 2.7a0: 62191 (PEP 377 - tweaked with statement semantics) */ -#define MAGIC (62181 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (62191 | ((long)'\r'<<16) | ((long)'\n'<<24)) /* Magic word as global; note that _PyImport_Init() can change the value of this global to accommodate for alterations of how the Index: Python/marshal.c =================================================================== --- Python/marshal.c (revision 70362) +++ Python/marshal.c (working copy) @@ -576,7 +576,7 @@ n = r_long(p); if (n < -INT_MAX || n > INT_MAX) { PyErr_SetString(PyExc_ValueError, - "bad marshal data"); + "bad marshal data (long size out of range)"); retval = NULL; break; } @@ -592,7 +592,7 @@ if (digit < 0) { Py_DECREF(ob); PyErr_SetString(PyExc_ValueError, - "bad marshal data"); + "bad marshal data (negative digit in long)"); ob = NULL; break; } @@ -709,7 +709,7 @@ case TYPE_STRING: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); retval = NULL; break; } @@ -738,7 +738,7 @@ case TYPE_STRINGREF: n = r_long(p); if (n < 0 || n >= PyList_GET_SIZE(p->strings)) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)"); retval = NULL; break; } @@ -754,7 +754,7 @@ n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); retval = NULL; break; } @@ -780,7 +780,7 @@ case TYPE_TUPLE: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); retval = NULL; break; } @@ -794,7 +794,7 @@ if ( v2 == NULL ) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data"); + "NULL object in marshal data for tuple"); Py_DECREF(v); v = NULL; break; @@ -807,7 +807,7 @@ case TYPE_LIST: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); retval = NULL; break; } @@ -821,7 +821,7 @@ if ( v2 == NULL ) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data"); + "NULL object in marshal data for list"); Py_DECREF(v); v = NULL; break; @@ -859,7 +859,7 @@ case TYPE_FROZENSET: n = r_long(p); if (n < 0 || n > INT_MAX) { - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); retval = NULL; break; } @@ -873,7 +873,7 @@ if ( v2 == NULL ) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data"); + "NULL object in marshal data for set"); Py_DECREF(v); v = NULL; break; @@ -973,7 +973,7 @@ default: /* Bogus data got written, which isn't ideal. This will let you keep working and recover. */ - PyErr_SetString(PyExc_ValueError, "bad marshal data"); + PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)"); retval = NULL; break; @@ -992,7 +992,7 @@ } v = r_object(p); if (v == NULL && !PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, "NULL object in marshal data"); + PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object"); return v; } Index: Python/compile.c =================================================================== --- Python/compile.c (revision 70362) +++ Python/compile.c (working copy) @@ -2802,72 +2802,123 @@ } /* - Implements the with statement from PEP 343. + Implements the with statement from PEP 343 and PEP 377. - The semantics outlined in that PEP are as follows: + The semantics outlined in those PEPs are as follows: - with EXPR as VAR: - BLOCK - + with EXPR as VAR: + BLOCK + It is implemented roughly as: - - context = EXPR - exit = context.__exit__ # not calling it - value = context.__enter__() - try: - VAR = value # if VAR present in the syntax - BLOCK - finally: - if an exception was raised: - exc = copy of (exception, instance, traceback) - else: - exc = (None, None, None) - exit(*exc) + + mgr = (EXPR) + exit = mgr.__exit__ # Not calling it yet + try: + value = mgr.__enter__() + except SkipStatement: + VAR = StatementSkipped + # Only if "as VAR" is present and + # VAR is a single name + # If VAR is a tuple of names, then StatementSkipped + # will be assigned to each name in the tuple + else: + call_exit = True + try: + try: + VAR = value # Only if "as VAR" is present + BLOCK + except: + # The exceptional case is handled here + call_exit = False + if not exit(*sys.exc_info()): + raise + # The exception is swallowed if exit() returns true + finally: + # The normal and non-local-goto cases are handled here + if call_exit: + exit(None, None, None) */ static int compiler_with(struct compiler *c, stmt_ty s) { - static identifier enter_attr, exit_attr; - basicblock *block, *finally; + static identifier enter_attr, exit_attr, skip_exc, skipped; + basicblock *outer_try, *outer_except, *outer_reraise; + basicblock *outer_else, *outer_end; + basicblock *inner_try, *inner_finally; identifier tmpvalue = NULL; + expr_ty vars_expr; assert(s->kind == With_kind); if (!enter_attr) { - enter_attr = PyString_InternFromString("__enter__"); - if (!enter_attr) - return 0; + enter_attr = PyString_InternFromString("__enter__"); + if (!enter_attr) + return 0; } if (!exit_attr) { - exit_attr = PyString_InternFromString("__exit__"); - if (!exit_attr) - return 0; + exit_attr = PyString_InternFromString("__exit__"); + if (!exit_attr) + return 0; } + if (!skip_exc) { + skip_exc = PyString_InternFromString("SkipStatement"); + if (!skip_exc) + return 0; + } - block = compiler_new_block(c); - finally = compiler_new_block(c); - if (!block || !finally) - return 0; + if (!skipped) { + skipped = PyString_InternFromString("StatementSkipped"); + if (!skipped) + return 0; + } - if (s->v.With.optional_vars) { - /* Create a temporary variable to hold context.__enter__(). - We need to do this rather than preserving it on the stack - because SETUP_FINALLY remembers the stack level. - We need to do the assignment *inside* the try/finally - so that context.__exit__() is called when the assignment - fails. But we need to call context.__enter__() *before* - the try/finally so that if it fails we won't call - context.__exit__(). - */ - tmpvalue = compiler_new_tmpname(c); - if (tmpvalue == NULL) - return 0; - PyArena_AddPyObject(c->c_arena, tmpvalue); + + outer_try = compiler_new_block(c); + outer_except = compiler_new_block(c); + outer_reraise = compiler_new_block(c); + outer_else = compiler_new_block(c); + inner_try = compiler_new_block(c); + inner_finally = compiler_new_block(c); + outer_end = compiler_new_block(c); + if (!outer_try || !outer_except || !outer_reraise || !outer_else || + !inner_try || !inner_finally || !outer_end) + return 0; + + vars_expr = s->v.With.optional_vars; + if (vars_expr) { + /* Create a temporary variable to hold context.__enter__(). + We need to do this rather than preserving it on the stack + because SETUP_FINALLY remembers the stack level. + We need to do the assignment *inside* the try/finally + so that context.__exit__() is called if the assignment + fails. But we need to call context.__enter__() *before* + the try/finally so that if it fails we won't call + context.__exit__(). + */ + tmpvalue = compiler_new_tmpname(c); + if (tmpvalue == NULL) + return 0; + PyArena_AddPyObject(c->c_arena, tmpvalue); } /* Evaluate EXPR */ VISIT(c, expr, s->v.With.context_expr); + /* XXX (ncoghlan): The with statement setup code has reached a + point were it is just crying out for a SETUP_WITH opcode. + Everything from here to the "BLOCK code" comment is bytecode + generated entirely by the compiler - that is a *lot* of trips + around the eval loop which could conceivably be avoided + */ + /* Set up the outer try/except that allows context.__enter__ to + skip the body of the with statement and resume execution + with the following statement + */ + ADDOP_JREL(c, SETUP_EXCEPT, outer_except); + compiler_use_next_block(c, outer_try); + if (!compiler_push_fblock(c, EXCEPT, outer_try)) + return 0; + /* Squirrel away context.__exit__ by stuffing it under context */ ADDOP(c, DUP_TOP); ADDOP_O(c, LOAD_ATTR, exit_attr, names); @@ -2877,43 +2928,77 @@ ADDOP_O(c, LOAD_ATTR, enter_attr, names); ADDOP_I(c, CALL_FUNCTION, 0); - if (s->v.With.optional_vars) { - /* Store it in tmpvalue */ - if (!compiler_nameop(c, tmpvalue, Store)) - return 0; + if (vars_expr) { + /* Store result in tmpvalue */ + if (!compiler_nameop(c, tmpvalue, Store)) + return 0; + } else { + /* Discard result from context.__enter__() */ + ADDOP(c, POP_TOP); } - else { - /* Discard result from context.__enter__() */ - ADDOP(c, POP_TOP); + + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, EXCEPT, outer_try); + ADDOP_JREL(c, JUMP_FORWARD, outer_else); + + /* Specific exception handler for SkipStatement */ + compiler_use_next_block(c, outer_except); + ADDOP(c, DUP_TOP); + ADDOP_O(c, LOAD_GLOBAL, skip_exc, names); + ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH); + ADDOP_JABS(c, POP_JUMP_IF_FALSE, outer_reraise); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + ADDOP(c, POP_TOP); + if (vars_expr) { + /* Assign StatementSkipped to all VAR targets */ + ADDOP_O(c, LOAD_GLOBAL, skipped, names); + if (vars_expr->kind == Tuple_kind) { + int i; + int n = asdl_seq_LEN(vars_expr->v.Tuple.elts); + for (i = 1; i < n; i++) { + ADDOP(c, DUP_TOP); + } + ADDOP_I(c, BUILD_TUPLE, n); + } + VISIT(c, expr, s->v.With.optional_vars); } + ADDOP_JREL(c, JUMP_FORWARD, outer_end); - /* Start the try block */ - ADDOP_JREL(c, SETUP_FINALLY, finally); + /* Jump target for exceptions that don't match SkipStatement */ + compiler_use_next_block(c, outer_reraise); + ADDOP(c, END_FINALLY); - compiler_use_next_block(c, block); - if (!compiler_push_fblock(c, FINALLY_TRY, block)) { - return 0; + /* Start the outer else clause */ + compiler_use_next_block(c, outer_else); + + /* Start the inner try block */ + ADDOP_JREL(c, SETUP_FINALLY, inner_finally); + + compiler_use_next_block(c, inner_try); + if (!compiler_push_fblock(c, FINALLY_TRY, inner_try)) { + return 0; } if (s->v.With.optional_vars) { - /* Bind saved result of context.__enter__() to VAR */ - if (!compiler_nameop(c, tmpvalue, Load) || - !compiler_nameop(c, tmpvalue, Del)) - return 0; - VISIT(c, expr, s->v.With.optional_vars); + /* Bind saved result of context.__enter__() to VAR */ + if (!compiler_nameop(c, tmpvalue, Load) || + !compiler_nameop(c, tmpvalue, Del)) + return 0; + VISIT(c, expr, s->v.With.optional_vars); } /* BLOCK code */ VISIT_SEQ(c, stmt, s->v.With.body); - /* End of try block; start the finally block */ + /* End of inner try block; start the finally block */ ADDOP(c, POP_BLOCK); - compiler_pop_fblock(c, FINALLY_TRY, block); + compiler_pop_fblock(c, FINALLY_TRY, inner_try); ADDOP_O(c, LOAD_CONST, Py_None, consts); - compiler_use_next_block(c, finally); - if (!compiler_push_fblock(c, FINALLY_END, finally)) - return 0; + compiler_use_next_block(c, inner_finally); + if (!compiler_push_fblock(c, FINALLY_END, inner_finally)) + return 0; /* Finally block starts; context.__exit__ is on the stack under the exception or return information. Just issue our magic @@ -2922,7 +3007,10 @@ /* Finally block ends. */ ADDOP(c, END_FINALLY); - compiler_pop_fblock(c, FINALLY_END, finally); + compiler_pop_fblock(c, FINALLY_END, inner_finally); + + /* Jump target for the end of the try/except statement */ + compiler_use_next_block(c, outer_end); return 1; } Index: Python/bltinmodule.c =================================================================== --- Python/bltinmodule.c (revision 70362) +++ Python/bltinmodule.c (working copy) @@ -2569,6 +2569,7 @@ SETBUILTIN("None", Py_None); SETBUILTIN("Ellipsis", Py_Ellipsis); SETBUILTIN("NotImplemented", Py_NotImplemented); + SETBUILTIN("StatementSkipped", Py_StatementSkipped); SETBUILTIN("False", Py_False); SETBUILTIN("True", Py_True); SETBUILTIN("basestring", &PyBaseString_Type); Index: Include/object.h =================================================================== --- Include/object.h (revision 70362) +++ Include/object.h (working copy) @@ -826,6 +826,13 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) +/* +Py_StatementSkipped is a singleton used to signal that the body of +a with statement was skipped. +*/ +PyAPI_DATA(PyObject) _Py_StatementSkippedStruct; /* Don't use this directly */ +#define Py_StatementSkipped (&_Py_StatementSkippedStruct) + /* Rich comparison opcodes */ #define Py_LT 0 #define Py_LE 1 Index: Include/pyerrors.h =================================================================== --- Include/pyerrors.h (revision 70362) +++ Include/pyerrors.h (working copy) @@ -119,6 +119,7 @@ PyAPI_DATA(PyObject *) PyExc_Exception; PyAPI_DATA(PyObject *) PyExc_StopIteration; PyAPI_DATA(PyObject *) PyExc_GeneratorExit; +PyAPI_DATA(PyObject *) PyExc_SkipStatement; PyAPI_DATA(PyObject *) PyExc_StandardError; PyAPI_DATA(PyObject *) PyExc_ArithmeticError; PyAPI_DATA(PyObject *) PyExc_LookupError; Index: Objects/object.c =================================================================== --- Objects/object.c (revision 70362) +++ Objects/object.c (working copy) @@ -2023,6 +2023,37 @@ 1, &PyNotImplemented_Type }; +/* StatementSkipped is an object that can be used to signal that the + body of a with statement was skipped */ + +static PyObject * +StatementSkipped_repr(PyObject *op) +{ + return PyString_FromString("StatementSkipped"); +} + +static PyTypeObject PyStatementSkipped_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "StatementSkippedType", + 0, + 0, + none_dealloc, /*tp_dealloc*/ /*never called*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + StatementSkipped_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ +}; + +PyObject _Py_StatementSkippedStruct = { + _PyObject_EXTRA_INIT + 1, &PyStatementSkipped_Type +}; + void _Py_ReadyTypes(void) { @@ -2049,6 +2080,9 @@ if (PyType_Ready(&PyNotImplemented_Type) < 0) Py_FatalError("Can't initialize type(NotImplemented)"); + + if (PyType_Ready(&PyStatementSkipped_Type) < 0) + Py_FatalError("Can't initialize type(StatementSkipped)"); } Index: Objects/exceptions.c =================================================================== --- Objects/exceptions.c (revision 70362) +++ Objects/exceptions.c (working copy) @@ -473,6 +473,14 @@ /* + * SkipStatement extends BaseException + */ +SimpleExtendsException(PyExc_BaseException, SkipStatement, + "Signal that a with statement body should not be " + "executed."); + + +/* * SystemExit extends BaseException */ @@ -1984,6 +1992,7 @@ PRE_INIT(TypeError) PRE_INIT(StopIteration) PRE_INIT(GeneratorExit) + PRE_INIT(SkipStatement) PRE_INIT(SystemExit) PRE_INIT(KeyboardInterrupt) PRE_INIT(ImportError) @@ -2052,6 +2061,7 @@ POST_INIT(TypeError) POST_INIT(StopIteration) POST_INIT(GeneratorExit) + POST_INIT(SkipStatement) POST_INIT(SystemExit) POST_INIT(KeyboardInterrupt) POST_INIT(ImportError) Index: Lib/contextlib.py =================================================================== --- Lib/contextlib.py (revision 70362) +++ Lib/contextlib.py (working copy) @@ -15,7 +15,11 @@ try: return self.gen.next() except StopIteration: - raise RuntimeError("generator didn't yield") + # The generator never yielded, so tell + # the interpreter to skip over the body + # of the with statement + print "Skipping statement body" + raise SkipStatement def __exit__(self, type, value, traceback): if type is None: Index: Lib/test/test_with.py =================================================================== --- Lib/test/test_with.py (revision 70362) +++ Lib/test/test_with.py (working copy) @@ -238,7 +238,6 @@ def testInlineGeneratorBoundSyntax(self): with mock_contextmanager_generator() as foo: self.assertInWithGeneratorInvariants(foo) - # FIXME: In the future, we'll try to keep the bound names from leaking self.assertAfterWithGeneratorInvariantsNoError(foo) def testInlineGeneratorBoundToExistingVariable(self): @@ -591,6 +590,49 @@ self.fail("Didn't raise RuntimeError") + def testWithSkipStatement(self): + # PEP 377: Allow __enter__() to skip the + # with statement body, and bind StatementSkipped + # to any context variables + class skip(): + def __enter__(self): + raise SkipStatement + def __exit__(self, *args): + pass + counter = 0 + with skip(): + counter += 10 + with skip() as v1: + counter += 10 + self.assertEqual(v1, StatementSkipped) + with skip() as (v2,): + counter += 10 + self.assertEqual(v2, StatementSkipped) + with skip() as (v3, v4, v5): + counter += 10 + self.assertEqual(v3, StatementSkipped) + self.assertEqual(v4, StatementSkipped) + self.assertEqual(v5, StatementSkipped) + seq = [None, None, None] + with skip() as seq[0]: + counter += 10 + self.assertEqual(seq[0], StatementSkipped) + with skip() as (seq[1], seq[2]): + counter += 10 + self.assertEqual(seq[1], StatementSkipped) + self.assertEqual(seq[2], StatementSkipped) + class Storage: pass + store = Storage() + with skip() as store.a: + counter += 10 + self.assertEqual(store.a, StatementSkipped) + with skip() as (store.b, store.c): + counter += 10 + self.assertEqual(store.b, StatementSkipped) + self.assertEqual(store.c, StatementSkipped) + self.assertEqual(counter, 0) + + class AssignmentTargetTestCase(unittest.TestCase): def testSingleComplexTarget(self): Index: Lib/test/test_contextlib.py =================================================================== --- Lib/test/test_contextlib.py (revision 70362) +++ Lib/test/test_contextlib.py (working copy) @@ -100,6 +100,42 @@ self.assertEqual(baz.foo, 'bar') self.assertEqual(baz.__doc__, "Whee!") + def test_contextmanager_skip_statement(self): + # PEP 377 - skip statement body if generator + # doesn't yield + @contextmanager + def no_yield(): + x = 0 + if x: + yield + x = "skipped" + with no_yield(): + x = "didn't skip" + self.assertEqual(x, "skipped") + @contextmanager + def a(): + try: + yield + except: + # Swallow the exception + pass + @contextmanager + def b(): + 1/0 + yield + @contextmanager + def ab(): + with a(): + with b(): + yield + try: + x = "skipped" + with ab(): + x = "didn't skip" + self.assertEqual(x, "skipped") + except ZeroDivisionError: + self.fail("Didn't swallow ZeroDivisionError") + class NestedTestCase(unittest.TestCase): # XXX This needs more work @@ -222,6 +258,30 @@ return 10 self.assertEqual(foo(), 1) + def test_nested_skip_statement(self): + # PEP 377 - skip statement body if inner + # context manager raises an exception that + # is swallowed by an outer context manager + @contextmanager + def a(): + try: + yield + except: + # Swallow the exception + pass + @contextmanager + def b(): + 1/0 + yield + try: + x = "skipped" + with nested(a(), b()): + x = "didn't skip" + self.assertEqual(x, "skipped") + except ZeroDivisionError: + self.fail("Didn't swallow ZeroDivisionError") + + class ClosingTestCase(unittest.TestCase): # XXX This needs more work