comparing with ssh://hg@hg.python.org/cpython searching for changes diff -r a9bb1909cbed Lib/test/test_binop.py --- a/Lib/test/test_binop.py Tue Mar 15 19:21:03 2011 +0200 +++ b/Lib/test/test_binop.py Tue Mar 15 13:48:16 2011 -0400 @@ -1,4 +1,6 @@ -"""Tests for binary operators on subtypes of built-in types.""" +"""Tests for binary operators on subtypes of built-in types and other + details of the binop implementation +""" import unittest from test import support @@ -373,6 +375,105 @@ self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) + def test_issue11477_sequence_concatenation(self): + # Check overloading for A + B and A += B + # when A only implements sq_concat (but not nb_add) + testcase = self + class RHS: + def __init__(self): + self.allow_radd = True + def __iter__(self): + yield "Excellent!" + def __radd__(self, other): + if not self.allow_radd: + testcase.fail("RHS __radd__ called!") + return other + type(other)(self) + lhs = [] + rhs = RHS() + self.assertEqual(lhs.__add__(rhs), NotImplemented) + self.assertEqual(lhs + rhs, ["Excellent!"]) + with self.assertRaises(TypeError): + lhs + 1 + rhs.allow_radd = False + orig_lhs = lhs + lhs += rhs + self.assertIs(lhs, orig_lhs) + self.assertEqual(lhs, ["Excellent!"]) + with self.assertRaises(TypeError): + lhs += 1 + + def test_issue11477_sequence_concatenation_subclass(self): + # Check overloading for A + B and A += B + # when A only implements sq_concat (but not nb_add) + # and B is a subclass of A + testcase = self + # NOTE: We *cannot* create another subclass of list for the LHS + # here, since doing so moves the __add__ implementation from + # sq_concat to nb_add, bypassing the underlying behaviour + # we're trying to exercise. So we craft a test that can work + # with the standard behaviour of list.__add__ + class RHS(list): + def __init__(self): + list.__init__(self, ["Excellent!"]) + def __radd__(self, other): + return [] + lhs = [] + rhs = RHS() + self.assertEqual(lhs.__add__(rhs), ["Excellent!"]) + self.assertEqual(lhs + rhs, []) + orig_lhs = lhs + lhs += rhs + self.assertIs(lhs, orig_lhs) + self.assertEqual(lhs, ["Excellent!"]) + + def test_issue11477_sequence_repetition(self): + # Check overloading for A * B and A *= B + # when A only implements sq_repeat (but not nb_mul) + testcase = self + class Scalar: + def __index__(self): + return 2 + def __mul__(self, other): + return NotImplemented + def __rmul__(self, other): + testcase.fail("RHS __rmul__ called!") + seq = [None] + scalar = Scalar() + self.assertEqual(seq * scalar, [None, None]) + self.assertEqual(scalar * seq, [None, None]) + with self.assertRaises(TypeError): + seq * object() + orig_seq = seq + seq *= scalar + self.assertIs(seq, orig_seq) + self.assertEqual(seq, [None, None]) + with self.assertRaises(TypeError): + seq *= object() + + def test_issue11477_sequence_repetition_subclass(self): + # Check overloading for A * B and A *= B + # when A only implements sq_repeat (but not nb_mul) + # and B is a subclass of A + # This case may seem unlikely, but may arise for something + # like a 0-d matrix/scalar + testcase = self + class Scalar(list): + def __index__(self): + return 2 + def __mul__(self, other): + return NotImplemented + def __rmul__(self, other): + return 3 * other + seq = [None] + scalar = Scalar() + self.assertEqual(seq * scalar, [None, None, None]) + self.assertEqual(scalar * seq, [None, None]) + orig_seq = seq + seq *= scalar + self.assertIs(seq, orig_seq) + self.assertEqual(seq, [None, None]) + + def test_main(): support.run_unittest(RatTestCase, OperationOrderTests) diff -r a9bb1909cbed Objects/abstract.c --- a/Objects/abstract.c Tue Mar 15 19:21:03 2011 +0200 +++ b/Objects/abstract.c Tue Mar 15 13:48:16 2011 -0400 @@ -917,14 +917,36 @@ PyObject * PyNumber_Add(PyObject *v, PyObject *w) { - PyObject *result = binary_op1(v, w, NB_SLOT(nb_add)); - if (result == Py_NotImplemented) { - PySequenceMethods *m = v->ob_type->tp_as_sequence; - Py_DECREF(result); - if (m && m->sq_concat) { - return (*m->sq_concat)(v, w); + PyNumberMethods *m_num = v->ob_type->tp_as_number; + PySequenceMethods *m_seq = v->ob_type->tp_as_sequence; + PyObject *result = NULL; + /* We do a tapdance here so sq_concat is tried on the LH + * operand before __radd__ is tried on the RH operand. + * If the LH implements both nb_add and sq_concat,only nb_add + * is tried. + */ + int try_binop = 1; + int have_add = m_num && m_num->nb_add; + int have_concat = m_seq && m_seq->sq_concat; + if (have_add || !have_concat || + PyType_IsSubtype(w->ob_type, v->ob_type)) { + try_binop = 0; + result = binary_op1(v, w, NB_SLOT(nb_add)); + } + if (try_binop || result == Py_NotImplemented) { + if (!have_add && have_concat) { + Py_XDECREF(result); + result = (*m_seq->sq_concat)(v, w); + try_binop = result == Py_NotImplemented; } - result = binop_type_error(v, w, "+"); + if (try_binop) { + Py_XDECREF(result); + result = binary_op1(v, w, NB_SLOT(nb_add)); + } + if (result == Py_NotImplemented) { + Py_DECREF(result); + result = binop_type_error(v, w, "+"); + } } return result; } @@ -948,20 +970,46 @@ PyObject * PyNumber_Multiply(PyObject *v, PyObject *w) { - PyObject *result = binary_op1(v, w, NB_SLOT(nb_multiply)); - if (result == Py_NotImplemented) { - PySequenceMethods *mv = v->ob_type->tp_as_sequence; - PySequenceMethods *mw = w->ob_type->tp_as_sequence; - Py_DECREF(result); - if (mv && mv->sq_repeat) { - return sequence_repeat(mv->sq_repeat, v, w); + PyNumberMethods *m_num = v->ob_type->tp_as_number; + PySequenceMethods *m_seq = v->ob_type->tp_as_sequence; + PyObject *result = NULL; + /* We do a tapdance here so sq_repeat is tried on the LH + * operand before __rmul__ is tried on the RH operand. + * If the LH implements both nb_multiply and sq_repeat,only + * nb_multiply is tried. + */ + int try_binop = 1; + int have_mul = m_num && m_num->nb_multiply; + int have_repeat = m_seq && m_seq->sq_repeat; + if (have_mul || !have_repeat || + PyType_IsSubtype(w->ob_type, v->ob_type)) { + try_binop = 0; + result = binary_op1(v, w, NB_SLOT(nb_multiply)); + } + if (try_binop || result == Py_NotImplemented) { + if (!have_mul && have_repeat) { + Py_XDECREF(result); + result = sequence_repeat(m_seq->sq_repeat, v, w); + try_binop = result == Py_NotImplemented; } - else if (mw && mw->sq_repeat) { - return sequence_repeat(mw->sq_repeat, w, v); + if (try_binop) { + Py_XDECREF(result); + result = binary_op1(v, w, NB_SLOT(nb_multiply)); } - result = binop_type_error(v, w, "*"); + if (result == Py_NotImplemented) { + PySequenceMethods *m_seq_rhs = w->ob_type->tp_as_sequence; + if (m_seq_rhs && m_seq_rhs->sq_repeat) { + Py_DECREF(result); + result = sequence_repeat(m_seq_rhs->sq_repeat, w, v); + } + } + if (result == Py_NotImplemented) { + Py_DECREF(result); + result = binop_type_error(v, w, "*"); + } } return result; + } PyObject * @@ -1063,20 +1111,41 @@ PyObject * PyNumber_InPlaceAdd(PyObject *v, PyObject *w) { - PyObject *result = binary_iop1(v, w, NB_SLOT(nb_inplace_add), - NB_SLOT(nb_add)); - if (result == Py_NotImplemented) { - PySequenceMethods *m = v->ob_type->tp_as_sequence; - Py_DECREF(result); - if (m != NULL) { - binaryfunc f = NULL; - f = m->sq_inplace_concat; - if (f == NULL) - f = m->sq_concat; - if (f != NULL) - return (*f)(v, w); + PyNumberMethods *m_num = v->ob_type->tp_as_number; + PySequenceMethods *m_seq = v->ob_type->tp_as_sequence; + PyObject *result = NULL; + /* Similar tapdance to PyNumber_Add, but with extra checks to + * handle the in-place versions of the nb_* and sq_* methods + */ + int try_binop = 1; + int have_add = m_num && m_num->nb_add; + int have_inplace_add = m_num && m_num->nb_inplace_add; + int have_concat = m_seq && m_seq->sq_concat; + int have_inplace_concat = m_seq && m_seq->sq_inplace_concat; + if (have_inplace_add || (!have_inplace_concat && !have_concat)) { + try_binop = 0; + result = binary_iop1(v, w, NB_SLOT(nb_inplace_add), + NB_SLOT(nb_add)); + } + if (try_binop || result == Py_NotImplemented) { + if (!have_inplace_add && have_inplace_concat) { + Py_XDECREF(result); + result = (*m_seq->sq_inplace_concat)(v, w); + try_binop = result == Py_NotImplemented; } - result = binop_type_error(v, w, "+="); + if (try_binop) { + Py_XDECREF(result); + result = binary_iop1(v, w, NB_SLOT(nb_inplace_add), + NB_SLOT(nb_add)); + } + if (!have_add && have_concat && result == Py_NotImplemented) { + Py_DECREF(result); + result = (*m_seq->sq_concat)(v, w); + } + if (result == Py_NotImplemented) { + Py_DECREF(result); + result = binop_type_error(v, w, "+="); + } } return result; } @@ -1084,28 +1153,51 @@ PyObject * PyNumber_InPlaceMultiply(PyObject *v, PyObject *w) { - PyObject *result = binary_iop1(v, w, NB_SLOT(nb_inplace_multiply), - NB_SLOT(nb_multiply)); - if (result == Py_NotImplemented) { - ssizeargfunc f = NULL; - PySequenceMethods *mv = v->ob_type->tp_as_sequence; - PySequenceMethods *mw = w->ob_type->tp_as_sequence; - Py_DECREF(result); - if (mv != NULL) { - f = mv->sq_inplace_repeat; - if (f == NULL) - f = mv->sq_repeat; - if (f != NULL) - return sequence_repeat(f, v, w); + PyNumberMethods *m_num = v->ob_type->tp_as_number; + PySequenceMethods *m_seq = v->ob_type->tp_as_sequence; + PyObject *result = NULL; + /* Similar tapdance to PyNumber_Multiply, but with extra checks to + * handle the in-place versions of the nb_* and sq_* methods + */ + int try_binop = 1; + int have_mul = m_num && m_num->nb_multiply; + int have_inplace_mul = m_num && m_num->nb_inplace_multiply; + int have_repeat = m_seq && m_seq->sq_repeat; + int have_inplace_repeat = m_seq && m_seq->sq_inplace_repeat; + if (have_inplace_mul || (!have_inplace_repeat && !have_repeat)) { + try_binop = 0; + result = binary_iop1(v, w, NB_SLOT(nb_inplace_multiply), + NB_SLOT(nb_multiply)); + } + if (try_binop || result == Py_NotImplemented) { + if (!have_inplace_mul && have_inplace_repeat) { + Py_XDECREF(result); + result = sequence_repeat(m_seq->sq_inplace_repeat, v, w); + try_binop = result == Py_NotImplemented; } - else if (mw != NULL) { + if (try_binop) { + Py_XDECREF(result); + result = binary_iop1(v, w, NB_SLOT(nb_inplace_multiply), + NB_SLOT(nb_multiply)); + } + if (!have_mul && have_repeat && result == Py_NotImplemented) { + Py_DECREF(result); + result = sequence_repeat(m_seq->sq_repeat, v, w); + } + if (result == Py_NotImplemented) { /* Note that the right hand operand should not be * mutated in this case so sq_inplace_repeat is not - * used. */ - if (mw->sq_repeat) - return sequence_repeat(mw->sq_repeat, w, v); + * tried. */ + PySequenceMethods *m_seq_rhs = w->ob_type->tp_as_sequence; + if (m_seq_rhs && m_seq_rhs->sq_repeat) { + Py_DECREF(result); + result = sequence_repeat(m_seq_rhs->sq_repeat, w, v); + } } - result = binop_type_error(v, w, "*="); + if (result == Py_NotImplemented) { + Py_DECREF(result); + result = binop_type_error(v, w, "*="); + } } return result; } diff -r a9bb1909cbed Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c Tue Mar 15 19:21:03 2011 +0200 +++ b/Objects/bytearrayobject.c Tue Mar 15 13:48:16 2011 -0400 @@ -228,14 +228,14 @@ { Py_ssize_t size; Py_buffer va, vb; - PyByteArrayObject *result = NULL; + PyObject *result = NULL; va.len = -1; vb.len = -1; if (_getbuffer(a, &va) < 0 || _getbuffer(b, &vb) < 0) { - PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + result = Py_NotImplemented; + Py_INCREF(result); goto done; } @@ -245,10 +245,11 @@ goto done; } - result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, size); + result = PyByteArray_FromStringAndSize(NULL, size); if (result != NULL) { - memcpy(result->ob_bytes, va.buf, va.len); - memcpy(result->ob_bytes + va.len, vb.buf, vb.len); + void * data = ((PyByteArrayObject *) result)->ob_bytes; + memcpy(data, va.buf, va.len); + memcpy(data + va.len, vb.buf, vb.len); } done: diff -r a9bb1909cbed Objects/bytesobject.c --- a/Objects/bytesobject.c Tue Mar 15 19:21:03 2011 +0200 +++ b/Objects/bytesobject.c Tue Mar 15 13:48:16 2011 -0400 @@ -675,8 +675,8 @@ vb.len = -1; if (_getbuffer(a, &va) < 0 || _getbuffer(b, &vb) < 0) { - PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + result = Py_NotImplemented; + Py_INCREF(result); goto done; } diff -r a9bb1909cbed Objects/listobject.c --- a/Objects/listobject.c Tue Mar 15 19:21:03 2011 +0200 +++ b/Objects/listobject.c Tue Mar 15 13:48:16 2011 -0400 @@ -469,10 +469,8 @@ PyObject **src, **dest; PyListObject *np; if (!PyList_Check(bb)) { - PyErr_Format(PyExc_TypeError, - "can only concatenate list (not \"%.200s\") to list", - bb->ob_type->tp_name); - return NULL; + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; } #define b ((PyListObject *)bb) size = Py_SIZE(a) + Py_SIZE(b); diff -r a9bb1909cbed Objects/typeobject.c --- a/Objects/typeobject.c Tue Mar 15 19:21:03 2011 +0200 +++ b/Objects/typeobject.c Tue Mar 15 13:48:16 2011 -0400 @@ -5466,10 +5466,8 @@ SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, "x.__len__() <==> len(x)"), /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. - The logic in abstract.c always falls back to nb_add/nb_multiply in - this case. Defining both the nb_* and the sq_* slots to call the - user-defined methods has unexpected side-effects, as shown by - test_descr.notimplemented() */ + The logic in abstract.c ignores them if nb_add/nb_multiply are + defined anyway. */ SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, "x.__add__(y) <==> x+y"), SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, diff -r a9bb1909cbed Objects/unicodeobject.c --- a/Objects/unicodeobject.c Tue Mar 15 19:21:03 2011 +0200 +++ b/Objects/unicodeobject.c Tue Mar 15 13:48:16 2011 -0400 @@ -7444,11 +7444,22 @@ /* Coerce the two arguments */ u = (PyUnicodeObject *)PyUnicode_FromObject(left); - if (u == NULL) - goto onError; + if (u == NULL) { + PyObject *ni = Py_NotImplemented; + /* XXX: PyErr_Matches check??? */ + PyErr_Clear(); + Py_INCREF(ni); + return ni; + } v = (PyUnicodeObject *)PyUnicode_FromObject(right); - if (v == NULL) - goto onError; + if (v == NULL) { + PyObject *ni = Py_NotImplemented; + /* XXX: PyErr_Matches check??? */ + PyErr_Clear(); + Py_DECREF(u); + Py_INCREF(ni); + return ni; + } /* Shortcuts */ if (v == unicode_empty) {