diff -r b0087e17cd5e Lib/test/test_unicode.py --- a/Lib/test/test_unicode.py Fri Jul 01 17:57:30 2016 +0300 +++ b/Lib/test/test_unicode.py Tue Jul 05 23:16:54 2016 -0400 @@ -2578,10 +2578,70 @@ class S(str): def __iadd__(self, o): return "3" + def __radd__(self, o): + return "4" s = S("1") s += "4" self.assertEqual(s, "3") + # __radd__ + s = "" + S("1") + self.assertEqual(s, "4") + + def test_subclass_add_return_type(self): + class S(str): + pass + + s = S("1") + S("2") + self.assertEqual(s, "12") + self.assertIs(type(s), str) + + s = "" + S("1") + self.assertEqual(s, "1") + self.assertIs(type(s), str) + + s = S("1") + "" + self.assertEqual(s, "1") + self.assertIs(type(s), str) + + s = S("1") + s += "2" + self.assertEqual(s, "12") + self.assertIs(type(s), str) + + s = "1" + s += S("2") + self.assertEqual(s, "12") + self.assertIs(type(s), str) + + @support.cpython_only + def test_subclass_add_no_interning(self): + import _testcapi + class S(str): + pass + + # Longer strings to prevent interning + base_left = "# ! ~" * 50 + base_right = "! @ ^" * 50 + + self.assertFalse(_testcapi.unicode_is_interned(base_left)) + self.assertFalse(_testcapi.unicode_is_interned(base_right)) + + s = S(base_left) + s += S(base_right) + self.assertEqual(s, base_left + base_right) + self.assertIs(type(s), str) + + s = S(base_left) + s += base_right + self.assertEqual(s, base_left + base_right) + self.assertIs(type(s), str) + + s = base_left + s += S(base_right) + self.assertEqual(s, base_left + base_right) + self.assertIs(type(s), str) + @support.cpython_only def test_encode_decimal(self): from _testcapi import unicode_encodedecimal diff -r b0087e17cd5e Modules/_testcapimodule.c --- a/Modules/_testcapimodule.c Fri Jul 01 17:57:30 2016 +0300 +++ b/Modules/_testcapimodule.c Tue Jul 05 23:16:54 2016 -0400 @@ -1793,6 +1793,21 @@ } static PyObject * +unicode_is_interned(PyObject *self, PyObject *arg) +{ + if (!PyUnicode_Check(arg)) { + PyErr_BadArgument(); + return NULL; + } + + if (PyUnicode_CHECK_INTERNED(arg)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +static PyObject * unicode_aswidechar(PyObject *self, PyObject *args) { PyObject *unicode, *result; @@ -4033,6 +4048,7 @@ {"test_u_code", (PyCFunction)test_u_code, METH_NOARGS}, {"test_Z_code", (PyCFunction)test_Z_code, METH_NOARGS}, {"test_widechar", (PyCFunction)test_widechar, METH_NOARGS}, + {"unicode_is_interned", unicode_is_interned, METH_O}, {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, {"unicode_aswidecharstring",unicode_aswidecharstring, METH_VARARGS}, {"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS}, diff -r b0087e17cd5e Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Jul 01 17:57:30 2016 +0300 +++ b/Objects/unicodeobject.c Tue Jul 05 23:16:54 2016 -0400 @@ -1809,8 +1809,6 @@ return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; - if (!PyUnicode_CheckExact(unicode)) - return 0; #ifdef Py_DEBUG /* singleton refcount is greater than 1 */ assert(!unicode_is_singleton(unicode)); @@ -11162,6 +11160,51 @@ return result; } +PyObject* +cast_unicode_subtype_to_base(PyObject *old) { + /* Based on code from _PyUnicode_New(Py_ssize_t). + Should be kept in sync with unicode_dealloc. + + Converts a unicode-subtype object to the base type, + without incurring a memory copy. Caller should ensure + that object is safe to modify via unicode_modifiable */ + PyUnicodeObject *new; + + new = PyObject_New(PyUnicodeObject, &PyUnicode_Type); + if (!new) { + return NULL; + } + + /* Grab all the data from the old object */ + _PyUnicode_WSTR_LENGTH(new) = _PyUnicode_WSTR_LENGTH(old); + _PyUnicode_HASH(new) = _PyUnicode_HASH(old); + + _PyUnicode_STATE(new).interned = _PyUnicode_STATE(old).interned; + _PyUnicode_STATE(new).kind = _PyUnicode_STATE(old).kind; + _PyUnicode_STATE(new).compact = _PyUnicode_STATE(old).compact; + _PyUnicode_STATE(new).ready = _PyUnicode_STATE(old).ready; + _PyUnicode_STATE(new).ascii = _PyUnicode_STATE(old).ascii; + + _PyUnicode_DATA_ANY(new) = _PyUnicode_DATA_ANY(old); + _PyUnicode_LENGTH(new) = _PyUnicode_LENGTH(old); + + _PyUnicode_UTF8(new) = _PyUnicode_UTF8(old); + _PyUnicode_UTF8_LENGTH(new) = _PyUnicode_UTF8_LENGTH(old); + + _PyUnicode_WSTR(new) = _PyUnicode_WSTR(old); + _PyUnicode_WSTR_LENGTH(new) = _PyUnicode_WSTR_LENGTH(old); + + /* Before we can deallocate, we need to ensure that unicode_deallocate + won't free the space it allocated for the data. */ + _PyUnicode_WSTR(old) = NULL; + _PyUnicode_UTF8(old) = NULL; + _PyUnicode_DATA_ANY(old) = NULL; + /* Deallocate the old object now that we have stolen all it's data */ + Py_DECREF(old); + + return (PyObject*) new; +} + void PyUnicode_Append(PyObject **p_left, PyObject *right) { @@ -11189,13 +11232,50 @@ /* Shortcuts */ if (left == unicode_empty) { + /* This code exists to ensure that + appending a unicode object with + an object that is a subclass of + unicode returns a unicode object */ + if (!PyUnicode_CheckExact(right)) { + if (unicode_modifiable(right)) { + right = cast_unicode_subtype_to_base(right); + if (!right) { + goto error; + } + } else { + Py_DECREF(left); + *p_left = _PyUnicode_Copy(right); + if (!(*p_left)) { + goto error; + } + return; + } + } Py_DECREF(left); Py_INCREF(right); *p_left = right; return; } - if (right == unicode_empty) + if (right == unicode_empty) { + /* See comment in the left conditon above + for why this special handling of non + exact types is necessary. */ + if (!PyUnicode_CheckExact(left)) { + if (unicode_modifiable(left)) { + *p_left = cast_unicode_subtype_to_base(left); + if (!(*p_left)) { + goto error; + } + } else { + *p_left = _PyUnicode_Copy(left); + if (!(*p_left)) { + goto error; + } + Py_DECREF(left); + } + } return; + } left_len = PyUnicode_GET_LENGTH(left); right_len = PyUnicode_GET_LENGTH(right); @@ -11206,14 +11286,14 @@ } new_len = left_len + right_len; - if (unicode_modifiable(left) - && PyUnicode_CheckExact(right) - && PyUnicode_KIND(right) <= PyUnicode_KIND(left) + if (unicode_modifiable(left) && + PyUnicode_Check(right) && + PyUnicode_KIND(right) <= PyUnicode_KIND(left) && /* Don't resize for ascii += latin1. Convert ascii to latin1 requires to change the structure size, but characters are stored just after the structure, and so it requires to move all characters which is not so different than duplicating the string. */ - && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) + !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) { /* append inplace */ if (unicode_resize(p_left, new_len) != 0) @@ -11221,6 +11301,15 @@ /* copy 'right' into the newly allocated area of 'left' */ _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); + + /* ensure that the returned object is of type PyUnicode */ + left = *p_left; + if (!PyUnicode_CheckExact(left)) { + *p_left = cast_unicode_subtype_to_base(left); + if (!(*p_left)) { + goto error; + } + } } else { maxchar = PyUnicode_MAX_CHAR_VALUE(left); diff -r b0087e17cd5e Python/ceval.c --- a/Python/ceval.c Fri Jul 01 17:57:30 2016 +0300 +++ b/Python/ceval.c Tue Jul 05 23:16:54 2016 -0400 @@ -1562,8 +1562,17 @@ PyObject *right = POP(); PyObject *left = TOP(); PyObject *sum; - if (PyUnicode_CheckExact(left) && - PyUnicode_CheckExact(right)) { + /* Hit the faster unicode_concatenate method if and only if + all the following conditions are true: + + 1. The left operand is a unicode type + 2. The right operand is a unicode type + 3. The left operand's __add__ method isn't overriden + 4. The right operand's __radd__ method isn't overriden */ + if (PyUnicode_Check(left) && PyUnicode_Check(right) && + Py_TYPE(left)->tp_as_number->nb_add == 0 && + Py_TYPE(left)->tp_as_sequence->sq_concat == PyUnicode_Concat && + Py_TYPE(right)->tp_as_number->nb_add == 0) { sum = unicode_concatenate(left, right, f, next_instr); /* unicode_concatenate consumed the ref to v */ } @@ -1762,7 +1771,21 @@ PyObject *right = POP(); PyObject *left = TOP(); PyObject *sum; - if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { + /* Hit the faster unicode_concatenate method if and only if + all the following conditions are true: + + 1. The left operand is a unicode type + 2. The right operand is a unicode type + 3. The left operand's __iadd__ method isn't overriden + 4. The left operand's __add__ method isn't overriden + 5. The right operand's __radd__ method isn't overriden */ + if (PyUnicode_Check(left) && PyUnicode_Check(right) && + Py_TYPE(left)->tp_as_number->nb_inplace_add == 0 && + Py_TYPE(left)->tp_as_sequence->sq_inplace_concat == 0 && + Py_TYPE(left)->tp_as_number->nb_add == 0 && + Py_TYPE(left)->tp_as_sequence->sq_concat == PyUnicode_Concat && + Py_TYPE(right)->tp_as_number->nb_add == 0) { + sum = unicode_concatenate(left, right, f, next_instr); /* unicode_concatenate consumed the ref to v */ }