diff -r 128a3f03ddeb Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Tue Sep 29 12:32:13 2015 +0200 +++ b/Lib/test/test_codecs.py Tue Sep 29 13:27:15 2015 +0200 @@ -361,6 +361,12 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual("[\uDC80]".encode(self.encoding, "replace"), "[?]".encode(self.encoding)) + # sequential surrogate characters + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"), + "[]".encode(self.encoding)) + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"), + "[??]".encode(self.encoding)) + bom = "".encode(self.encoding) for before, after in [("\U00010fff", "A"), ("[", "]"), ("A", "\U00010fff")]: @@ -753,6 +759,7 @@ class UTF8Test(ReadTest, unittest.TestCa encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" ill_formed_sequence_replace = "\ufffd" * 3 + BOM = b'' def test_partial(self): self.check_partial( @@ -785,23 +792,32 @@ class UTF8Test(ReadTest, unittest.TestCa super().test_lone_surrogates() # not sure if this is making sense for # UTF-16 and UTF-32 - self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"), - b'[\x80]') + self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"), + self.BOM + b'[\x80]') + + with self.assertRaises(UnicodeEncodeError) as cm: + "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape") + exc = cm.exception + self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') def test_surrogatepass_handler(self): - self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"), - b"abc\xed\xa0\x80def") - self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), + self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), + self.BOM + b"abc\xed\xa0\x80def") + self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"), + self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"), + self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]') + + self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"), "abc\ud800def") - self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), - b"\xf0\x90\xbf\xbf\xed\xa0\x80") - self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"), "\U00010fff\uD800") + self.assertTrue(codecs.lookup_error("surrogatepass")) with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0".decode("utf-8", "surrogatepass") + b"abc\xed\xa0".decode(self.encoding, "surrogatepass") with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + b"abc\xed\xa0z".decode(self.encoding, "surrogatepass") @unittest.skipUnless(sys.platform == 'win32', @@ -1008,6 +1024,7 @@ class ReadBufferTest(unittest.TestCase): class UTF8SigTest(UTF8Test, unittest.TestCase): encoding = "utf-8-sig" + BOM = codecs.BOM_UTF8 def test_partial(self): self.check_partial( diff -r 128a3f03ddeb Objects/stringlib/codecs.h --- a/Objects/stringlib/codecs.h Tue Sep 29 12:32:13 2015 +0200 +++ b/Objects/stringlib/codecs.h Tue Sep 29 13:27:15 2015 +0200 @@ -268,9 +268,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicod Py_ssize_t nallocated; /* number of result bytes allocated */ Py_ssize_t nneeded; /* number of result bytes needed */ #if STRINGLIB_SIZEOF_CHAR > 1 - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; PyObject *rep = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; #endif #if STRINGLIB_SIZEOF_CHAR == 1 const Py_ssize_t max_char_size = 2; @@ -326,72 +327,118 @@ STRINGLIB(utf8_encoder)(PyObject *unicod } #if STRINGLIB_SIZEOF_CHAR > 1 else if (Py_UNICODE_IS_SURROGATE(ch)) { - Py_ssize_t newpos; - Py_ssize_t repsize, k, startpos; + Py_ssize_t startpos, endpos, newpos; + Py_ssize_t repsize, k; + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + startpos = i-1; - rep = unicode_encode_call_errorhandler( - errors, &errorHandler, "utf-8", "surrogates not allowed", - unicode, &exc, startpos, startpos+1, &newpos); - if (!rep) - goto error; + endpos = startpos+1; - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_LENGTH(rep); + while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos])) + endpos++; - if (repsize > max_char_size) { - Py_ssize_t offset; + switch (error_handler) + { + case _Py_ERROR_REPLACE: + memset(p, '?', endpos - startpos); + p += (endpos - startpos); + /* fall through the ignore handler */ + case _Py_ERROR_IGNORE: + i += (endpos - startpos - 1); + break; - if (result == NULL) - offset = p - stackbuf; + + case _Py_ERROR_SURROGATEPASS: + for (k=startpos; k> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + i += (endpos - startpos - 1); + break; + + case _Py_ERROR_SURROGATEESCAPE: + for (k=startpos; k= endpos) { + i += (endpos - startpos - 1); + break; + } + startpos = k; + assert(startpos < endpos); + /* fall through the default handler */ + + default: + rep = unicode_encode_call_errorhandler( + errors, &error_handler_obj, "utf-8", "surrogates not allowed", + unicode, &exc, startpos, endpos, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); else - offset = p - PyBytes_AS_STRING(result); + repsize = PyUnicode_GET_LENGTH(rep); - if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { - /* integer overflow */ - PyErr_NoMemory(); - goto error; - } - nallocated += repsize - max_char_size; - if (result != NULL) { - if (_PyBytes_Resize(&result, nallocated) < 0) - goto error; - } else { - result = PyBytes_FromStringAndSize(NULL, nallocated); + if (repsize > max_char_size) { + Py_ssize_t offset; + if (result == NULL) - goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); - } - p = PyBytes_AS_STRING(result) + offset; - } + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); - if (PyBytes_Check(rep)) { - char *prep = PyBytes_AS_STRING(rep); - for(k = repsize; k > 0; k--) - *p++ = *prep++; - } else /* rep is unicode */ { - enum PyUnicode_Kind repkind; - void *repdata; - - if (PyUnicode_READY(rep) < 0) - goto error; - repkind = PyUnicode_KIND(rep); - repdata = PyUnicode_DATA(rep); - - for(k=0; k PY_SSIZE_T_MAX - repsize + max_char_size) { + /* integer overflow */ + PyErr_NoMemory(); goto error; } - *p++ = (char)c; + nallocated += repsize - max_char_size; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + p = PyBytes_AS_STRING(result) + offset; } + + if (PyBytes_Check(rep)) { + char *prep = PyBytes_AS_STRING(rep); + for(k = repsize; k > 0; k--) + *p++ = *prep++; + } else /* rep is unicode */ { + enum PyUnicode_Kind repkind; + void *repdata; + + if (PyUnicode_READY(rep) < 0) + goto error; + repkind = PyUnicode_KIND(rep); + repdata = PyUnicode_DATA(rep); + + for(k=0; k 2 @@ -430,7 +477,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicod } #if STRINGLIB_SIZEOF_CHAR > 1 - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif return result; @@ -438,7 +485,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicod #if STRINGLIB_SIZEOF_CHAR > 1 error: Py_XDECREF(rep); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); Py_XDECREF(result); return NULL; diff -r 128a3f03ddeb Objects/unicodeobject.c --- a/Objects/unicodeobject.c Tue Sep 29 12:32:13 2015 +0200 +++ b/Objects/unicodeobject.c Tue Sep 29 13:27:15 2015 +0200 @@ -297,6 +297,7 @@ typedef enum { _Py_ERROR_UNKNOWN=0, _Py_ERROR_STRICT, _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_SURROGATEPASS, _Py_ERROR_REPLACE, _Py_ERROR_IGNORE, _Py_ERROR_XMLCHARREFREPLACE, @@ -312,6 +313,8 @@ get_error_handler(const char *errors) return _Py_ERROR_STRICT; if (strcmp(errors, "surrogateescape") == 0) return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "surrogatepass") == 0) + return _Py_ERROR_SURROGATEPASS; if (strcmp(errors, "ignore") == 0) return _Py_ERROR_IGNORE; if (strcmp(errors, "replace") == 0) @@ -6479,8 +6482,8 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; case _Py_ERROR_REPLACE: - while (collstart++ < collend) - *str++ = '?'; + memset(str, '?', collend - collstart); + str += (collend - collstart); /* fall through ignore error handler */ case _Py_ERROR_IGNORE: pos = collend;