diff -r b07488490001 Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Fri Apr 20 14:36:47 2012 +0200 +++ b/Lib/test/test_codecs.py Wed Apr 25 20:08:37 2012 +0300 @@ -540,8 +540,19 @@ ) def test_errors(self): - self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, - b"\xff", "strict", True) + tests = [ + (b'\xff', '\ufffd'), + (b'A\x00Z', 'A\ufffd'), + (b'A\x00B\x00C\x00D\x00Z', 'ABCD\ufffd'), + (b'\x00\xd8', '\ufffd'), + (b'\x00\xd8A', '\ufffd'), + (b'\x00\xd8A\x00', '\ufffdA'), + (b'\x00\xdcA\x00', '\ufffdA'), + ] + for raw, expected in tests: + self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode, + raw, 'strict', True) + self.assertEqual(raw.decode('utf-16le', 'replace'), expected) def test_nonbmp(self): self.assertEqual("\U00010203".encode(self.encoding), @@ -568,8 +579,19 @@ ) def test_errors(self): - self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, - b"\xff", "strict", True) + tests = [ + (b'\xff', '\ufffd'), + (b'\x00A\xff', 'A\ufffd'), + (b'\x00A\x00B\x00C\x00DZ', 'ABCD\ufffd'), + (b'\xd8\x00', '\ufffd'), + (b'\xd8\x00\xdc', '\ufffd'), + (b'\xd8\x00\x00A', '\ufffdA'), + (b'\xdc\x00\x00A', '\ufffdA'), + ] + for raw, expected in tests: + self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, + raw, 'strict', True) + self.assertEqual(raw.decode('utf-16be', 'replace'), expected) def test_nonbmp(self): self.assertEqual("\U00010203".encode(self.encoding), diff -r b07488490001 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Apr 20 14:36:47 2012 +0200 +++ b/Objects/unicodeobject.c Wed Apr 25 20:08:37 2012 +0300 @@ -3425,7 +3425,7 @@ /* Unpack UTF-16 encoded data */ p = unicode->str; q = (unsigned char *)s; - e = q + size - 1; + e = q + size; if (byteorder) bo = *byteorder; @@ -3476,8 +3476,20 @@ #endif aligned_end = (const unsigned char *) ((size_t) e & ~LONG_PTR_MASK); - while (q < e) { + while (1) { Py_UNICODE ch; + if (e - q < 2) { + /* remaining byte at the end? (size should be even) */ + if (q == e || consumed) + break; + errmsg = "truncated data"; + startinpos = ((const char *)q) - starts; + endinpos = ((const char *)e) - starts; + outpos = p - PyUnicode_AS_UNICODE(unicode); + goto utf16Error; + /* The remaining input chars are ignored if the callback + chooses to skip the input */ + } /* First check for possible aligned read of a C 'long'. Unaligned reads are more expensive, better to defer to another iteration. */ if (!((size_t) q & LONG_PTR_MASK)) { @@ -3546,8 +3558,8 @@ } p = _p; q = _q; - if (q >= e) - break; + if (e - q < 2) + continue; } ch = (q[ihi] << 8) | q[ilo]; @@ -3559,10 +3571,10 @@ } /* UTF-16 code pair: */ - if (q > e) { + if (e - q < 2) { errmsg = "unexpected end of data"; startinpos = (((const char *)q) - 2) - starts; - endinpos = ((const char *)e) + 1 - starts; + endinpos = ((const char *)e) - starts; goto utf16Error; } if (0xD800 <= ch && ch <= 0xDBFF) { @@ -3606,31 +3618,9 @@ &outpos, &p)) goto onError; - } - /* remaining byte at the end? (size should be even) */ - if (e == q) { - if (!consumed) { - errmsg = "truncated data"; - startinpos = ((const char *)q) - starts; - endinpos = ((const char *)e) + 1 - starts; - outpos = p - PyUnicode_AS_UNICODE(unicode); - if (unicode_decode_call_errorhandler( - errors, - &errorHandler, - "utf16", errmsg, - &starts, - (const char **)&e, - &startinpos, - &endinpos, - &exc, - (const char **)&q, - &unicode, - &outpos, - &p)) - goto onError; - /* The remaining input chars are ignored if the callback - chooses to skip the input */ - } + /* Update data because unicode_decode_call_errorhandler might have + changed the input object. */ + aligned_end = (const unsigned char *) ((size_t) e & ~LONG_PTR_MASK); } if (byteorder)