diff -r b4d257c64db7 Lib/test/test_unicode.py --- a/Lib/test/test_unicode.py Thu May 24 21:57:38 2012 +0800 +++ b/Lib/test/test_unicode.py Sat May 26 11:01:07 2012 +0300 @@ -1321,7 +1321,7 @@ # with start byte of a 2-byte sequence (b'\xc2', FFFD), # only the start byte (b'\xc2\xc2', FFFD*2), # 2 start bytes - (b'\xc2\xc2\xc2', FFFD*3), # 2 start bytes + (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes (b'\xc2\x41', FFFD+'A'), # invalid continuation byte # with start byte of a 3-byte sequence (b'\xe1', FFFD), # only the start byte @@ -1391,6 +1391,233 @@ self.assertEqual(seq.decode('utf-8', 'ignore'), res.replace('\uFFFD', '')) + def to_bytestring(self, seq): + return bytes(int(c, 16) for c in seq.split()) + + def assertCorrectUTF8Decoding(self, seq, res, err): + """ + Check that an invalid UTF-8 sequence raises an UnicodeDecodeError when + 'strict' is used, returns res when 'replace' is used, and that doesn't + return anything when 'ignore' is used. + """ + #with self.assertRaises(UnicodeDecodeError) as cm: + #seq.decode('utf-8') + #exc = cm.exception + try: + seq.decode('utf-8') + except UnicodeDecodeError as e: + exc = str(e) + else: + self.fail( + "%r.decode('utf-8') didn't raise UnicodeDecodeError." % seq) + + self.assertIn(err, str(exc)) + self.assertEqual(seq.decode('utf-8', 'replace'), res) + self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'), + 'aaaa' + res + 'bbbb') + res = res.replace('\ufffd', '') + self.assertEqual(seq.decode('utf-8', 'ignore'), res) + self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'), + 'aaaa' + res + 'bbbb') + + def test_invalid_start_byte(self): + """ + Test that an 'invalid start byte' error is raised when the first byte + is not in the ASCII range or is not a valid start byte of a 2-, 3-, or + 4-bytes sequence. The invalid start byte is replaced with a single + U+FFFD when errors='replace'. + E.g. <80> is a continuation byte and can appear only after a start byte. + """ + FFFD = '\ufffd' + for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF': + self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd', + 'invalid start byte') + + def test_unexpected_end_of_data(self): + """ + Test that an 'unexpected end of data' error is raised when the string + ends after a start byte of a 2-, 3-, or 4-bytes sequence without having + enough continuation bytes. The incomplete sequence is replaced with a + single U+FFFD when errors='replace'. + E.g. in the sequence , F3 is the start byte of a 4-bytes + sequence, but it's followed by only 2 valid continuation bytes and the + last continuation bytes is missing. + Note: the continuation bytes must be all valid, if one of them is + invalid another error will be raised. + """ + sequences = [ + 'C2', 'DF', + 'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF', + 'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF', + 'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF', + 'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF', + 'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF', + 'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF' + ] + FFFD = '\ufffd' + for seq in sequences: + self.assertCorrectUTF8Decoding(self.to_bytestring(seq), '\ufffd', + 'unexpected end of data') + + def test_invalid_cb_for_2bytes_seq(self): + """ + Test that an 'invalid continuation byte' error is raised when the + continuation byte of a 2-bytes sequence is invalid. The start byte + is replaced by a single U+FFFD and the second byte is handled + separately when errors='replace'. + E.g. in the sequence , C2 is the start byte of a 2-bytes + sequence, but 41 is not a valid continuation byte because it's the + ASCII letter 'A'. + """ + FFFD = '\ufffd' + FFFDx2 = FFFD * 2 + sequences = [ + ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'), + ('C2 C0', FFFDx2), ('C2 FF', FFFDx2), + ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'), + ('DF C0', FFFDx2), ('DF FF', FFFDx2), + ] + for seq, res in sequences: + self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res, + 'invalid continuation byte') + + def test_invalid_cb_for_3bytes_seq(self): + """ + Test that an 'invalid continuation byte' error is raised when the + continuation byte(s) of a 3-bytes sequence are invalid. When + errors='replace', if the first continuation byte is valid, the first + two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the + third byte is handled separately, otherwise only the start byte is + replaced with a U+FFFD and the other continuation bytes are handled + separately. + E.g. in the sequence , E1 is the start byte of a 3-bytes + sequence, 80 is a valid continuation byte, but 41 is not a valid cb + because it's the ASCII letter 'A'. + Note: when the start byte is E0 or ED, the valid ranges for the first + continuation byte are limited to A0..BF and 80..9F respectively. + However, when the start byte is ED, Python 2 considers all the bytes + in range 80..BF valid. This is fixed in Python 3. + """ + FFFD = '\ufffd' + FFFDx2 = FFFD * 2 + sequences = [ + ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2), + ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2), + ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'), + ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2), + ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'), + ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'), + ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2), + ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'), + ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2), + ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'), + ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'), + ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2), + ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'), + ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2), + ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'), + ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'), + ('ED 7F', FFFD+'\x7f'), + # ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^ + ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'), + ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2), + ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'), + ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2), + ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'), + ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2), + ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'), + ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2), + ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'), + ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'), + ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2), + ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'), + ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2), + ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'), + ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2), + ] + for seq, res in sequences: + self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res, + 'invalid continuation byte') + + def test_invalid_cb_for_4bytes_seq(self): + """ + Test that an 'invalid continuation byte' error is raised when the + continuation byte(s) of a 4-bytes sequence are invalid. When + errors='replace',the start byte and all the following valid + continuation bytes are replaced with a single U+FFFD, and all the bytes + starting from the first invalid continuation bytes (included) are + handled separately. + E.g. in the sequence , E1 is the start byte of a 3-bytes + sequence, 80 is a valid continuation byte, but 41 is not a valid cb + because it's the ASCII letter 'A'. + Note: when the start byte is E0 or ED, the valid ranges for the first + continuation byte are limited to A0..BF and 80..9F respectively. + However, when the start byte is ED, Python 2 considers all the bytes + in range 80..BF valid. This is fixed in Python 3. + """ + FFFD = '\ufffd' + FFFDx2 = FFFD * 2 + sequences = [ + ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2), + ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2), + ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'), + ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2), + ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'), + ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2), + ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'), + ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2), + ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'), + ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2), + ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'), + ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2), + ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'), + ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2), + ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2), + ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'), + ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2), + ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'), + ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2), + ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'), + ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2), + ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'), + ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2), + ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'), + ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2), + ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'), + ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2), + ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'), + ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2), + ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'), + ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2), + ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'), + ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2), + ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'), + ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2), + ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'), + ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2), + ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'), + ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2), + ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'), + ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2), + ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2), + ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2), + ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'), + ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2), + ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'), + ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2), + ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'), + ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2), + ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'), + ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2), + ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'), + ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2), + ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'), + ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2) + ] + for seq, res in sequences: + self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res, + 'invalid continuation byte') + def test_codecs_idna(self): # Test whether trailing dot is preserved self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.") diff -r b4d257c64db7 Objects/stringlib/codecs.h --- a/Objects/stringlib/codecs.h Thu May 24 21:57:38 2012 +0800 +++ b/Objects/stringlib/codecs.h Sat May 26 11:01:07 2012 +0300 @@ -15,6 +15,9 @@ # error C 'long' size should be either 4 or 8! #endif +/* 10xxxxxx */ +#define IS_CONTINUATION_BYTE(ch) ((signed char)(ch) < -0x40) + Py_LOCAL_INLINE(Py_UCS4) STRINGLIB(utf8_decode)(const char **inptr, const char *end, STRINGLIB_CHAR *dest, @@ -91,32 +94,32 @@ } } - if (ch < 0xC2) { - /* invalid sequence - \x80-\xBF -- continuation byte - \xC0-\xC1 -- fake 0000-007F */ - goto InvalidStart; - } - if (ch < 0xE0) { /* \xC2\x80-\xDF\xBF -- 0080-07FF */ Py_UCS4 ch2; + if (ch < 0xC2) { + /* invalid sequence + \x80-\xBF -- continuation byte + \xC0-\xC1 -- fake 0000-007F */ + goto InvalidStart; + } if (end - s < 2) { /* unexpected end of data: the caller will decide whether it's an error or not */ break; } ch2 = (unsigned char)s[1]; - if ((ch2 & 0xC0) != 0x80) + if (!IS_CONTINUATION_BYTE(ch2)) /* invalid continuation byte */ - goto InvalidContinuation; + goto InvalidContinuation1; ch = (ch << 6) + ch2 - ((0xC0 << 6) + 0x80); assert ((ch > 0x007F) && (ch <= 0x07FF)); s += 2; if (STRINGLIB_MAX_CHAR <= 0x007F || (STRINGLIB_MAX_CHAR < 0x07FF && ch > STRINGLIB_MAX_CHAR)) - goto Overflow; + /* Out-of-range */ + goto Return; *p++ = ch; continue; } @@ -127,28 +130,37 @@ if (end - s < 3) { /* unexpected end of data: the caller will decide whether it's an error or not */ + if (end - s < 2) + break; + ch2 = (unsigned char)s[1]; + if (!IS_CONTINUATION_BYTE(ch2) || + (ch2 < 0xA0 ? ch == 0xE0 : ch == 0xED)) + /* for clarification see comments below */ + goto InvalidContinuation1; break; } ch2 = (unsigned char)s[1]; ch3 = (unsigned char)s[2]; - if ((ch2 & 0xC0) != 0x80 || - (ch3 & 0xC0) != 0x80) { + if (!IS_CONTINUATION_BYTE(ch2)) { /* invalid continuation byte */ - goto InvalidContinuation; + goto InvalidContinuation1; } if (ch == 0xE0) { if (ch2 < 0xA0) /* invalid sequence \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */ - goto InvalidContinuation; - } - else if (ch == 0xED && ch2 > 0x9F) { + goto InvalidContinuation1; + } else if (ch == 0xED && ch2 >= 0xA0) { /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF will result in surrogates in range D800-DFFF. Surrogates are not valid UTF-8 so they are rejected. See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ - goto InvalidContinuation; + goto InvalidContinuation1; + } + if (!IS_CONTINUATION_BYTE(ch3)) { + /* invalid continuation byte */ + goto InvalidContinuation2; } ch = (ch << 12) + (ch2 << 6) + ch3 - ((0xE0 << 12) + (0x80 << 6) + 0x80); @@ -156,7 +168,8 @@ s += 3; if (STRINGLIB_MAX_CHAR <= 0x07FF || (STRINGLIB_MAX_CHAR < 0xFFFF && ch > STRINGLIB_MAX_CHAR)) - goto Overflow; + /* Out-of-range */ + goto Return; *p++ = ch; continue; } @@ -167,27 +180,44 @@ if (end - s < 4) { /* unexpected end of data: the caller will decide whether it's an error or not */ + if (end - s < 2) + break; + ch2 = (unsigned char)s[1]; + if (!IS_CONTINUATION_BYTE(ch2) || + (ch2 < 0x90 ? ch == 0xF0 : ch == 0xF4)) + /* for clarification see comments below */ + goto InvalidContinuation1; + if (end - s < 3) + break; + ch3 = (unsigned char)s[2]; + if (!IS_CONTINUATION_BYTE(ch3)) + goto InvalidContinuation2; break; } ch2 = (unsigned char)s[1]; ch3 = (unsigned char)s[2]; ch4 = (unsigned char)s[3]; - if ((ch2 & 0xC0) != 0x80 || - (ch3 & 0xC0) != 0x80 || - (ch4 & 0xC0) != 0x80) { + if (!IS_CONTINUATION_BYTE(ch2)) { /* invalid continuation byte */ - goto InvalidContinuation; + goto InvalidContinuation1; } if (ch == 0xF0) { if (ch2 < 0x90) /* invalid sequence - \xF0\x80\x80\x80-\xF0\x80\xBF\xBF -- fake 0000-FFFF */ - goto InvalidContinuation; - } - else if (ch == 0xF4 && ch2 > 0x8F) { + \xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF */ + goto InvalidContinuation1; + } else if (ch == 0xF4 && ch2 >= 0x90) { /* invalid sequence \xF4\x90\x80\80- -- 110000- overflow */ - goto InvalidContinuation; + goto InvalidContinuation1; + } + if (!IS_CONTINUATION_BYTE(ch3)) { + /* invalid continuation byte */ + goto InvalidContinuation2; + } + if (!IS_CONTINUATION_BYTE(ch4)) { + /* invalid continuation byte */ + goto InvalidContinuation3; } ch = (ch << 18) + (ch2 << 12) + (ch3 << 6) + ch4 - ((0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80); @@ -195,14 +225,14 @@ s += 4; if (STRINGLIB_MAX_CHAR <= 0xFFFF || (STRINGLIB_MAX_CHAR < 0x10FFFF && ch > STRINGLIB_MAX_CHAR)) - goto Overflow; + /* Out-of-range */ + goto Return; *p++ = ch; continue; } goto InvalidStart; } ch = 0; -Overflow: Return: *inptr = s; *outpos = p - dest; @@ -210,9 +240,15 @@ InvalidStart: ch = 1; goto Return; -InvalidContinuation: +InvalidContinuation1: ch = 2; goto Return; +InvalidContinuation2: + ch = 3; + goto Return; +InvalidContinuation3: + ch = 4; + goto Return; } #undef ASCII_CHAR_MASK diff -r b4d257c64db7 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Thu May 24 21:57:38 2012 +0800 +++ b/Objects/unicodeobject.c Sat May 26 11:01:07 2012 +0300 @@ -4769,9 +4769,7 @@ goto End; errmsg = "unexpected end of data"; startinpos = s - starts; - endinpos = startinpos + 1; - while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80) - endinpos++; + endinpos = end - starts; break; case 1: errmsg = "invalid start byte"; @@ -4779,11 +4777,11 @@ endinpos = startinpos + 1; break; case 2: + case 3: + case 4: errmsg = "invalid continuation byte"; startinpos = s - starts; - endinpos = startinpos + 1; - while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80) - endinpos++; + endinpos = startinpos + ch - 1; break; default: if (unicode_putchar(&unicode, &outpos, ch) < 0)