diff -r bcf64c1c92f6 Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Wed May 14 21:51:37 2014 +0300 +++ b/Lib/test/test_codecs.py Thu May 15 12:41:51 2014 +0300 @@ -2807,6 +2807,9 @@ ('[\u20ac]', 'replace', b'[?]'), ('[\xff]', 'backslashreplace', b'[\\xff]'), ('[\xff]', 'xmlcharrefreplace', b'[ÿ]'), + ('\udcff', 'strict', None), + ('[\udcff]', 'surrogateescape', b'[\xff]'), + ('[\udcff]', 'surrogatepass', None), )) self.check_decode(932, ( (b'abc', 'strict', 'abc'), @@ -2816,6 +2819,7 @@ (b'[\xff]', 'ignore', '[]'), (b'[\xff]', 'replace', '[\ufffd]'), (b'[\xff]', 'surrogateescape', '[\udcff]'), + (b'[\xff]', 'surrogatepass', None), (b'\x81\x00abc', 'strict', None), (b'\x81\x00abc', 'ignore', '\x00abc'), (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), @@ -2826,14 +2830,23 @@ ('abc', 'strict', b'abc'), ('\xe9\u20ac', 'strict', b'\xe9\x80'), ('\xff', 'strict', b'\xff'), + # test error handlers ('\u0141', 'strict', None), ('\u0141', 'ignore', b''), ('\u0141', 'replace', b'L'), + ('\udc98', 'surrogateescape', b'\x98'), + ('\udc98', 'surrogatepass', None), )) self.check_decode(1252, ( (b'abc', 'strict', 'abc'), (b'\xe9\x80', 'strict', '\xe9\u20ac'), (b'\xff', 'strict', '\xff'), + # invalid bytes + (b'[\x98]', 'strict', None), + (b'[\x98]', 'ignore', '[]'), + (b'[\x98]', 'replace', '[\ufffd]'), + (b'[\x98]', 'surrogateescape', '[\udc98]'), + (b'[\x98]', 'surrogatepass', None), )) def test_cp_utf7(self): diff -r bcf64c1c92f6 Python/codecs.c --- a/Python/codecs.c Wed May 14 21:51:37 2014 +0300 +++ b/Python/codecs.c Thu May 15 12:41:51 2014 +0300 @@ -901,6 +901,7 @@ } } +#define ENC_UNKNOWN -1 #define ENC_UTF8 0 #define ENC_UTF16BE 1 #define ENC_UTF16LE 2 @@ -916,7 +917,11 @@ encoding += 3; if (*encoding == '-' || *encoding == '_' ) encoding++; - if (encoding[0] == '1' && encoding[1] == '6') { + if (encoding[0] == '8' && encoding[1] == '\0') { + *bytelength = 3; + return ENC_UTF8; + } + else if (encoding[0] == '1' && encoding[1] == '6') { encoding += 2; *bytelength = 2; if (*encoding == '\0') { @@ -955,9 +960,7 @@ } } } - /* utf-8 */ - *bytelength = 3; - return ENC_UTF8; + return ENC_UNKNOWN; } /* This handler is declared static until someone demonstrates @@ -994,6 +997,12 @@ } code = get_standard_encoding(encoding, &bytelength); Py_DECREF(encode); + if (code == ENC_UNKNOWN) { + /* Not supported, fail with original exception */ + PyErr_SetObject(PyExceptionInstance_Class(exc), exc); + Py_DECREF(object); + return NULL; + } res = PyBytes_FromStringAndSize(NULL, bytelength*(end-start)); if (!res) { @@ -1068,6 +1077,12 @@ } code = get_standard_encoding(encoding, &bytelength); Py_DECREF(encode); + if (code == ENC_UNKNOWN) { + /* Not supported, fail with original exception */ + PyErr_SetObject(PyExceptionInstance_Class(exc), exc); + Py_DECREF(object); + return NULL; + } /* Try decoding a single surrogate character. If there are more, let the codec call us again. */