diff -r 22594c5060eb Lib/test/test_codeccallbacks.py --- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:31:43 2013 +0200 +++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:51:39 2013 +0200 @@ -262,12 +262,12 @@ self.assertEqual( b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"), - "\u3042[<92><117><51><120>]xx" + "\u3042[<92><117><51>]xxx" ) self.assertEqual( b"\\u3042\u3xx".decode("unicode-escape", "test.handler1"), - "\u3042[<92><117><51><120><120>]" + "\u3042[<92><117><51>]xx" ) self.assertEqual( diff -r 22594c5060eb Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Fri Jan 25 23:31:43 2013 +0200 +++ b/Lib/test/test_codecs.py Sat Jan 26 00:51:39 2013 +0200 @@ -1846,6 +1846,85 @@ self.assertEqual(codecs.raw_unicode_escape_decode(r"\u1234"), ("\u1234", 6)) self.assertEqual(codecs.raw_unicode_escape_decode(br"\u1234"), ("\u1234", 6)) + +class UnicodeEscapeTest(unittest.TestCase): + def test_empty(self): + self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) + self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) + + def test_raw_encode(self): + encode = codecs.unicode_escape_encode + for b in range(32, 127): + if b != b'\\'[0]: + self.assertEqual(encode(chr(b)), (bytes([b]), 1)) + + def test_raw_decode(self): + decode = codecs.unicode_escape_decode + for b in range(256): + if b != b'\\'[0]: + self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2)) + + def test_escape_encode(self): + encode = codecs.unicode_escape_encode + self.assertEqual(encode('\t'), (br'\t', 1)) + self.assertEqual(encode('\n'), (br'\n', 1)) + self.assertEqual(encode('\r'), (br'\r', 1)) + self.assertEqual(encode('\\'), (br'\\', 1)) + for b in range(32): + if chr(b) not in '\t\n\r': + self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1)) + for b in range(127, 256): + self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1)) + self.assertEqual(encode('\u20ac'), (br'\u20ac', 1)) + self.assertEqual(encode('\U0001d120'), + (br'\U0001d120', len('\U0001d120'))) + + def test_escape_decode(self): + decode = codecs.unicode_escape_decode + self.assertEqual(decode(b"[\\\n]"), ("[]", 4)) + self.assertEqual(decode(br'[\"]'), ('["]', 4)) + self.assertEqual(decode(br"[\']"), ("[']", 4)) + self.assertEqual(decode(br"[\\]"), (r"[\]", 4)) + self.assertEqual(decode(br"[\a]"), ("[\x07]", 4)) + self.assertEqual(decode(br"[\b]"), ("[\x08]", 4)) + self.assertEqual(decode(br"[\t]"), ("[\x09]", 4)) + self.assertEqual(decode(br"[\n]"), ("[\x0a]", 4)) + self.assertEqual(decode(br"[\v]"), ("[\x0b]", 4)) + self.assertEqual(decode(br"[\f]"), ("[\x0c]", 4)) + self.assertEqual(decode(br"[\r]"), ("[\x0d]", 4)) + self.assertEqual(decode(br"[\7]"), ("[\x07]", 4)) + self.assertEqual(decode(br"[\8]"), (r"[\8]", 4)) + self.assertEqual(decode(br"[\78]"), ("[\x078]", 5)) + self.assertEqual(decode(br"[\41]"), ("[!]", 5)) + self.assertEqual(decode(br"[\418]"), ("[!8]", 6)) + self.assertEqual(decode(br"[\101]"), ("[A]", 6)) + self.assertEqual(decode(br"[\1010]"), ("[A0]", 7)) + self.assertEqual(decode(br"[\x41]"), ("[A]", 6)) + self.assertEqual(decode(br"[\x410]"), ("[A0]", 7)) + self.assertEqual(decode(br"\u20ac"), ("\u20ac", 6)) + self.assertEqual(decode(br"\U0001d120"), ("\U0001d120", 10)) + for b in range(256): + if b not in b'\n"\'\\abtnvfr01234567xuUN': + self.assertEqual(decode(b'\\' + bytes([b])), + ('\\' + chr(b), 2)) + + def test_decode_errors(self): + decode = codecs.unicode_escape_decode + for c, d in (b'x', 2), (b'u', 4), (b'U', 4): + for i in range(d): + self.assertRaises(UnicodeDecodeError, decode, + b"\\" + c + b"0"*i) + self.assertRaises(UnicodeDecodeError, decode, + b"[\\" + c + b"0"*i + b"]") + data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i + self.assertEqual(decode(data, "ignore"), ("[]", len(data))) + self.assertEqual(decode(data, "replace"), + ("[\ufffd]\ufffd", len(data))) + self.assertRaises(UnicodeDecodeError, decode, br"\U00110000") + self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) + self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) + + class SurrogateEscapeTest(unittest.TestCase): def test_utf8(self): @@ -2011,6 +2090,7 @@ CharmapTest, WithStmtTest, TypesTest, + UnicodeEscapeTest, SurrogateEscapeTest, BomTest, TransformCodecTest, diff -r 22594c5060eb Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Jan 25 23:31:43 2013 +0200 +++ b/Objects/unicodeobject.c Sat Jan 26 00:51:39 2013 +0200 @@ -3756,7 +3756,6 @@ Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; - int i; PyUnicodeObject *v; Py_UNICODE *p; const char *end; @@ -3842,29 +3841,19 @@ message = "truncated \\UXXXXXXXX escape"; hexescape: chr = 0; - outpos = p-PyUnicode_AS_UNICODE(v); - if (s+digits>end) { - endinpos = size; - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", "end of string in escape sequence", - &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; - goto nextByte; - } - for (i = 0; i < digits; ++i) { - c = (unsigned char) s[i]; - if (!Py_ISXDIGIT(c)) { - endinpos = (s+i+1)-starts; - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; - goto nextByte; + if (end - s < digits) { + /* count only hex digits */ + for (; s < end; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; } + goto error; + } + for (; digits--; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; chr = (chr<<4) & ~0xF; if (c >= '0' && c <= '9') chr += c - '0'; @@ -3873,7 +3862,6 @@ else chr += 10 + c - 'A'; } - s += i; if (chr == 0xffffffff && PyErr_Occurred()) /* _decoding_error will have already written into the target buffer. */ @@ -3894,14 +3882,8 @@ *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); #endif } else { - endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", "illegal Unicode character", - &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; + message = "illegal Unicode character"; + goto error; } break; @@ -3928,28 +3910,13 @@ goto store; } } - endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; - break; + goto error; default: if (s > end) { message = "\\ at end of string"; s--; - endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; + goto error; } else { *p++ = '\\'; @@ -3957,8 +3924,18 @@ } break; } - nextByte: - ; + continue; + + error: + endinpos = s-starts; + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", message, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) + goto onError; + continue; } if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) goto onError;