diff -r d8c2ce63f5a4 Lib/test/test_codeccallbacks.py --- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:53:29 2013 +0200 +++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:52:02 2013 +0200 @@ -271,12 +271,12 @@ self.assertEqual( b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1"), - "\u3042[<92><117><51><120>]xx" + "\u3042[<92><117><51>]xxx" ) self.assertEqual( b"\\u3042\u3xx".decode("unicode-escape", "test.handler1"), - "\u3042[<92><117><51><120><120>]" + "\u3042[<92><117><51>]xx" ) self.assertEqual( diff -r d8c2ce63f5a4 Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Fri Jan 25 23:53:29 2013 +0200 +++ b/Lib/test/test_codecs.py Sat Jan 26 00:52:02 2013 +0200 @@ -2009,6 +2009,84 @@ self.assertRaises(UnicodeDecodeError, codecs.raw_unicode_escape_decode, br"\U00110000") self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10)) + +class UnicodeEscapeTest(unittest.TestCase): + def test_empty(self): + self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) + self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) + + def test_raw_encode(self): + encode = codecs.unicode_escape_encode + for b in range(32, 127): + if b != b'\\'[0]: + self.assertEqual(encode(chr(b)), (bytes([b]), 1)) + + def test_raw_decode(self): + decode = codecs.unicode_escape_decode + for b in range(256): + if b != b'\\'[0]: + self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2)) + + def test_escape_encode(self): + encode = codecs.unicode_escape_encode + self.assertEqual(encode('\t'), (br'\t', 1)) + self.assertEqual(encode('\n'), (br'\n', 1)) + self.assertEqual(encode('\r'), (br'\r', 1)) + self.assertEqual(encode('\\'), (br'\\', 1)) + for b in range(32): + if chr(b) not in '\t\n\r': + self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1)) + for b in range(127, 256): + self.assertEqual(encode(chr(b)), (('\\x%02x' % b).encode(), 1)) + self.assertEqual(encode('\u20ac'), (br'\u20ac', 1)) + self.assertEqual(encode('\U0001d120'), (br'\U0001d120', 1)) + + def test_escape_decode(self): + decode = codecs.unicode_escape_decode + self.assertEqual(decode(b"[\\\n]"), ("[]", 4)) + self.assertEqual(decode(br'[\"]'), ('["]', 4)) + self.assertEqual(decode(br"[\']"), ("[']", 4)) + self.assertEqual(decode(br"[\\]"), (r"[\]", 4)) + self.assertEqual(decode(br"[\a]"), ("[\x07]", 4)) + self.assertEqual(decode(br"[\b]"), ("[\x08]", 4)) + self.assertEqual(decode(br"[\t]"), ("[\x09]", 4)) + self.assertEqual(decode(br"[\n]"), ("[\x0a]", 4)) + self.assertEqual(decode(br"[\v]"), ("[\x0b]", 4)) + self.assertEqual(decode(br"[\f]"), ("[\x0c]", 4)) + self.assertEqual(decode(br"[\r]"), ("[\x0d]", 4)) + self.assertEqual(decode(br"[\7]"), ("[\x07]", 4)) + self.assertEqual(decode(br"[\8]"), (r"[\8]", 4)) + self.assertEqual(decode(br"[\78]"), ("[\x078]", 5)) + self.assertEqual(decode(br"[\41]"), ("[!]", 5)) + self.assertEqual(decode(br"[\418]"), ("[!8]", 6)) + self.assertEqual(decode(br"[\101]"), ("[A]", 6)) + self.assertEqual(decode(br"[\1010]"), ("[A0]", 7)) + self.assertEqual(decode(br"[\x41]"), ("[A]", 6)) + self.assertEqual(decode(br"[\x410]"), ("[A0]", 7)) + self.assertEqual(decode(br"\u20ac"), ("\u20ac", 6)) + self.assertEqual(decode(br"\U0001d120"), ("\U0001d120", 10)) + for b in range(256): + if b not in b'\n"\'\\abtnvfr01234567xuUN': + self.assertEqual(decode(b'\\' + bytes([b])), + ('\\' + chr(b), 2)) + + def test_decode_errors(self): + decode = codecs.unicode_escape_decode + for c, d in (b'x', 2), (b'u', 4), (b'U', 4): + for i in range(d): + self.assertRaises(UnicodeDecodeError, decode, + b"\\" + c + b"0"*i) + self.assertRaises(UnicodeDecodeError, decode, + b"[\\" + c + b"0"*i + b"]") + data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i + self.assertEqual(decode(data, "ignore"), ("[]", len(data))) + self.assertEqual(decode(data, "replace"), + ("[\ufffd]\ufffd", len(data))) + self.assertRaises(UnicodeDecodeError, decode, br"\U00110000") + self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) + self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) + + class SurrogateEscapeTest(unittest.TestCase): def test_utf8(self): diff -r d8c2ce63f5a4 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Jan 25 23:53:29 2013 +0200 +++ b/Objects/unicodeobject.c Sat Jan 26 00:52:02 2013 +0200 @@ -5376,7 +5376,6 @@ const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; - int j; _PyUnicodeWriter writer; const char *end; char* message; @@ -5500,28 +5499,19 @@ message = "truncated \\UXXXXXXXX escape"; hexescape: chr = 0; - if (s+digits>end) { - endinpos = size; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "unicodeescape", "end of string in escape sequence", - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - goto nextByte; - } - for (j = 0; j < digits; ++j) { - c = (unsigned char) s[j]; - if (!Py_ISXDIGIT(c)) { - endinpos = (s+j+1)-starts; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - goto nextByte; + if (end - s < digits) { + /* count only hex digits */ + for (; s < end; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; } + goto error; + } + for (; digits--; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; chr = (chr<<4) & ~0xF; if (c >= '0' && c <= '9') chr += c - '0'; @@ -5530,24 +5520,16 @@ else chr += 10 + c - 'A'; } - s += j; if (chr == 0xffffffff && PyErr_Occurred()) /* _decoding_error will have already written into the target buffer. */ break; store: /* when we get here, chr is a 32-bit unicode character */ - if (chr <= MAX_UNICODE) { - WRITECHAR(chr); - } else { - endinpos = s-starts; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "unicodeescape", "illegal Unicode character", - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - } + message = "illegal Unicode character"; + if (chr > MAX_UNICODE) + goto error; + WRITECHAR(chr); break; /* \N{name} */ @@ -5575,26 +5557,13 @@ goto store; } } - endinpos = s-starts; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - break; + goto error; default: if (s > end) { message = "\\ at end of string"; s--; - endinpos = s-starts; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + goto error; } else { WRITECHAR('\\'); @@ -5602,8 +5571,17 @@ } break; } - nextByte: - ; + continue; + + error: + endinpos = s-starts; + if (unicode_decode_call_errorhandler_writer( + errors, &errorHandler, + "unicodeescape", message, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) + goto onError; + continue; } #undef WRITECHAR