diff -r 5970c90dd8d1 Lib/test/test_codeccallbacks.py --- a/Lib/test/test_codeccallbacks.py Fri Jan 25 23:30:50 2013 +0200 +++ b/Lib/test/test_codeccallbacks.py Sat Jan 26 00:51:30 2013 +0200 @@ -262,12 +262,12 @@ self.assertEqual( "\\u3042\u3xxx".decode("unicode-escape", "test.handler1"), - u"\u3042[<92><117><51><120>]xx" + u"\u3042[<92><117><51>]xxx" ) self.assertEqual( "\\u3042\u3xx".decode("unicode-escape", "test.handler1"), - u"\u3042[<92><117><51><120><120>]" + u"\u3042[<92><117><51>]xx" ) self.assertEqual( diff -r 5970c90dd8d1 Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Fri Jan 25 23:30:50 2013 +0200 +++ b/Lib/test/test_codecs.py Sat Jan 26 00:51:30 2013 +0200 @@ -1786,6 +1786,84 @@ self.assertEqual(srw.read(), u"\xfc") +class UnicodeEscapeTest(unittest.TestCase): + def test_empty(self): + self.assertEqual(codecs.unicode_escape_encode(u""), ("", 0)) + self.assertEqual(codecs.unicode_escape_decode(""), (u"", 0)) + + def test_raw_encode(self): + encode = codecs.unicode_escape_encode + for b in range(32, 127): + if b != ord('\\'): + self.assertEqual(encode(unichr(b)), (chr(b), 1)) + + def test_raw_decode(self): + decode = codecs.unicode_escape_decode + for b in range(256): + if b != ord('\\'): + self.assertEqual(decode(chr(b) + '0'), (unichr(b) + u'0', 2)) + + def test_escape_encode(self): + encode = codecs.unicode_escape_encode + self.assertEqual(encode(u'\t'), (r'\t', 1)) + self.assertEqual(encode(u'\n'), (r'\n', 1)) + self.assertEqual(encode(u'\r'), (r'\r', 1)) + self.assertEqual(encode(u'\\'), (r'\\', 1)) + for b in range(32): + if chr(b) not in '\t\n\r': + self.assertEqual(encode(unichr(b)), ('\\x%02x' % b, 1)) + for b in range(127, 256): + self.assertEqual(encode(unichr(b)), ('\\x%02x' % b, 1)) + self.assertEqual(encode(u'\u20ac'), (r'\u20ac', 1)) + self.assertEqual(encode(u'\U0001d120'), + (r'\U0001d120', len(u'\U0001d120'))) + + def test_escape_decode(self): + decode = codecs.unicode_escape_decode + self.assertEqual(decode("[\\\n]"), (u"[]", 4)) + self.assertEqual(decode(r'[\"]'), (u'["]', 4)) + self.assertEqual(decode(r"[\']"), (u"[']", 4)) + self.assertEqual(decode(r"[\\]"), (ur"[\]", 4)) + self.assertEqual(decode(r"[\a]"), (u"[\x07]", 4)) + self.assertEqual(decode(r"[\b]"), (u"[\x08]", 4)) + self.assertEqual(decode(r"[\t]"), (u"[\x09]", 4)) + self.assertEqual(decode(r"[\n]"), (u"[\x0a]", 4)) + self.assertEqual(decode(r"[\v]"), (u"[\x0b]", 4)) + self.assertEqual(decode(r"[\f]"), (u"[\x0c]", 4)) + self.assertEqual(decode(r"[\r]"), (u"[\x0d]", 4)) + self.assertEqual(decode(r"[\7]"), (u"[\x07]", 4)) + self.assertEqual(decode(r"[\8]"), (ur"[\8]", 4)) + self.assertEqual(decode(r"[\78]"), (u"[\x078]", 5)) + self.assertEqual(decode(r"[\41]"), (u"[!]", 5)) + self.assertEqual(decode(r"[\418]"), (u"[!8]", 6)) + self.assertEqual(decode(r"[\101]"), (u"[A]", 6)) + self.assertEqual(decode(r"[\1010]"), (u"[A0]", 7)) + self.assertEqual(decode(r"[\x41]"), (u"[A]", 6)) + self.assertEqual(decode(r"[\x410]"), (u"[A0]", 7)) + self.assertEqual(decode(r"\u20ac"), (u"\u20ac", 6)) + self.assertEqual(decode(r"\U0001d120"), (u"\U0001d120", 10)) + for b in range(256): + if chr(b) not in '\n"\'\\abtnvfr01234567xuUN': + self.assertEqual(decode('\\' + chr(b)), + (u'\\' + unichr(b), 2)) + + def test_decode_errors(self): + decode = codecs.unicode_escape_decode + for c, d in ('x', 2), ('u', 4), ('U', 4): + for i in range(d): + self.assertRaises(UnicodeDecodeError, decode, + "\\" + c + "0"*i) + self.assertRaises(UnicodeDecodeError, decode, + "[\\" + c + "0"*i + "]") + data = "[\\" + c + "0"*i + "]\\" + c + "0"*i + self.assertEqual(decode(data, "ignore"), (u"[]", len(data))) + self.assertEqual(decode(data, "replace"), + (u"[\ufffd]\ufffd", len(data))) + self.assertRaises(UnicodeDecodeError, decode, r"\U00110000") + self.assertEqual(decode(r"\U00110000", "ignore"), (u"", 10)) + self.assertEqual(decode(r"\U00110000", "replace"), (u"\ufffd", 10)) + + class BomTest(unittest.TestCase): def test_seek0(self): data = u"1234567890" @@ -1871,6 +1949,7 @@ BasicStrTest, CharmapTest, WithStmtTest, + UnicodeEscapeTest, BomTest, ) diff -r 5970c90dd8d1 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Jan 25 23:30:50 2013 +0200 +++ b/Objects/unicodeobject.c Sat Jan 26 00:51:30 2013 +0200 @@ -2732,7 +2732,6 @@ Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; - int i; PyUnicodeObject *v; Py_UNICODE *p; const char *end; @@ -2818,29 +2817,19 @@ message = "truncated \\UXXXXXXXX escape"; hexescape: chr = 0; - outpos = p-PyUnicode_AS_UNICODE(v); - if (s+digits>end) { - endinpos = size; - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", "end of string in escape sequence", - starts, size, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; - goto nextByte; + if (end - s < digits) { + /* count only hex digits */ + for (; s < end; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; + } + goto error; } - for (i = 0; i < digits; ++i) { - c = (unsigned char) s[i]; - if (!isxdigit(c)) { - endinpos = (s+i+1)-starts; - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", message, - starts, size, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; - goto nextByte; - } + for (; digits--; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; chr = (chr<<4) & ~0xF; if (c >= '0' && c <= '9') chr += c - '0'; @@ -2849,7 +2838,6 @@ else chr += 10 + c - 'A'; } - s += i; if (chr == 0xffffffff && PyErr_Occurred()) /* _decoding_error will have already written into the target buffer. */ @@ -2870,14 +2858,8 @@ *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); #endif } else { - endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", "illegal Unicode character", - starts, size, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; + message = "illegal Unicode character"; + goto error; } break; @@ -2904,28 +2886,13 @@ goto store; } } - endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", message, - starts, size, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; - break; + goto error; default: if (s > end) { message = "\\ at end of string"; s--; - endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "unicodeescape", message, - starts, size, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) - goto onError; + goto error; } else { *p++ = '\\'; @@ -2933,8 +2900,18 @@ } break; } - nextByte: - ; + continue; + + error: + endinpos = s-starts; + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicodeescape", message, + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) + goto onError; + continue; } if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) goto onError;