diff -r bfe042a363e6 Objects/stringlib/codecs.h --- a/Objects/stringlib/codecs.h Sat Oct 20 20:13:42 2012 +1000 +++ b/Objects/stringlib/codecs.h Sat Oct 20 21:29:51 2012 +0300 @@ -626,4 +626,65 @@ #endif #undef SWAB2 } + + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(utf32_decode)(const unsigned char **inptr, const unsigned char *end, + STRINGLIB_CHAR *dest, Py_ssize_t *outpos, + int native_ordering) +{ + Py_UCS4 ch; + const unsigned char *s = *inptr; + const unsigned char *last = end - 4; + STRINGLIB_CHAR *p = dest + *outpos; + + if (_Py_IS_ALIGNED(s, 4)) { + if (native_ordering) + while (s <= last) { + ch = *(const PY_UINT32_T *)s; + if (ch > STRINGLIB_MAX_CHAR) + goto Overflow; + *p++ = ch; + s += 4; + } + else + while (s <= last) { + ch = *(const PY_UINT32_T *)s; + /* 1234 -> 2143 */ + ch = ((ch & 0xFF00FF00u) >> 8) | ((ch & 0x00FF00FFu) << 8); + /* 2143 -> 4321 */ + ch = ((ch & 0xFFFF0000u) >> 16) | ((ch & 0x0000FFFFu) << 16); + if (ch > STRINGLIB_MAX_CHAR) + goto Overflow; + *p++ = ch; + s += 4; + } + } else { +#ifdef WORDS_BIGENDIAN + if (!native_ordering) +#else + if (native_ordering) +#endif + while (s <= last) { + ch = (s[3] << 24) | (s[2] << 16) | (s[1] << 8) | s[0]; + if (ch > STRINGLIB_MAX_CHAR) + goto Overflow; + *p++ = ch; + s += 4; + } + else + while (s <= last) { + ch = (s[0] << 24) | (s[1] << 16) | (s[2] << 8) | s[3]; + if (ch > STRINGLIB_MAX_CHAR) + goto Overflow; + *p++ = ch; + s += 4; + } + } + ch = 0; +Overflow: + *inptr = s; + *outpos = p - dest; + return ch; +} #endif /* STRINGLIB_IS_UNICODE */ diff -r bfe042a363e6 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Sat Oct 20 20:13:42 2012 +1000 +++ b/Objects/unicodeobject.c Sat Oct 20 21:29:51 2012 +0300 @@ -4802,14 +4802,8 @@ Py_ssize_t outpos; PyObject *unicode; const unsigned char *q, *e; - int bo = 0; /* assume native ordering by default */ + int native_ordering, bo = 0; /* assume native ordering by default */ const char *errmsg = ""; - /* Offsets from q for retrieving bytes in the right order. */ -#if PY_LITTLE_ENDIAN - int iorder[] = {0, 1, 2, 3}; -#else - int iorder[] = {3, 2, 1, 0}; -#endif PyObject *errorHandler = NULL; PyObject *exc = NULL; @@ -4823,94 +4817,97 @@ byte order setting accordingly. In native mode, the leading BOM mark is skipped, in all other modes, it is copied to the output stream as-is (giving a ZWNBSP character). */ - if (bo == 0) { - if (size >= 4) { - const Py_UCS4 bom = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) | - (q[iorder[1]] << 8) | q[iorder[0]]; -#if PY_LITTLE_ENDIAN - if (bom == 0x0000FEFF) { + if (bo == 0 && size >= 4) { + Py_UCS4 bom = (q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0]; + if (bom == 0x0000FEFF) { + bo = -1; + q += 4; + } + else if (bom == 0xFFFE0000) { + bo = 1; + q += 4; + } + if (byteorder) + *byteorder = bo; + } + + if (q == e) { + if (consumed) + *consumed = size; + Py_INCREF(unicode_empty); + return unicode_empty; + } + +#ifdef WORDS_BIGENDIAN + native_ordering = bo >= 0; +#else + native_ordering = bo <= 0; +#endif + + unicode = PyUnicode_New((e - q + 3) / 4, 127); + if (!unicode) + return NULL; + + outpos = 0; + while (1) { + Py_UCS4 ch = 0; + if (e - q >= 4) { + enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(unicode)) + ch = asciilib_utf32_decode(&q, e, + PyUnicode_1BYTE_DATA(unicode), &outpos, + native_ordering); + else + ch = ucs1lib_utf32_decode(&q, e, + PyUnicode_1BYTE_DATA(unicode), &outpos, + native_ordering); + break; + case PyUnicode_2BYTE_KIND: + ch = ucs2lib_utf32_decode(&q, e, + PyUnicode_2BYTE_DATA(unicode), &outpos, + native_ordering); + break; + case PyUnicode_4BYTE_KIND: + ch = ucs4lib_utf32_decode(&q, e, + PyUnicode_4BYTE_DATA(unicode), &outpos, + native_ordering); + break; + default: + assert(0); + } + } + if (ch) { + if (ch < 0x110000) { + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; q += 4; - bo = -1; - } - else if (bom == 0xFFFE0000) { - q += 4; - bo = 1; - } -#else - if (bom == 0x0000FEFF) { - q += 4; - bo = 1; - } - else if (bom == 0xFFFE0000) { - q += 4; - bo = -1; - } -#endif - } - } - - if (bo == -1) { - /* force LE */ - iorder[0] = 0; - iorder[1] = 1; - iorder[2] = 2; - iorder[3] = 3; - } - else if (bo == 1) { - /* force BE */ - iorder[0] = 3; - iorder[1] = 2; - iorder[2] = 1; - iorder[3] = 0; - } - - /* This might be one to much, because of a BOM */ - unicode = PyUnicode_New((size+3)/4, 127); - if (!unicode) - return NULL; - if (size == 0) - return unicode; - outpos = 0; - - while (q < e) { - Py_UCS4 ch; - /* remaining bytes at the end? (size should be divisible by 4) */ - if (e-q<4) { - if (consumed) + continue; + } + errmsg = "codepoint not in range(0x110000)"; + startinpos = ((const char *)q)-starts; + endinpos = startinpos+4; + } + else { + /* remaining bytes at the end? (size should be divisible by 4) */ + if (q == e || consumed) break; errmsg = "truncated data"; startinpos = ((const char *)q)-starts; endinpos = ((const char *)e)-starts; - goto utf32Error; /* The remaining input chars are ignored if the callback chooses to skip the input */ } - ch = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) | - (q[iorder[1]] << 8) | q[iorder[0]]; - - if (ch >= 0x110000) - { - errmsg = "codepoint not in range(0x110000)"; - startinpos = ((const char *)q)-starts; - endinpos = startinpos+4; - goto utf32Error; - } - if (unicode_putchar(&unicode, &outpos, ch) < 0) - goto onError; - q += 4; - continue; - utf32Error: if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf32", errmsg, - &starts, (const char **)&e, &startinpos, &endinpos, &exc, (const char **)&q, + &starts, (const char **)&e, &startinpos, &endinpos, + &exc, (const char **)&q, &unicode, &outpos)) goto onError; } - if (byteorder) - *byteorder = bo; - if (consumed) *consumed = (const char *)q-starts;