diff -r bfe042a363e6 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Sat Oct 20 20:13:42 2012 +1000 +++ b/Objects/unicodeobject.c Sat Oct 20 21:03:13 2012 +0300 @@ -4802,14 +4802,8 @@ Py_ssize_t outpos; PyObject *unicode; const unsigned char *q, *e; - int bo = 0; /* assume native ordering by default */ + int le, bo = 0; /* assume native ordering by default */ const char *errmsg = ""; - /* Offsets from q for retrieving bytes in the right order. */ -#if PY_LITTLE_ENDIAN - int iorder[] = {0, 1, 2, 3}; -#else - int iorder[] = {3, 2, 1, 0}; -#endif PyObject *errorHandler = NULL; PyObject *exc = NULL; @@ -4823,82 +4817,82 @@ byte order setting accordingly. In native mode, the leading BOM mark is skipped, in all other modes, it is copied to the output stream as-is (giving a ZWNBSP character). */ - if (bo == 0) { - if (size >= 4) { - const Py_UCS4 bom = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) | - (q[iorder[1]] << 8) | q[iorder[0]]; -#if PY_LITTLE_ENDIAN - if (bom == 0x0000FEFF) { - q += 4; - bo = -1; - } - else if (bom == 0xFFFE0000) { - q += 4; - bo = 1; - } + if (bo == 0 && size >= 4) { + Py_UCS4 bom = (q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0]; + if (bom == 0x0000FEFF) { + bo = -1; + q += 4; + } + else if (bom == 0xFFFE0000) { + bo = 1; + q += 4; + } + if (byteorder) + *byteorder = bo; + } + + if (q == e) { + if (consumed) + *consumed = size; + Py_INCREF(unicode_empty); + return unicode_empty; + } + +#ifdef WORDS_BIGENDIAN + le = bo < 0; #else - if (bom == 0x0000FEFF) { - q += 4; - bo = 1; - } - else if (bom == 0xFFFE0000) { - q += 4; - bo = -1; - } -#endif - } - } - - if (bo == -1) { - /* force LE */ - iorder[0] = 0; - iorder[1] = 1; - iorder[2] = 2; - iorder[3] = 3; - } - else if (bo == 1) { - /* force BE */ - iorder[0] = 3; - iorder[1] = 2; - iorder[2] = 1; - iorder[3] = 0; - } - - /* This might be one to much, because of a BOM */ - unicode = PyUnicode_New((size+3)/4, 127); + le = bo <= 0; +#endif + + unicode = PyUnicode_New((e - q + 3) / 4, 127); if (!unicode) return NULL; - if (size == 0) - return unicode; + outpos = 0; - - while (q < e) { + while (1) { Py_UCS4 ch; + if (e - q >= 4) { + enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + void *data = PyUnicode_DATA(unicode); + Py_UCS4 maxch = PyUnicode_MAX_CHAR_VALUE(unicode); + const unsigned char *last = e - 4; + if (le) + while (q <= last) { + ch = (q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0]; + if (ch > maxch) + goto Overflow; + PyUnicode_WRITE(kind, data, outpos++, ch); + q += 4; + } + else + while (q <= last) { + ch = (q[0] << 24) | (q[1] << 16) | (q[2] << 8) | q[3]; + if (ch > maxch) + goto Overflow; + PyUnicode_WRITE(kind, data, outpos++, ch); + q += 4; + } + } /* remaining bytes at the end? (size should be divisible by 4) */ - if (e-q<4) { - if (consumed) - break; - errmsg = "truncated data"; - startinpos = ((const char *)q)-starts; - endinpos = ((const char *)e)-starts; - goto utf32Error; - /* The remaining input chars are ignored if the callback - chooses to skip the input */ - } - ch = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) | - (q[iorder[1]] << 8) | q[iorder[0]]; - - if (ch >= 0x110000) - { - errmsg = "codepoint not in range(0x110000)"; - startinpos = ((const char *)q)-starts; - endinpos = startinpos+4; - goto utf32Error; - } - if (unicode_putchar(&unicode, &outpos, ch) < 0) - goto onError; - q += 4; - continue; + if (q == e || consumed) + break; + errmsg = "truncated data"; + startinpos = ((const char *)q) - starts; + endinpos = ((const char *)e) - starts; + goto utf32Error; + /* The remaining input chars are ignored if the callback + chooses to skip the input */ +Overflow: + if (ch < 0x110000) { + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; + q += 4; + continue; + } + errmsg = "codepoint not in range(0x110000)"; + startinpos = ((const char *)q) - starts; + endinpos = startinpos + 4; + goto utf32Error; utf32Error: if (unicode_decode_call_errorhandler( errors, &errorHandler, @@ -4908,9 +4902,6 @@ goto onError; } - if (byteorder) - *byteorder = bo; - if (consumed) *consumed = (const char *)q-starts;