diff -r c820aa9c0c00 Objects/stringlib/codecs.h --- a/Objects/stringlib/codecs.h Fri Apr 20 18:04:03 2012 -0400 +++ b/Objects/stringlib/codecs.h Sun Apr 22 15:02:05 2012 +0300 @@ -21,15 +21,15 @@ const char **src_pos, Py_ssize_t *dest_index) { int ret; - Py_ssize_t n; const char *s = start; const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK); STRINGLIB_CHAR *p = dest; while (s < end) { - Py_UCS4 ch = (unsigned char)*s; + Py_UCS4 ch; + int ch1 = (signed char)*s; - if (ch < 0x80) { + if (ch1 >= 0) { /* Fast path for runs of ASCII characters. Given that common UTF-8 input will consist of an overwhelming majority of ASCII characters, we try to optimize for this case by checking @@ -48,15 +48,33 @@ unsigned long value = *(unsigned long *) _s; if (value & ASCII_CHAR_MASK) break; - _p[0] = _s[0]; - _p[1] = _s[1]; - _p[2] = _s[2]; - _p[3] = _s[3]; -#if (SIZEOF_LONG == 8) - _p[4] = _s[4]; - _p[5] = _s[5]; - _p[6] = _s[6]; - _p[7] = _s[7]; +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + _p[0] = (STRINGLIB_CHAR)(value & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); +#if SIZEOF_LONG == 8 + _p[4] = (STRINGLIB_CHAR)((value >> 32) & 0xFFu); + _p[5] = (STRINGLIB_CHAR)((value >> 40) & 0xFFu); + _p[6] = (STRINGLIB_CHAR)((value >> 48) & 0xFFu); + _p[7] = (STRINGLIB_CHAR)((value >> 56) & 0xFFu); +#endif +#else +#if SIZEOF_LONG == 8 + _p[0] = (STRINGLIB_CHAR)((value >> 56) & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 48) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 40) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)((value >> 32) & 0xFFu); + _p[4] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); + _p[5] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[6] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[7] = (STRINGLIB_CHAR)(value & 0xFFu); +#else + _p[0] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)(value & 0xFFu); +#endif #endif _s += SIZEOF_LONG; _p += SIZEOF_LONG; @@ -65,80 +83,100 @@ p = _p; if (s == end) break; - ch = (unsigned char)*s; + ch1 = (signed char)*s; + } + if (ch1 >= 0) { + s++; + *p++ = ch1; + continue; } } - if (ch < 0x80) { - s++; + if (ch1 < -0x3E) + goto _error; + + if (ch1 < -0x20) { + int ch2; + if (end - s < 2) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + goto _error; + } + ch2 = (signed char)s[1]; + if (ch2 >= -0x40) + /* invalid continuation byte */ + goto _error; + ch = (ch1 << 6) + ch2 + 010200; + assert ((ch > 0x007F) && (ch <= 0x07FF)); + s += 2; *p++ = ch; continue; } - n = utf8_code_length[ch]; - - if (s + n > end) { - /* unexpected end of data: the caller will decide whether - it's an error or not */ - goto _error; - } - - switch (n) { - case 0: - /* invalid start byte */ - goto _error; - case 1: - /* internal error */ - goto _error; - case 2: - if ((s[1] & 0xc0) != 0x80) +#if STRINGLIB_SIZEOF_CHAR >= 2 + if (ch1 < -0x10) { + int ch2, ch3; + if (end - s < 3) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + goto _error; + } + ch2 = (signed char)s[1]; + ch3 = (signed char)s[2]; + if (ch2 >= -0x40 || ch3 >= -0x40) /* invalid continuation byte */ goto _error; - ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f); - assert ((ch > 0x007F) && (ch <= 0x07FF)); - s += 2; - *p++ = ch; - break; - - case 3: - /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf - will result in surrogates in range d800-dfff. Surrogates are - not valid UTF-8 so they are rejected. - See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf - (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - ((unsigned char)s[0] == 0xE0 && - (unsigned char)s[1] < 0xA0) || - ((unsigned char)s[0] == 0xED && - (unsigned char)s[1] > 0x9F)) { - /* invalid continuation byte */ + if (ch1 == -0x20) { + if (ch2 < -0x60) + /* invalid sequence */ + goto _error; + } + else if (ch1 == -0x13 && ch2 >= -0x60) { + /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf + will result in surrogates in range d800-dfff. Surrogates are + not valid UTF-8 so they are rejected. + See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf + (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ goto _error; } - ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); + ch = (ch1 << 12) + (ch2 << 6) + ch3 + 0420200; assert ((ch > 0x07FF) && (ch <= 0xFFFF)); s += 3; *p++ = ch; - break; + continue; + } - case 4: - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - (s[3] & 0xc0) != 0x80 || - ((unsigned char)s[0] == 0xF0 && - (unsigned char)s[1] < 0x90) || - ((unsigned char)s[0] == 0xF4 && - (unsigned char)s[1] > 0x8F)) { +#if STRINGLIB_SIZEOF_CHAR >= 4 + if (ch1 < -0x0B) { + int ch2, ch3, ch4; + if (end - s < 4) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + goto _error; + } + ch2 = (signed char)s[1]; + ch3 = (signed char)s[2]; + ch4 = (signed char)s[3]; + if (ch2 >= -0x40 || ch3 >= -0x40 || ch4 >= -0x40) /* invalid continuation byte */ goto _error; + if (ch1 == -0x10) { + if (ch2 < -0x70) + /* invalid sequence */ + goto _error; } - ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) + - ((s[2] & 0x3f) << 6) + (s[3] & 0x3f); + else if (ch1 == -0x0C && ch2 >= -0x70) + /* invalid sequence */ + goto _error; + ch = (ch1 << 18) + (ch2 << 12) + (ch3 << 6) + ch4 + 022020200; assert ((ch > 0xFFFF) && (ch <= 0x10ffff)); s += 4; *p++ = ch; - break; + continue; } +#endif +#endif + goto _error; } ret = 0; goto _ok;