startswith and endswith don't accept None as slice index

Objects/unicodeobject.c

 /*
 
 Unicode implementation based on original code by Fredrik Lundh,
 modified by Marc-Andre Lemburg <mal@lemburg.com> according to the
 Unicode Integration Proposal (see file Misc/unicode.txt).
 
 Major speed upgrades to the method implementations at the Reykjavik
 NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
 
 Copyright (c) Corporation for National Research Initiatives.
@@ skipping 23 matching lines @@
 ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
 OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 --------------------------------------------------------------------
 
 */
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
+#include "bytes_methods.h"
+
+#include "unicodeobject.h"
 #include "ucnhash.h"
 
 #ifdef MS_WINDOWS
 #include <windows.h>
 #endif
 
 /* Limit for the Unicode object free list */
 
 #define PyUnicode_MAXFREELIST       1024
 
@@ skipping 50 matching lines @@
 static PyUnicodeObject *free_list;
 static int numfree;
 
 /* The empty Unicode object is shared to improve performance. */
 static PyUnicodeObject *unicode_empty;
 
 /* Single character Unicode strings in the Latin-1 range are being
    shared as well. */
 static PyUnicodeObject *unicode_latin1[256];
 
+/* Default encoding to use and assume when NULL is passed as encoding
+   parameter; it is fixed to "utf-8".  Always use the
+   PyUnicode_GetDefaultEncoding() API to access this global.
+
+   Don't forget to alter Py_FileSystemDefaultEncoding if you change the
+   hard coded default!
+*/
+static const char unicode_default_encoding[] = "utf-8";
+
 /* Fast detection of the most frequent whitespace characters */
 const unsigned char _Py_ascii_whitespace[] = {
     0, 0, 0, 0, 0, 0, 0, 0,
-/*     case 0x0009: * CHARACTER TABULATION */
+/*     case 0x0009: * HORIZONTAL TABULATION */
 /*     case 0x000A: * LINE FEED */
-/*     case 0x000B: * LINE TABULATION */
+/*     case 0x000B: * VERTICAL TABULATION */
 /*     case 0x000C: * FORM FEED */
 /*     case 0x000D: * CARRIAGE RETURN */
     0, 1, 1, 1, 1, 1, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
 /*     case 0x001C: * FILE SEPARATOR */
 /*     case 0x001D: * GROUP SEPARATOR */
 /*     case 0x001E: * RECORD SEPARATOR */
 /*     case 0x001F: * UNIT SEPARATOR */
     0, 0, 0, 0, 1, 1, 1, 1,
 /*     case 0x0020: * SPACE */
     1, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
 
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0
 };
 
-static PyObject *
-unicode_encode_call_errorhandler(const char *errors,
+static PyObject *unicode_encode_call_errorhandler(const char *errors,
        PyObject **errorHandler,const char *encoding, const char *reason,
        const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
        Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos);
 
-static void
-raise_encode_exception(PyObject **exceptionObject,
-                       const char *encoding,
-                       const Py_UNICODE *unicode, Py_ssize_t size,
-                       Py_ssize_t startpos, Py_ssize_t endpos,
-                       const char *reason);
+static void raise_encode_exception(PyObject **exceptionObject,
+                                   const char *encoding,
+                                   const Py_UNICODE *unicode, Py_ssize_t size,
+                                   Py_ssize_t startpos, Py_ssize_t endpos,
+                                   const char *reason);
 
 /* Same for linebreaks */
 static unsigned char ascii_linebreak[] = {
     0, 0, 0, 0, 0, 0, 0, 0,
 /*         0x000A, * LINE FEED */
-/*         0x000B, * LINE TABULATION */
-/*         0x000C, * FORM FEED */
 /*         0x000D, * CARRIAGE RETURN */
-    0, 0, 1, 1, 1, 1, 0, 0,
+    0, 0, 1, 0, 0, 1, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
 /*         0x001C, * FILE SEPARATOR */
 /*         0x001D, * GROUP SEPARATOR */
 /*         0x001E, * RECORD SEPARATOR */
     0, 0, 0, 0, 1, 1, 1, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0,
 
@@ skipping 21 matching lines @@
 }
 
 /* --- Bloom Filters ----------------------------------------------------- */
 
 /* stuff to implement simple "bloom filters" for Unicode characters.
    to keep things simple, we use a single bitmask, using the least 5
    bits from each unicode characters as the bit index. */
 
 /* the linebreak mask is set up by Unicode_Init below */
 
-#if LONG_BIT >= 128
-#define BLOOM_WIDTH 128
-#elif LONG_BIT >= 64
-#define BLOOM_WIDTH 64
-#elif LONG_BIT >= 32
-#define BLOOM_WIDTH 32
-#else
-#error "LONG_BIT is smaller than 32"
-#endif
-
 #define BLOOM_MASK unsigned long
 
 static BLOOM_MASK bloom_linebreak;
 
-#define BLOOM_ADD(mask, ch) ((mask |= (1UL << ((ch) & (BLOOM_WIDTH - 1)))))
-#define BLOOM(mask, ch)     ((mask &  (1UL << ((ch) & (BLOOM_WIDTH - 1)))))
+#define BLOOM(mask, ch) ((mask & (1 << ((ch) & 0x1F))))
 
 #define BLOOM_LINEBREAK(ch)                                             \
     ((ch) < 128U ? ascii_linebreak[(ch)] :                              \
      (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch)))
 
-Py_LOCAL_INLINE(BLOOM_MASK)
-make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len)
+Py_LOCAL_INLINE(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len)
 {
     /* calculate simple bloom-style bitmask for a given unicode string */
 
-    BLOOM_MASK mask;
+    long mask;
     Py_ssize_t i;
 
     mask = 0;
     for (i = 0; i < len; i++)
-        BLOOM_ADD(mask, ptr[i]);
+        mask |= (1 << (ptr[i] & 0x1F));
 
     return mask;
 }
 
-Py_LOCAL_INLINE(int)
-unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen)
+Py_LOCAL_INLINE(int) unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen)
 {
     Py_ssize_t i;
 
     for (i = 0; i < setlen; i++)
         if (set[i] == chr)
             return 1;
 
     return 0;
 }
 
 #define BLOOM_MEMBER(mask, chr, set, setlen)                    \
     BLOOM(mask, chr) && unicode_member(chr, set, setlen)
 
 /* --- Unicode Object ----------------------------------------------------- */
 
-static int
-unicode_resize(register PyUnicodeObject *unicode,
-               Py_ssize_t length)
+static
+int unicode_resize(register PyUnicodeObject *unicode,
+                   Py_ssize_t length)
 {
     void *oldstr;
 
     /* Shortcut if there's nothing much to do. */
     if (unicode->length == length)
         goto reset;
 
     /* Resizing shared object (unicode_empty or single character
        objects) in-place is not allowed. Use PyUnicode_Resize()
        instead ! */
@@ skipping 35 matching lines @@
 
 /* We allocate one more byte to make sure the string is
    Ux0000 terminated; some code (e.g. new_identifier)
    relies on that.
 
    XXX This allocator could further be enhanced by assuring that the
    free list never reduces its size below 1.
 
 */
 
-static PyUnicodeObject *
-_PyUnicode_New(Py_ssize_t length)
+static
+PyUnicodeObject *_PyUnicode_New(Py_ssize_t length)
 {
     register PyUnicodeObject *unicode;
 
     /* Optimization for empty strings */
     if (length == 0 && unicode_empty != NULL) {
         Py_INCREF(unicode_empty);
         return unicode_empty;
     }
 
     /* Ensure we won't overflow the size. */
@@ skipping 50 matching lines @@
     return unicode;
 
   onError:
     /* XXX UNREF/NEWREF interface should be more symmetrical */
     _Py_DEC_REFTOTAL;
     _Py_ForgetReference((PyObject *)unicode);
     PyObject_Del(unicode);
     return NULL;
 }
 
-static void
-unicode_dealloc(register PyUnicodeObject *unicode)
+static
+void unicode_dealloc(register PyUnicodeObject *unicode)
 {
     switch (PyUnicode_CHECK_INTERNED(unicode)) {
     case SSTATE_NOT_INTERNED:
         break;
 
     case SSTATE_INTERNED_MORTAL:
         /* revive dead object temporarily for DelItem */
         Py_REFCNT(unicode) = 3;
         if (PyDict_DelItem(interned, (PyObject *)unicode) != 0)
             Py_FatalError(
@@ skipping 23 matching lines @@
         free_list = unicode;
         numfree++;
     }
     else {
         PyObject_DEL(unicode->str);
         Py_XDECREF(unicode->defenc);
         Py_TYPE(unicode)->tp_free((PyObject *)unicode);
     }
 }
 
-static int
-_PyUnicode_Resize(PyUnicodeObject **unicode, Py_ssize_t length)
+static
+int _PyUnicode_Resize(PyUnicodeObject **unicode, Py_ssize_t length)
 {
     register PyUnicodeObject *v;
 
     /* Argument checks */
     if (unicode == NULL) {
         PyErr_BadInternalCall();
         return -1;
     }
     v = *unicode;
     if (v == NULL || !PyUnicode_Check(v) || Py_REFCNT(v) != 1 || length < 0) {
@@ skipping 14 matching lines @@
         Py_DECREF(*unicode);
         *unicode = w;
         return 0;
     }
 
     /* Note that we don't have to modify *unicode for unshared Unicode
        objects, since we can modify them in-place. */
     return unicode_resize(v, length);
 }
 
-int
-PyUnicode_Resize(PyObject **unicode, Py_ssize_t length)
+int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length)
 {
     return _PyUnicode_Resize((PyUnicodeObject **)unicode, length);
 }
 
-PyObject *
-PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size)
+PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,
+                                Py_ssize_t size)
 {
     PyUnicodeObject *unicode;
 
     /* If the Unicode data is known at construction time, we can apply
        some optimizations which share commonly used objects. */
     if (u != NULL) {
 
         /* Optimization for empty strings */
         if (size == 0 && unicode_empty != NULL) {
             Py_INCREF(unicode_empty);
@@ skipping 20 matching lines @@
     if (!unicode)
         return NULL;
 
     /* Copy the Unicode data into the new object */
     if (u != NULL)
         Py_UNICODE_COPY(unicode->str, u, size);
 
     return (PyObject *)unicode;
 }
 
-PyObject *
-PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size)
+PyObject *PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size)
 {
     PyUnicodeObject *unicode;
 
     if (size < 0) {
         PyErr_SetString(PyExc_SystemError,
                         "Negative size passed to PyUnicode_FromStringAndSize");
         return NULL;
     }
 
     /* If the Unicode data is known at construction time, we can apply
@@ skipping 26 matching lines @@
         return PyUnicode_DecodeUTF8(u, size, NULL);
     }
 
     unicode = _PyUnicode_New(size);
     if (!unicode)
         return NULL;
 
     return (PyObject *)unicode;
 }
 
-PyObject *
-PyUnicode_FromString(const char *u)
+PyObject *PyUnicode_FromString(const char *u)
 {
     size_t size = strlen(u);
     if (size > PY_SSIZE_T_MAX) {
         PyErr_SetString(PyExc_OverflowError, "input too long");
         return NULL;
     }
 
     return PyUnicode_FromStringAndSize(u, size);
 }
 
 #ifdef HAVE_WCHAR_H
 
 #if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4)
 # define CONVERT_WCHAR_TO_SURROGATES
 #endif
 
 #ifdef CONVERT_WCHAR_TO_SURROGATES
 
 /* Here sizeof(wchar_t) is 4 but Py_UNICODE_SIZE == 2, so we need
    to convert from UTF32 to UTF16. */
 
-PyObject *
-PyUnicode_FromWideChar(register const wchar_t *w, Py_ssize_t size)
+PyObject *PyUnicode_FromWideChar(register const wchar_t *w,
+                                 Py_ssize_t size)
 {
     PyUnicodeObject *unicode;
     register Py_ssize_t i;
     Py_ssize_t alloc;
     const wchar_t *orig_w;
 
     if (w == NULL) {
         if (size == 0)
             return PyUnicode_FromStringAndSize(NULL, 0);
         PyErr_BadInternalCall();
@@ skipping 29 matching lines @@
             }
             else
                 *u++ = *w++;
         }
     }
     return (PyObject *)unicode;
 }
 
 #else
 
-PyObject *
-PyUnicode_FromWideChar(register const wchar_t *w, Py_ssize_t size)
+PyObject *PyUnicode_FromWideChar(register const wchar_t *w,
+                                 Py_ssize_t size)
 {
     PyUnicodeObject *unicode;
 
     if (w == NULL) {
         if (size == 0)
             return PyUnicode_FromStringAndSize(NULL, 0);
         PyErr_BadInternalCall();
         return NULL;
     }
 
     if (size == -1) {
         size = wcslen(w);
     }
 
     unicode = _PyUnicode_New(size);
     if (!unicode)
         return NULL;
 
     /* Copy the wchar_t data into the new object */
-#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T
+#ifdef HAVE_USABLE_WCHAR_T
     memcpy(unicode->str, w, size * sizeof(wchar_t));
 #else
     {
         register Py_UNICODE *u;
         register Py_ssize_t i;
         u = PyUnicode_AS_UNICODE(unicode);
         for (i = size; i > 0; i--)
             *u++ = *w++;
     }
 #endif
 
     return (PyObject *)unicode;
 }
 
 #endif /* CONVERT_WCHAR_TO_SURROGATES */
 
 #undef CONVERT_WCHAR_TO_SURROGATES
 
 static void
-makefmt(char *fmt, int longflag, int longlongflag, int size_tflag,
-        int zeropad, int width, int precision, char c)
+makefmt(char *fmt, int longflag, int size_tflag, int zeropad, int width, int precision, char c)
 {
     *fmt++ = '%';
     if (width) {
         if (zeropad)
             *fmt++ = '0';
         fmt += sprintf(fmt, "%d", width);
     }
     if (precision)
         fmt += sprintf(fmt, ".%d", precision);
     if (longflag)
         *fmt++ = 'l';
-    else if (longlongflag) {
-        /* longlongflag should only ever be nonzero on machines with
-           HAVE_LONG_LONG defined */
-#ifdef HAVE_LONG_LONG
-        char *f = PY_FORMAT_LONG_LONG;
-        while (*f)
-            *fmt++ = *f++;
-#else
-        /* we shouldn't ever get here */
-        assert(0);
-        *fmt++ = 'l';
-#endif
-    }
     else if (size_tflag) {
         char *f = PY_FORMAT_SIZE_T;
         while (*f)
             *fmt++ = *f++;
     }
     *fmt++ = c;
     *fmt = '\0';
 }
 
-/* helper for PyUnicode_FromFormatV() */
-
-static const char*
-parse_format_flags(const char *f,
-                   int *p_width, int *p_precision,
-                   int *p_longflag, int *p_longlongflag, int *p_size_tflag)
-{
-    int width, precision, longflag, longlongflag, size_tflag;
-
-    /* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */
-    f++;
-    width = 0;
-    while (Py_ISDIGIT((unsigned)*f))
-        width = (width*10) + *f++ - '0';
-    precision = 0;
-    if (*f == '.') {
-        f++;
-        while (Py_ISDIGIT((unsigned)*f))
-            precision = (precision*10) + *f++ - '0';
-        if (*f == '%') {
-            /* "%.3%s" => f points to "3" */
-            f--;
-        }
-    }
-    if (*f == '\0') {
-        /* bogus format "%.1" => go backward, f points to "1" */
-        f--;
-    }
-    if (p_width != NULL)
-        *p_width = width;
-    if (p_precision != NULL)
-        *p_precision = precision;
-
-    /* Handle %ld, %lu, %lld and %llu. */
-    longflag = 0;
-    longlongflag = 0;
-    size_tflag = 0;
-
-    if (*f == 'l') {
-        if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') {
-            longflag = 1;
-            ++f;
-        }
-#ifdef HAVE_LONG_LONG
-        else if (f[1] == 'l' &&
-                 (f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) {
-            longlongflag = 1;
-            f += 2;
-        }
-#endif
-    }
-    /* handle the size_t flag. */
-    else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) {
-        size_tflag = 1;
-        ++f;
-    }
-    if (p_longflag != NULL)
-        *p_longflag = longflag;
-    if (p_longlongflag != NULL)
-        *p_longlongflag = longlongflag;
-    if (p_size_tflag != NULL)
-        *p_size_tflag = size_tflag;
-    return f;
-}
-
 #define appendstring(string) {for (copy = string;*copy;) *s++ = *copy++;}
-
-/* size of fixed-size buffer for formatting single arguments */
-#define ITEM_BUFFER_LEN 21
-/* maximum number of characters required for output of %ld.  21 characters
-   allows for 64-bit integers (in decimal) and an optional sign. */
-#define MAX_LONG_CHARS 21
-/* maximum number of characters required for output of %lld.
-   We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits,
-   plus 1 for the sign.  53/22 is an upper bound for log10(256). */
-#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22)
 
 PyObject *
 PyUnicode_FromFormatV(const char *format, va_list vargs)
 {
     va_list count;
     Py_ssize_t callcount = 0;
     PyObject **callresults = NULL;
     PyObject **callresult = NULL;
     Py_ssize_t n = 0;
     int width = 0;
     int precision = 0;
     int zeropad;
     const char* f;
     Py_UNICODE *s;
     PyObject *string;
     /* used by sprintf */
-    char buffer[ITEM_BUFFER_LEN+1];
+    char buffer[21];
     /* use abuffer instead of buffer, if we need more space
      * (which can happen if there's a format specifier with width). */
     char *abuffer = NULL;
     char *realbuffer;
     Py_ssize_t abuffersize = 0;
-    char fmt[61]; /* should be enough for %0width.precisionlld */
+    char fmt[60]; /* should be enough for %0width.precisionld */
     const char *copy;
 
-    Py_VA_COPY(count, vargs);
+#ifdef VA_LIST_IS_ARRAY
+    Py_MEMCPY(count, vargs, sizeof(va_list));
+#else
+#ifdef  __va_copy
+    __va_copy(count, vargs);
+#else
+    count = vargs;
+#endif
+#endif
     /* step 1: count the number of %S/%R/%A/%s format specifications
      * (we call PyObject_Str()/PyObject_Repr()/PyObject_ASCII()/
      * PyUnicode_DecodeUTF8() for these objects once during step 3 and put the
      * result in an array) */
     for (f = format; *f; f++) {
          if (*f == '%') {
-             /* skip width or width.precision (eg. "1.2" of "%1.2f") */
-             f = parse_format_flags(f, NULL, NULL, NULL, NULL, NULL);
-             if (*f == 's' || *f=='S' || *f=='R' || *f=='A' || *f=='V')
+             if (*(f+1)=='%')
+                 continue;
+             if (*(f+1)=='S' || *(f+1)=='R' || *(f+1)=='A')
                  ++callcount;
-         }
-         else if (128 <= (unsigned char)*f) {
-             PyErr_Format(PyExc_ValueError,
-                "PyUnicode_FromFormatV() expects an ASCII-encoded format "
-                "string, got a non-ASCII byte: 0x%02x",
-                (unsigned char)*f);
-             return NULL;
+             while (ISDIGIT((unsigned)*f))
+                 width = (width*10) + *f++ - '0';
+             while (*++f && *f != '%' && !ISALPHA((unsigned)*f))
+                 ;
+             if (*f == 's')
+                 ++callcount;
          }
     }
     /* step 2: allocate memory for the results of
      * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */
     if (callcount) {
         callresults = PyObject_Malloc(sizeof(PyObject *)*callcount);
         if (!callresults) {
             PyErr_NoMemory();
             return NULL;
         }
         callresult = callresults;
     }
     /* step 3: figure out how large a buffer we need */
     for (f = format; *f; f++) {
         if (*f == '%') {
-#ifdef HAVE_LONG_LONG
-            int longlongflag;
-#endif
-            const char* p;
-
-            p = f;
-            f = parse_format_flags(f, &width, NULL,
-                                   NULL, &longlongflag, NULL);
+            const char* p = f;
+            width = 0;
+            while (ISDIGIT((unsigned)*f))
+                width = (width*10) + *f++ - '0';
+            while (*++f && *f != '%' && !ISALPHA((unsigned)*f))
+                ;
+
+            /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since
+             * they don't affect the amount of space we reserve.
+             */
+            if ((*f == 'l' || *f == 'z') &&
+                (f[1] == 'd' || f[1] == 'u'))
+                ++f;
 
             switch (*f) {
             case 'c':
-            {
-#ifndef Py_UNICODE_WIDE
-                int ordinal = va_arg(count, int);
-                if (ordinal > 0xffff)
-                    n += 2;
-                else
-                    n++;
-#else
                 (void)va_arg(count, int);
-                n++;
-#endif
-                break;
-            }
+                /* fall through... */
             case '%':
                 n++;
                 break;
             case 'd': case 'u': case 'i': case 'x':
                 (void) va_arg(count, int);
-#ifdef HAVE_LONG_LONG
-                if (longlongflag) {
-                    if (width < MAX_LONG_LONG_CHARS)
-                        width = MAX_LONG_LONG_CHARS;
-                }
-                else
-#endif
-                    /* MAX_LONG_CHARS is enough to hold a 64-bit integer,
-                       including sign.  Decimal takes the most space.  This
-                       isn't enough for octal.  If a width is specified we
-                       need more (which we allocate later). */
-                    if (width < MAX_LONG_CHARS)
-                        width = MAX_LONG_CHARS;
+                /* 20 bytes is enough to hold a 64-bit
+                   integer.  Decimal takes the most space.
+                   This isn't enough for octal.
+                   If a width is specified we need more
+                   (which we allocate later). */
+                if (width < 20)
+                    width = 20;
                 n += width;
-                /* XXX should allow for large precision here too. */
                 if (abuffersize < width)
                     abuffersize = width;
                 break;
             case 's':
             {
                 /* UTF-8 */
                 const char *s = va_arg(count, const char*);
                 PyObject *str = PyUnicode_DecodeUTF8(s, strlen(s), "replace");
                 if (!str)
                     goto fail;
                 n += PyUnicode_GET_SIZE(str);
                 /* Remember the str and switch to the next slot */
                 *callresult++ = str;
                 break;
             }
             case 'U':
             {
                 PyObject *obj = va_arg(count, PyObject *);
                 assert(obj && PyUnicode_Check(obj));
                 n += PyUnicode_GET_SIZE(obj);
                 break;
             }
             case 'V':
             {
                 PyObject *obj = va_arg(count, PyObject *);
                 const char *str = va_arg(count, const char *);
-                PyObject *str_obj;
                 assert(obj || str);
                 assert(!obj || PyUnicode_Check(obj));
-                if (obj) {
+                if (obj)
                     n += PyUnicode_GET_SIZE(obj);
-                    *callresult++ = NULL;
-                }
-                else {
-                    str_obj = PyUnicode_DecodeUTF8(str, strlen(str), "replace");
-                    if (!str_obj)
-                        goto fail;
-                    n += PyUnicode_GET_SIZE(str_obj);
-                    *callresult++ = str_obj;
-                }
+                else
+                    n += strlen(str);
                 break;
             }
             case 'S':
             {
                 PyObject *obj = va_arg(count, PyObject *);
                 PyObject *str;
                 assert(obj);
                 str = PyObject_Str(obj);
                 if (!str)
                     goto fail;
@@ skipping 44 matching lines @@
                    string. (we cannot just skip the
                    code, since there's no way to know
                    what's in the argument list) */
                 n += strlen(p);
                 goto expand;
             }
         } else
             n++;
     }
   expand:
-    if (abuffersize > ITEM_BUFFER_LEN) {
-        /* add 1 for sprintf's trailing null byte */
-        abuffer = PyObject_Malloc(abuffersize + 1);
+    if (abuffersize > 20) {
+        abuffer = PyObject_Malloc(abuffersize);
         if (!abuffer) {
             PyErr_NoMemory();
             goto fail;
         }
         realbuffer = abuffer;
     }
     else
         realbuffer = buffer;
     /* step 4: fill the buffer */
     /* Since we've analyzed how much space we need for the worst case,
        we don't have to resize the string.
        There can be no errors beyond this point. */
     string = PyUnicode_FromUnicode(NULL, n);
     if (!string)
         goto fail;
 
     s = PyUnicode_AS_UNICODE(string);
     callresult = callresults;
 
     for (f = format; *f; f++) {
         if (*f == '%') {
-            const char* p;
-            int longflag;
-            int longlongflag;
-            int size_tflag;
-
-            p = f;
-            zeropad = (f[1] == '0');
-            f = parse_format_flags(f, &width, &precision,
-                                   &longflag, &longlongflag, &size_tflag);
+            const char* p = f++;
+            int longflag = 0;
+            int size_tflag = 0;
+            zeropad = (*f == '0');
+            /* parse the width.precision part */
+            width = 0;
+            while (ISDIGIT((unsigned)*f))
+                width = (width*10) + *f++ - '0';
+            precision = 0;
+            if (*f == '.') {
+                f++;
+                while (ISDIGIT((unsigned)*f))
+                    precision = (precision*10) + *f++ - '0';
+            }
+            /* handle the long flag, but only for %ld and %lu.
+               others can be added when necessary. */
+            if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) {
+                longflag = 1;
+                ++f;
+            }
+            /* handle the size_t flag. */
+            if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) {
+                size_tflag = 1;
+                ++f;
+            }
 
             switch (*f) {
             case 'c':
-            {
-                int ordinal = va_arg(vargs, int);
-#ifndef Py_UNICODE_WIDE
-                if (ordinal > 0xffff) {
-                    ordinal -= 0x10000;
-                    *s++ = 0xD800 | (ordinal >> 10);
-                    *s++ = 0xDC00 | (ordinal & 0x3FF);
-                } else
-#endif
-                *s++ = ordinal;
+                *s++ = va_arg(vargs, int);
                 break;
-            }
-            case 'i':
             case 'd':
-                makefmt(fmt, longflag, longlongflag, size_tflag, zeropad,
-                        width, precision, *f);
+                makefmt(fmt, longflag, size_tflag, zeropad, width, precision, 'd');
                 if (longflag)
                     sprintf(realbuffer, fmt, va_arg(vargs, long));
-#ifdef HAVE_LONG_LONG
-                else if (longlongflag)
-                    sprintf(realbuffer, fmt, va_arg(vargs, PY_LONG_LONG));
-#endif
                 else if (size_tflag)
                     sprintf(realbuffer, fmt, va_arg(vargs, Py_ssize_t));
                 else
                     sprintf(realbuffer, fmt, va_arg(vargs, int));
                 appendstring(realbuffer);
                 break;
             case 'u':
-                makefmt(fmt, longflag, longlongflag, size_tflag, zeropad,
-                        width, precision, 'u');
+                makefmt(fmt, longflag, size_tflag, zeropad, width, precision, 'u');
                 if (longflag)
                     sprintf(realbuffer, fmt, va_arg(vargs, unsigned long));
-#ifdef HAVE_LONG_LONG
-                else if (longlongflag)
-                    sprintf(realbuffer, fmt, va_arg(vargs,
-                                                    unsigned PY_LONG_LONG));
-#endif
                 else if (size_tflag)
                     sprintf(realbuffer, fmt, va_arg(vargs, size_t));
                 else
                     sprintf(realbuffer, fmt, va_arg(vargs, unsigned int));
                 appendstring(realbuffer);
                 break;
+            case 'i':
+                makefmt(fmt, 0, 0, zeropad, width, precision, 'i');
+                sprintf(realbuffer, fmt, va_arg(vargs, int));
+                appendstring(realbuffer);
+                break;
             case 'x':
-                makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'x');
+                makefmt(fmt, 0, 0, zeropad, width, precision, 'x');
                 sprintf(realbuffer, fmt, va_arg(vargs, int));
                 appendstring(realbuffer);
                 break;
             case 's':
             {
                 /* unused, since we already have the result */
                 (void) va_arg(vargs, char *);
                 Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult),
                                 PyUnicode_GET_SIZE(*callresult));
                 s += PyUnicode_GET_SIZE(*callresult);
                 /* We're done with the unicode()/repr() => forget it */
                 Py_DECREF(*callresult);
                 /* switch to next unicode()/repr() result */
                 ++callresult;
                 break;
             }
             case 'U':
             {
                 PyObject *obj = va_arg(vargs, PyObject *);
                 Py_ssize_t size = PyUnicode_GET_SIZE(obj);
                 Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size);
                 s += size;
                 break;
             }
             case 'V':
             {
                 PyObject *obj = va_arg(vargs, PyObject *);
-                va_arg(vargs, const char *);
+                const char *str = va_arg(vargs, const char *);
                 if (obj) {
                     Py_ssize_t size = PyUnicode_GET_SIZE(obj);
                     Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size);
                     s += size;
                 } else {
-                    Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult),
-                                    PyUnicode_GET_SIZE(*callresult));
-                    s += PyUnicode_GET_SIZE(*callresult);
-                    Py_DECREF(*callresult);
+                    appendstring(str);
                 }
-                ++callresult;
                 break;
             }
             case 'S':
             case 'R':
-            case 'A':
             {
                 Py_UNICODE *ucopy;
                 Py_ssize_t usize;
                 Py_ssize_t upos;
                 /* unused, since we already have the result */
                 (void) va_arg(vargs, PyObject *);
                 ucopy = PyUnicode_AS_UNICODE(*callresult);
                 usize = PyUnicode_GET_SIZE(*callresult);
                 for (upos = 0; upos<usize;)
                     *s++ = ucopy[upos++];
@@ skipping 15 matching lines @@
                 }
                 appendstring(buffer);
                 break;
             case '%':
                 *s++ = '%';
                 break;
             default:
                 appendstring(p);
                 goto end;
             }
-        }
-        else
+        } else
             *s++ = *f;
     }
 
   end:
     if (callresults)
         PyObject_Free(callresults);
     if (abuffer)
         PyObject_Free(abuffer);
     PyUnicode_Resize(&string, s - PyUnicode_AS_UNICODE(string));
     return string;
   fail:
     if (callresults) {
         PyObject **callresult2 = callresults;
         while (callresult2 < callresult) {
-            Py_XDECREF(*callresult2);
+            Py_DECREF(*callresult2);
             ++callresult2;
         }
         PyObject_Free(callresults);
     }
     if (abuffer)
         PyObject_Free(abuffer);
     return NULL;
 }
 
 #undef appendstring
 
 PyObject *
 PyUnicode_FromFormat(const char *format, ...)
 {
     PyObject* ret;
     va_list vargs;
 
 #ifdef HAVE_STDARG_PROTOTYPES
     va_start(vargs, format);
 #else
     va_start(vargs);
 #endif
     ret = PyUnicode_FromFormatV(format, vargs);
     va_end(vargs);
     return ret;
 }
 
-/* Helper function for PyUnicode_AsWideChar() and PyUnicode_AsWideCharString():
-   convert a Unicode object to a wide character string.
-
-   - If w is NULL: return the number of wide characters (including the nul
-     character) required to convert the unicode object. Ignore size argument.
-
-   - Otherwise: return the number of wide characters (excluding the nul
-     character) written into w. Write at most size wide characters (including
-     the nul character). */
-static Py_ssize_t
-unicode_aswidechar(PyUnicodeObject *unicode,
-                   wchar_t *w,
-                   Py_ssize_t size)
-{
-#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T
-    Py_ssize_t res;
-    if (w != NULL) {
-        res = PyUnicode_GET_SIZE(unicode);
-        if (size > res)
-            size = res + 1;
-        else
-            res = size;
-        memcpy(w, unicode->str, size * sizeof(wchar_t));
-        return res;
-    }
-    else
-        return PyUnicode_GET_SIZE(unicode) + 1;
-#elif Py_UNICODE_SIZE == 2 && SIZEOF_WCHAR_T == 4
-    register const Py_UNICODE *u;
-    const Py_UNICODE *uend;
-    const wchar_t *worig, *wend;
-    Py_ssize_t nchar;
-
-    u = PyUnicode_AS_UNICODE(unicode);
-    uend = u + PyUnicode_GET_SIZE(unicode);
-    if (w != NULL) {
-        worig = w;
-        wend = w + size;
-        while (u != uend && w != wend) {
-            if (0xD800 <= u[0] && u[0] <= 0xDBFF
-                && 0xDC00 <= u[1] && u[1] <= 0xDFFF)
-            {
-                *w = (((u[0] & 0x3FF) << 10) | (u[1] & 0x3FF)) + 0x10000;
-                u += 2;
-            }
-            else {
-                *w = *u;
-                u++;
-            }
-            w++;
-        }
-        if (w != wend)
-            *w = L'\0';
-        return w - worig;
-    }
-    else {
-        nchar = 1; /* nul character at the end */
-        while (u != uend) {
-            if (0xD800 <= u[0] && u[0] <= 0xDBFF
-                && 0xDC00 <= u[1] && u[1] <= 0xDFFF)
-                u += 2;
-            else
-                u++;
-            nchar++;
-        }
-    }
-    return nchar;
-#elif Py_UNICODE_SIZE == 4 && SIZEOF_WCHAR_T == 2
-    register Py_UNICODE *u, *uend, ordinal;
-    register Py_ssize_t i;
-    wchar_t *worig, *wend;
-    Py_ssize_t nchar;
-
-    u = PyUnicode_AS_UNICODE(unicode);
-    uend = u + PyUnicode_GET_SIZE(u);
-    if (w != NULL) {
-        worig = w;
-        wend = w + size;
-        while (u != uend && w != wend) {
-            ordinal = *u;
-            if (ordinal > 0xffff) {
-                ordinal -= 0x10000;
-                *w++ = 0xD800 | (ordinal >> 10);
-                *w++ = 0xDC00 | (ordinal & 0x3FF);
-            }
-            else
-                *w++ = ordinal;
-            u++;
-        }
-        if (w != wend)
-            *w = 0;
-        return w - worig;
-    }
-    else {
-        nchar = 1; /* nul character */
-        while (u != uend) {
-            if (*u > 0xffff)
-                nchar += 2;
-            else
-                nchar++;
-            u++;
-        }
-        return nchar;
-    }
-#else
-#  error "unsupported wchar_t and Py_UNICODE sizes, see issue #8670"
-#endif
-}
-
-Py_ssize_t
-PyUnicode_AsWideChar(PyObject *unicode,
-                     wchar_t *w,
-                     Py_ssize_t size)
+Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode,
+                                wchar_t *w,
+                                Py_ssize_t size)
 {
     if (unicode == NULL) {
         PyErr_BadInternalCall();
         return -1;
     }
-    return unicode_aswidechar((PyUnicodeObject*)unicode, w, size);
-}
-
-wchar_t*
-PyUnicode_AsWideCharString(PyObject *unicode,
-                           Py_ssize_t *size)
-{
-    wchar_t* buffer;
-    Py_ssize_t buflen;
-
-    if (unicode == NULL) {
-        PyErr_BadInternalCall();
-        return NULL;
-    }
-
-    buflen = unicode_aswidechar((PyUnicodeObject *)unicode, NULL, 0);
-    if (PY_SSIZE_T_MAX / sizeof(wchar_t) < buflen) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
-    buffer = PyMem_MALLOC(buflen * sizeof(wchar_t));
-    if (buffer == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-    buflen = unicode_aswidechar((PyUnicodeObject *)unicode, buffer, buflen);
-    if (size != NULL)
-        *size = buflen;
-    return buffer;
-}
-
+
+    /* If possible, try to copy the 0-termination as well */
+    if (size > PyUnicode_GET_SIZE(unicode))
+        size = PyUnicode_GET_SIZE(unicode) + 1;
+
+#ifdef HAVE_USABLE_WCHAR_T
+    memcpy(w, unicode->str, size * sizeof(wchar_t));
+#else
+    {
+        register Py_UNICODE *u;
+        register Py_ssize_t i;
+        u = PyUnicode_AS_UNICODE(unicode);
+        for (i = size; i > 0; i--)
+            *w++ = *u++;
+    }
 #endif
 
-PyObject *
-PyUnicode_FromOrdinal(int ordinal)
+    if (size > PyUnicode_GET_SIZE(unicode))
+        return PyUnicode_GET_SIZE(unicode);
+    else
+        return size;
+}
+
+#endif
+
+PyObject *PyUnicode_FromOrdinal(int ordinal)
 {
     Py_UNICODE s[2];
 
     if (ordinal < 0 || ordinal > 0x10ffff) {
         PyErr_SetString(PyExc_ValueError,
                         "chr() arg not in range(0x110000)");
         return NULL;
     }
 
 #ifndef Py_UNICODE_WIDE
     if (ordinal > 0xffff) {
         ordinal -= 0x10000;
         s[0] = 0xD800 | (ordinal >> 10);
         s[1] = 0xDC00 | (ordinal & 0x3FF);
         return PyUnicode_FromUnicode(s, 2);
     }
 #endif
 
     s[0] = (Py_UNICODE)ordinal;
     return PyUnicode_FromUnicode(s, 1);
 }
 
-PyObject *
-PyUnicode_FromObject(register PyObject *obj)
+PyObject *PyUnicode_FromObject(register PyObject *obj)
 {
     /* XXX Perhaps we should make this API an alias of
        PyObject_Str() instead ?! */
     if (PyUnicode_CheckExact(obj)) {
         Py_INCREF(obj);
         return obj;
     }
     if (PyUnicode_Check(obj)) {
         /* For a Unicode subtype that's not a Unicode object,
            return a true Unicode object with the same data. */
         return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj),
                                      PyUnicode_GET_SIZE(obj));
     }
     PyErr_Format(PyExc_TypeError,
                  "Can't convert '%.100s' object to str implicitly",
                  Py_TYPE(obj)->tp_name);
     return NULL;
 }
 
-PyObject *
-PyUnicode_FromEncodedObject(register PyObject *obj,
-                            const char *encoding,
-                            const char *errors)
+PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,
+                                      const char *encoding,
+                                      const char *errors)
 {
     Py_buffer buffer;
     PyObject *v;
 
     if (obj == NULL) {
         PyErr_BadInternalCall();
         return NULL;
     }
 
     /* Decoding bytes objects is the most common case and should be fast */
@@ skipping 29 matching lines @@
         Py_INCREF(unicode_empty);
         v = (PyObject *) unicode_empty;
     }
     else
         v = PyUnicode_Decode((char*) buffer.buf, buffer.len, encoding, errors);
 
     PyBuffer_Release(&buffer);
     return v;
 }
 
-/* Convert encoding to lower case and replace '_' with '-' in order to
-   catch e.g. UTF_8. Return 0 on error (encoding is longer than lower_len-1),
-   1 on success. */
-static int
-normalize_encoding(const char *encoding,
-                   char *lower,
-                   size_t lower_len)
-{
+PyObject *PyUnicode_Decode(const char *s,
+                           Py_ssize_t size,
+                           const char *encoding,
+                           const char *errors)
+{
+    PyObject *buffer = NULL, *unicode;
+    Py_buffer info;
+    char lower[20];  /* Enough for any encoding name we recognize */
+    char *l;
     const char *e;
-    char *l;
-    char *l_end;
-
+
+    if (encoding == NULL)
+        encoding = PyUnicode_GetDefaultEncoding();
+
+    /* Convert encoding to lower case and replace '_' with '-' in order to
+       catch e.g. UTF_8 */
     e = encoding;
     l = lower;
-    l_end = &lower[lower_len - 1];
-    while (*e) {
-        if (l == l_end)
-            return 0;
-        if (Py_ISUPPER(*e)) {
-            *l++ = Py_TOLOWER(*e++);
+    while (*e && l < &lower[(sizeof lower) - 2]) {
+        if (ISUPPER(*e)) {
+            *l++ = TOLOWER(*e++);
         }
         else if (*e == '_') {
             *l++ = '-';
             e++;
         }
         else {
             *l++ = *e++;
         }
     }
     *l = '\0';
-    return 1;
-}
-
-PyObject *
-PyUnicode_Decode(const char *s,
-                 Py_ssize_t size,
-                 const char *encoding,
-                 const char *errors)
-{
-    PyObject *buffer = NULL, *unicode;
-    Py_buffer info;
-    char lower[11];  /* Enough for any encoding shortcut */
-
-    if (encoding == NULL)
+
+    /* Shortcuts for common default encodings */
+    if (strcmp(lower, "utf-8") == 0)
         return PyUnicode_DecodeUTF8(s, size, errors);
-
-    /* Shortcuts for common default encodings */
-    if (normalize_encoding(encoding, lower, sizeof(lower))) {
-        if ((strcmp(lower, "utf-8") == 0) ||
-            (strcmp(lower, "utf8") == 0))
-            return PyUnicode_DecodeUTF8(s, size, errors);
-        else if ((strcmp(lower, "latin-1") == 0) ||
-                 (strcmp(lower, "latin1") == 0) ||
-                 (strcmp(lower, "iso-8859-1") == 0))
-            return PyUnicode_DecodeLatin1(s, size, errors);
+    else if ((strcmp(lower, "latin-1") == 0) ||
+             (strcmp(lower, "iso-8859-1") == 0))
+        return PyUnicode_DecodeLatin1(s, size, errors);
 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
-        else if (strcmp(lower, "mbcs") == 0)
-            return PyUnicode_DecodeMBCS(s, size, errors);
+    else if (strcmp(lower, "mbcs") == 0)
+        return PyUnicode_DecodeMBCS(s, size, errors);
 #endif
-        else if (strcmp(lower, "ascii") == 0)
-            return PyUnicode_DecodeASCII(s, size, errors);
-        else if (strcmp(lower, "utf-16") == 0)
-            return PyUnicode_DecodeUTF16(s, size, errors, 0);
-        else if (strcmp(lower, "utf-32") == 0)
-            return PyUnicode_DecodeUTF32(s, size, errors, 0);
-    }
+    else if (strcmp(lower, "ascii") == 0)
+        return PyUnicode_DecodeASCII(s, size, errors);
+    else if (strcmp(lower, "utf-16") == 0)
+        return PyUnicode_DecodeUTF16(s, size, errors, 0);
+    else if (strcmp(lower, "utf-32") == 0)
+        return PyUnicode_DecodeUTF32(s, size, errors, 0);
 
     /* Decode via the codec registry */
     buffer = NULL;
     if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0)
         goto onError;
     buffer = PyMemoryView_FromBuffer(&info);
     if (buffer == NULL)
         goto onError;
     unicode = PyCodec_Decode(buffer, encoding, errors);
     if (unicode == NULL)
         goto onError;
     if (!PyUnicode_Check(unicode)) {
         PyErr_Format(PyExc_TypeError,
                      "decoder did not return a str object (type=%.400s)",
                      Py_TYPE(unicode)->tp_name);
         Py_DECREF(unicode);
         goto onError;
     }
     Py_DECREF(buffer);
     return unicode;
 
   onError:
     Py_XDECREF(buffer);
     return NULL;
 }
 
-PyObject *
-PyUnicode_AsDecodedObject(PyObject *unicode,
-                          const char *encoding,
-                          const char *errors)
+PyObject *PyUnicode_AsDecodedObject(PyObject *unicode,
+                                    const char *encoding,
+                                    const char *errors)
 {
     PyObject *v;
 
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         goto onError;
     }
 
     if (encoding == NULL)
         encoding = PyUnicode_GetDefaultEncoding();
 
     /* Decode via the codec registry */
     v = PyCodec_Decode(unicode, encoding, errors);
     if (v == NULL)
         goto onError;
     return v;
 
   onError:
     return NULL;
 }
 
-PyObject *
-PyUnicode_AsDecodedUnicode(PyObject *unicode,
-                           const char *encoding,
-                           const char *errors)
+PyObject *PyUnicode_AsDecodedUnicode(PyObject *unicode,
+                                     const char *encoding,
+                                     const char *errors)
 {
     PyObject *v;
 
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         goto onError;
     }
 
     if (encoding == NULL)
         encoding = PyUnicode_GetDefaultEncoding();
 
     /* Decode via the codec registry */
     v = PyCodec_Decode(unicode, encoding, errors);
     if (v == NULL)
         goto onError;
     if (!PyUnicode_Check(v)) {
         PyErr_Format(PyExc_TypeError,
                      "decoder did not return a str object (type=%.400s)",
                      Py_TYPE(v)->tp_name);
         Py_DECREF(v);
         goto onError;
     }
     return v;
 
   onError:
     return NULL;
 }
 
-PyObject *
-PyUnicode_Encode(const Py_UNICODE *s,
-                 Py_ssize_t size,
-                 const char *encoding,
-                 const char *errors)
+PyObject *PyUnicode_Encode(const Py_UNICODE *s,
+                           Py_ssize_t size,
+                           const char *encoding,
+                           const char *errors)
 {
     PyObject *v, *unicode;
 
     unicode = PyUnicode_FromUnicode(s, size);
     if (unicode == NULL)
         return NULL;
     v = PyUnicode_AsEncodedString(unicode, encoding, errors);
     Py_DECREF(unicode);
     return v;
 }
 
-PyObject *
-PyUnicode_AsEncodedObject(PyObject *unicode,
-                          const char *encoding,
-                          const char *errors)
+PyObject *PyUnicode_AsEncodedObject(PyObject *unicode,
+                                    const char *encoding,
+                                    const char *errors)
 {
     PyObject *v;
 
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         goto onError;
     }
 
     if (encoding == NULL)
         encoding = PyUnicode_GetDefaultEncoding();
 
     /* Encode via the codec registry */
     v = PyCodec_Encode(unicode, encoding, errors);
     if (v == NULL)
         goto onError;
     return v;
 
   onError:
     return NULL;
 }
 
-PyObject *
-PyUnicode_EncodeFSDefault(PyObject *unicode)
-{
-#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
-    return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode),
-                                PyUnicode_GET_SIZE(unicode),
-                                NULL);
-#elif defined(__APPLE__)
-    return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
-                                PyUnicode_GET_SIZE(unicode),
-                                "surrogateescape");
-#else
-    if (Py_FileSystemDefaultEncoding) {
-        return PyUnicode_AsEncodedString(unicode,
-                                         Py_FileSystemDefaultEncoding,
-                                         "surrogateescape");
-    }
-    else {
-        /* locale encoding with surrogateescape */
-        wchar_t *wchar;
-        char *bytes;
-        PyObject *bytes_obj;
-        size_t error_pos;
-
-        wchar = PyUnicode_AsWideCharString(unicode, NULL);
-        if (wchar == NULL)
-            return NULL;
-        bytes = _Py_wchar2char(wchar, &error_pos);
-        if (bytes == NULL) {
-            if (error_pos != (size_t)-1) {
-                char *errmsg = strerror(errno);
-                PyObject *exc = NULL;
-                if (errmsg == NULL)
-                    errmsg = "Py_wchar2char() failed";
-                raise_encode_exception(&exc,
-                    "filesystemencoding",
-                    PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode),
-                    error_pos, error_pos+1,
-                    errmsg);
-                Py_XDECREF(exc);
-            }
-            else
-                PyErr_NoMemory();
-            PyMem_Free(wchar);
-            return NULL;
-        }
-        PyMem_Free(wchar);
-
-        bytes_obj = PyBytes_FromString(bytes);
-        PyMem_Free(bytes);
-        return bytes_obj;
-    }
-#endif
-}
-
-PyObject *
-PyUnicode_AsEncodedString(PyObject *unicode,
-                          const char *encoding,
-                          const char *errors)
+PyObject *PyUnicode_AsEncodedString(PyObject *unicode,
+                                    const char *encoding,
+                                    const char *errors)
 {
     PyObject *v;
-    char lower[11];  /* Enough for any encoding shortcut */
 
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
 
-    if (encoding == NULL) {
-        if (errors == NULL || strcmp(errors, "strict") == 0)
+    if (encoding == NULL)
+        encoding = PyUnicode_GetDefaultEncoding();
+
+    /* Shortcuts for common default encodings */
+    if (errors == NULL) {
+        if (strcmp(encoding, "utf-8") == 0)
             return PyUnicode_AsUTF8String(unicode);
-        else
-            return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
-                                        PyUnicode_GET_SIZE(unicode),
-                                        errors);
-    }
-
-    /* Shortcuts for common default encodings */
-    if (normalize_encoding(encoding, lower, sizeof(lower))) {
-        if ((strcmp(lower, "utf-8") == 0) ||
-            (strcmp(lower, "utf8") == 0))
-        {
-            if (errors == NULL || strcmp(errors, "strict") == 0)
-                return PyUnicode_AsUTF8String(unicode);
-            else
-                return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
-                                            PyUnicode_GET_SIZE(unicode),
-                                            errors);
-        }
-        else if ((strcmp(lower, "latin-1") == 0) ||
-                 (strcmp(lower, "latin1") == 0) ||
-                 (strcmp(lower, "iso-8859-1") == 0))
-            return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode),
-                                          PyUnicode_GET_SIZE(unicode),
-                                          errors);
+        else if (strcmp(encoding, "latin-1") == 0)
+            return PyUnicode_AsLatin1String(unicode);
 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
-        else if (strcmp(lower, "mbcs") == 0)
-            return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode),
-                                        PyUnicode_GET_SIZE(unicode),
-                                        errors);
+        else if (strcmp(encoding, "mbcs") == 0)
+            return PyUnicode_AsMBCSString(unicode);
 #endif
-        else if (strcmp(lower, "ascii") == 0)
-            return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode),
-                                         PyUnicode_GET_SIZE(unicode),
-                                         errors);
+        else if (strcmp(encoding, "ascii") == 0)
+            return PyUnicode_AsASCIIString(unicode);
+        /* During bootstrap, we may need to find the encodings
+           package, to load the file system encoding, and require the
+           file system encoding in order to load the encodings
+           package.
+
+           Break out of this dependency by assuming that the path to
+           the encodings module is ASCII-only.  XXX could try wcstombs
+           instead, if the file system encoding is the locale's
+           encoding. */
+        else if (Py_FileSystemDefaultEncoding &&
+                 strcmp(encoding, Py_FileSystemDefaultEncoding) == 0 &&
+                 !PyThreadState_GET()->interp->codecs_initialized)
+            return PyUnicode_AsASCIIString(unicode);
     }
 
     /* Encode via the codec registry */
     v = PyCodec_Encode(unicode, encoding, errors);
     if (v == NULL)
         return NULL;
 
     /* The normal path */
     if (PyBytes_Check(v))
         return v;
 
     /* If the codec returns a buffer, raise a warning and convert to bytes */
     if (PyByteArray_Check(v)) {
-        int error;
+        char msg[100];
         PyObject *b;
-
-        error = PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
-            "encoder %s returned bytearray instead of bytes",
-            encoding);
-        if (error) {
+        PyOS_snprintf(msg, sizeof(msg),
+                      "encoder %s returned buffer instead of bytes",
+                      encoding);
+        if (PyErr_WarnEx(PyExc_RuntimeWarning, msg, 1) < 0) {
             Py_DECREF(v);
             return NULL;
         }
 
         b = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
         Py_DECREF(v);
         return b;
     }
 
     PyErr_Format(PyExc_TypeError,
                  "encoder did not return a bytes object (type=%.400s)",
                  Py_TYPE(v)->tp_name);
     Py_DECREF(v);
     return NULL;
 }
 
-PyObject *
-PyUnicode_AsEncodedUnicode(PyObject *unicode,
-                           const char *encoding,
-                           const char *errors)
+PyObject *PyUnicode_AsEncodedUnicode(PyObject *unicode,
+                                     const char *encoding,
+                                     const char *errors)
 {
     PyObject *v;
 
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         goto onError;
     }
 
     if (encoding == NULL)
         encoding = PyUnicode_GetDefaultEncoding();
 
     /* Encode via the codec registry */
     v = PyCodec_Encode(unicode, encoding, errors);
     if (v == NULL)
         goto onError;
     if (!PyUnicode_Check(v)) {
         PyErr_Format(PyExc_TypeError,
                      "encoder did not return an str object (type=%.400s)",
                      Py_TYPE(v)->tp_name);
         Py_DECREF(v);
         goto onError;
     }
     return v;
 
   onError:
     return NULL;
 }
 
-PyObject *
-_PyUnicode_AsDefaultEncodedString(PyObject *unicode)
+PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,
+                                            const char *errors)
 {
     PyObject *v = ((PyUnicodeObject *)unicode)->defenc;
     if (v)
         return v;
+    if (errors != NULL)
+        Py_FatalError("non-NULL encoding in _PyUnicode_AsDefaultEncodedString");
     v = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
                              PyUnicode_GET_SIZE(unicode),
                              NULL);
     if (!v)
         return NULL;
     ((PyUnicodeObject *)unicode)->defenc = v;
     return v;
 }
 
 PyObject*
 PyUnicode_DecodeFSDefault(const char *s) {
     Py_ssize_t size = (Py_ssize_t)strlen(s);
     return PyUnicode_DecodeFSDefaultAndSize(s, size);
 }
 
 PyObject*
 PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
 {
-#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
-    return PyUnicode_DecodeMBCS(s, size, NULL);
-#elif defined(__APPLE__)
-    return PyUnicode_DecodeUTF8(s, size, "surrogateescape");
-#else
     /* During the early bootstrapping process, Py_FileSystemDefaultEncoding
        can be undefined. If it is case, decode using UTF-8. The following assumes
        that Py_FileSystemDefaultEncoding is set to a built-in encoding during the
        bootstrapping process where the codecs aren't ready yet.
     */
     if (Py_FileSystemDefaultEncoding) {
+#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
+        if (strcmp(Py_FileSystemDefaultEncoding, "mbcs") == 0) {
+            return PyUnicode_DecodeMBCS(s, size, "replace");
+        }
+#elif defined(__APPLE__)
+        if (strcmp(Py_FileSystemDefaultEncoding, "utf-8") == 0) {
+            return PyUnicode_DecodeUTF8(s, size, "replace");
+        }
+#endif
         return PyUnicode_Decode(s, size,
                                 Py_FileSystemDefaultEncoding,
-                                "surrogateescape");
+                                "replace");
     }
     else {
-        /* locale encoding with surrogateescape */
-        wchar_t *wchar;
-        PyObject *unicode;
-        size_t len;
-
-        if (s[size] != '\0' || size != strlen(s)) {
-            PyErr_SetString(PyExc_TypeError, "embedded NUL character");
-            return NULL;
-        }
-
-        wchar = _Py_char2wchar(s, &len);
-        if (wchar == NULL)
-            return PyErr_NoMemory();
-
-        unicode = PyUnicode_FromWideChar(wchar, len);
-        PyMem_Free(wchar);
-        return unicode;
-    }
-#endif
-}
-
+        return PyUnicode_DecodeUTF8(s, size, "replace");
+    }
+}
+
+/* Convert the argument to a bytes object, according to the file
+   system encoding */
 
 int
 PyUnicode_FSConverter(PyObject* arg, void* addr)
 {
     PyObject *output = NULL;
     Py_ssize_t size;
     void *data;
     if (arg == NULL) {
         Py_DECREF(*(PyObject**)addr);
         return 1;
     }
-    if (PyBytes_Check(arg)) {
+    if (PyBytes_Check(arg) || PyByteArray_Check(arg)) {
         output = arg;
         Py_INCREF(output);
     }
     else {
         arg = PyUnicode_FromObject(arg);
         if (!arg)
             return 0;
-        output = PyUnicode_EncodeFSDefault(arg);
+        output = PyUnicode_AsEncodedObject(arg, 
+                                           Py_FileSystemDefaultEncoding,
+                                           "surrogateescape");
         Py_DECREF(arg);
         if (!output)
             return 0;
         if (!PyBytes_Check(output)) {
             Py_DECREF(output);
             PyErr_SetString(PyExc_TypeError, "encoder failed to return bytes");
             return 0;
         }
     }
-    size = PyBytes_GET_SIZE(output);
-    data = PyBytes_AS_STRING(output);
+    if (PyBytes_Check(output)) {
+         size = PyBytes_GET_SIZE(output);
+         data = PyBytes_AS_STRING(output);
+    } 
+    else {
+         size = PyByteArray_GET_SIZE(output);
+         data = PyByteArray_AS_STRING(output);
+    }
     if (size != strlen(data)) {
         PyErr_SetString(PyExc_TypeError, "embedded NUL character");
         Py_DECREF(output);
         return 0;
     }
     *(PyObject**)addr = output;
     return Py_CLEANUP_SUPPORTED;
 }
 
 
-int
-PyUnicode_FSDecoder(PyObject* arg, void* addr)
-{
-    PyObject *output = NULL;
-    Py_ssize_t size;
-    void *data;
-    if (arg == NULL) {
-        Py_DECREF(*(PyObject**)addr);
-        return 1;
-    }
-    if (PyUnicode_Check(arg)) {
-        output = arg;
-        Py_INCREF(output);
-    }
-    else {
-        arg = PyBytes_FromObject(arg);
-        if (!arg)
-            return 0;
-        output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(arg),
-                                                  PyBytes_GET_SIZE(arg));
-        Py_DECREF(arg);
-        if (!output)
-            return 0;
-        if (!PyUnicode_Check(output)) {
-            Py_DECREF(output);
-            PyErr_SetString(PyExc_TypeError, "decoder failed to return unicode");
-            return 0;
-        }
-    }
-    size = PyUnicode_GET_SIZE(output);
-    data = PyUnicode_AS_UNICODE(output);
-    if (size != Py_UNICODE_strlen(data)) {
-        PyErr_SetString(PyExc_TypeError, "embedded NUL character");
-        Py_DECREF(output);
-        return 0;
-    }
-    *(PyObject**)addr = output;
-    return Py_CLEANUP_SUPPORTED;
-}
-
-
 char*
 _PyUnicode_AsStringAndSize(PyObject *unicode, Py_ssize_t *psize)
 {
     PyObject *bytes;
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
-    bytes = _PyUnicode_AsDefaultEncodedString(unicode);
+    bytes = _PyUnicode_AsDefaultEncodedString(unicode, NULL);
     if (bytes == NULL)
         return NULL;
     if (psize != NULL)
         *psize = PyBytes_GET_SIZE(bytes);
     return PyBytes_AS_STRING(bytes);
 }
 
 char*
 _PyUnicode_AsString(PyObject *unicode)
 {
     return _PyUnicode_AsStringAndSize(unicode, NULL);
 }
 
-Py_UNICODE *
-PyUnicode_AsUnicode(PyObject *unicode)
+Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         goto onError;
     }
     return PyUnicode_AS_UNICODE(unicode);
 
   onError:
     return NULL;
 }
 
-Py_ssize_t
-PyUnicode_GetSize(PyObject *unicode)
+Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         goto onError;
     }
     return PyUnicode_GET_SIZE(unicode);
 
   onError:
     return -1;
 }
 
-const char *
-PyUnicode_GetDefaultEncoding(void)
-{
-    return "utf-8";
-}
-
-/* create or adjust a UnicodeDecodeError */
-static void
-make_decode_exception(PyObject **exceptionObject,
-                      const char *encoding,
-                      const char *input, Py_ssize_t length,
-                      Py_ssize_t startpos, Py_ssize_t endpos,
-                      const char *reason)
-{
-    if (*exceptionObject == NULL) {
-        *exceptionObject = PyUnicodeDecodeError_Create(
-            encoding, input, length, startpos, endpos, reason);
-    }
-    else {
-        if (PyUnicodeDecodeError_SetStart(*exceptionObject, startpos))
-            goto onError;
-        if (PyUnicodeDecodeError_SetEnd(*exceptionObject, endpos))
-            goto onError;
-        if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
-            goto onError;
-    }
-    return;
-
-onError:
-    Py_DECREF(*exceptionObject);
-    *exceptionObject = NULL;
+const char *PyUnicode_GetDefaultEncoding(void)
+{
+    return unicode_default_encoding;
+}
+
+int PyUnicode_SetDefaultEncoding(const char *encoding)
+{
+    if (strcmp(encoding, unicode_default_encoding) != 0) {
+        PyErr_Format(PyExc_ValueError,
+                     "Can only set default encoding to %s",
+                     unicode_default_encoding);
+        return -1;
+    }
+    return 0;
 }
 
 /* error handling callback helper:
    build arguments, call the callback and check the arguments,
    if no exception occurred, copy the replacement to the output
    and adjust various state variables.
    return 0 on success, -1 on error
 */
 
-static int
-unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
-                                 const char *encoding, const char *reason,
-                                 const char **input, const char **inend, Py_ssize_t *startinpos,
-                                 Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
-                                 PyUnicodeObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr)
+static
+int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
+                                     const char *encoding, const char *reason,
+                                     const char **input, const char **inend, Py_ssize_t *startinpos,
+                                     Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
+                                     PyUnicodeObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr)
 {
     static char *argparse = "O!n;decoding error handler must return (str, int) tuple";
 
     PyObject *restuple = NULL;
     PyObject *repunicode = NULL;
     Py_ssize_t outsize = PyUnicode_GET_SIZE(*output);
     Py_ssize_t insize;
     Py_ssize_t requiredsize;
     Py_ssize_t newpos;
     Py_UNICODE *repptr;
     PyObject *inputobj = NULL;
     Py_ssize_t repsize;
     int res = -1;
 
     if (*errorHandler == NULL) {
         *errorHandler = PyCodec_LookupError(errors);
         if (*errorHandler == NULL)
             goto onError;
     }
 
-    make_decode_exception(exceptionObject,
-        encoding,
-        *input, *inend - *input,
-        *startinpos, *endinpos,
-        reason);
-    if (*exceptionObject == NULL)
-        goto onError;
+    if (*exceptionObject == NULL) {
+        *exceptionObject = PyUnicodeDecodeError_Create(
+            encoding, *input, *inend-*input, *startinpos, *endinpos, reason);
+        if (*exceptionObject == NULL)
+            goto onError;
+    }
+    else {
+        if (PyUnicodeDecodeError_SetStart(*exceptionObject, *startinpos))
+            goto onError;
+        if (PyUnicodeDecodeError_SetEnd(*exceptionObject, *endinpos))
+            goto onError;
+        if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
+            goto onError;
+    }
 
     restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL);
     if (restuple == NULL)
         goto onError;
     if (!PyTuple_Check(restuple)) {
         PyErr_SetString(PyExc_TypeError, &argparse[4]);
         goto onError;
     }
     if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
         goto onError;
@@ skipping 122 matching lines @@
  * on whether we are encoding whitespace as itself.  RFC2152 makes it
  * clear that the answers to these questions vary between
  * applications, so this code needs to be flexible.  */
 
 #define ENCODE_DIRECT(c, directO, directWS)             \
     ((c) < 128 && (c) > 0 &&                            \
      ((utf7_category[(c)] == 0) ||                      \
       (directWS && (utf7_category[(c)] == 2)) ||        \
       (directO && (utf7_category[(c)] == 1))))
 
-PyObject *
-PyUnicode_DecodeUTF7(const char *s,
-                     Py_ssize_t size,
-                     const char *errors)
+PyObject *PyUnicode_DecodeUTF7(const char *s,
+                               Py_ssize_t size,
+                               const char *errors)
 {
     return PyUnicode_DecodeUTF7Stateful(s, size, errors, NULL);
 }
 
 /* The decoder.  The only state we preserve is our read position,
  * i.e. how many characters we have consumed.  So if we end in the
  * middle of a shift sequence we have to back off the read position
  * and the output to the beginning of the sequence, otherwise we lose
  * all the shift state (seen bits, number of bits seen, high
  * surrogate). */
 
-PyObject *
-PyUnicode_DecodeUTF7Stateful(const char *s,
-                             Py_ssize_t size,
-                             const char *errors,
-                             Py_ssize_t *consumed)
+PyObject *PyUnicode_DecodeUTF7Stateful(const char *s,
+                                       Py_ssize_t size,
+                                       const char *errors,
+                                       Py_ssize_t *consumed)
 {
     const char *starts = s;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     const char *e;
     PyUnicodeObject *unicode;
     Py_UNICODE *p;
     const char *errmsg = "";
     int inShift = 0;
@@ skipping 166 matching lines @@
     return (PyObject *)unicode;
 
   onError:
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     Py_DECREF(unicode);
     return NULL;
 }
 
 
-PyObject *
-PyUnicode_EncodeUTF7(const Py_UNICODE *s,
-                     Py_ssize_t size,
-                     int base64SetO,
-                     int base64WhiteSpace,
-                     const char *errors)
+PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
+                               Py_ssize_t size,
+                               int base64SetO,
+                               int base64WhiteSpace,
+                               const char *errors)
 {
     PyObject *v;
     /* It might be possible to tighten this worst case */
     Py_ssize_t allocated = 8 * size;
     int inShift = 0;
     Py_ssize_t i = 0;
     unsigned int base64bits = 0;
     unsigned long base64buffer = 0;
     char * out;
     char * start;
@@ skipping 84 matching lines @@
 #undef ENCODE_DIRECT
 
 /* --- UTF-8 Codec -------------------------------------------------------- */
 
 static
 char utf8_code_length[256] = {
     /* Map UTF-8 encoded prefix byte to sequence length.  Zero means
        illegal prefix.  See RFC 3629 for details */
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00-0F */
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 70-7F */
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 80-8F */
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* B0-BF */
     0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0-C1 + C2-CF */
     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* D0-DF */
     3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* E0-EF */
     4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0  /* F0-F4 + F5-FF */
 };
 
-PyObject *
-PyUnicode_DecodeUTF8(const char *s,
-                     Py_ssize_t size,
-                     const char *errors)
+PyObject *PyUnicode_DecodeUTF8(const char *s,
+                               Py_ssize_t size,
+                               const char *errors)
 {
     return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
 }
 
 /* Mask to check or force alignment of a pointer to C 'long' boundaries */
 #define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1)
 
 /* Mask to quickly check whether a C 'long' contains a
    non-ASCII, UTF8-encoded char. */
 #if (SIZEOF_LONG == 8)
 # define ASCII_CHAR_MASK 0x8080808080808080L
 #elif (SIZEOF_LONG == 4)
 # define ASCII_CHAR_MASK 0x80808080L
 #else
 # error C 'long' size should be either 4 or 8!
 #endif
 
-PyObject *
-PyUnicode_DecodeUTF8Stateful(const char *s,
-                             Py_ssize_t size,
-                             const char *errors,
-                             Py_ssize_t *consumed)
+PyObject *PyUnicode_DecodeUTF8Stateful(const char *s,
+                                       Py_ssize_t size,
+                                       const char *errors,
+                                       Py_ssize_t *consumed)
 {
     const char *starts = s;
     int n;
     int k;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     const char *e, *aligned_end;
     PyUnicodeObject *unicode;
     Py_UNICODE *p;
@@ skipping 200 matching lines @@
 
   onError:
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     Py_DECREF(unicode);
     return NULL;
 }
 
 #undef ASCII_CHAR_MASK
 
-#ifdef __APPLE__
-
-/* Simplified UTF-8 decoder using surrogateescape error handler,
-   used to decode the command line arguments on Mac OS X. */
-
-wchar_t*
-_Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size)
-{
-    int n;
-    const char *e;
-    wchar_t *unicode, *p;
-
-    /* Note: size will always be longer than the resulting Unicode
-       character count */
-    if (PY_SSIZE_T_MAX / sizeof(wchar_t) < (size + 1)) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-    unicode = PyMem_Malloc((size + 1) * sizeof(wchar_t));
-    if (!unicode)
-        return NULL;
-
-    /* Unpack UTF-8 encoded data */
-    p = unicode;
-    e = s + size;
-    while (s < e) {
-        Py_UCS4 ch = (unsigned char)*s;
-
-        if (ch < 0x80) {
-            *p++ = (wchar_t)ch;
-            s++;
-            continue;
-        }
-
-        n = utf8_code_length[ch];
-        if (s + n > e) {
-            goto surrogateescape;
-        }
-
-        switch (n) {
-        case 0:
-        case 1:
-            goto surrogateescape;
-
-        case 2:
-            if ((s[1] & 0xc0) != 0x80)
-                goto surrogateescape;
-            ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
-            assert ((ch > 0x007F) && (ch <= 0x07FF));
-            *p++ = (wchar_t)ch;
-            break;
-
-        case 3:
-            /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf
-               will result in surrogates in range d800-dfff. Surrogates are
-               not valid UTF-8 so they are rejected.
-               See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
-               (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
-            if ((s[1] & 0xc0) != 0x80 ||
-                (s[2] & 0xc0) != 0x80 ||
-                ((unsigned char)s[0] == 0xE0 &&
-                 (unsigned char)s[1] < 0xA0) ||
-                ((unsigned char)s[0] == 0xED &&
-                 (unsigned char)s[1] > 0x9F)) {
-
-                goto surrogateescape;
-            }
-            ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
-            assert ((ch > 0x07FF) && (ch <= 0xFFFF));
-            *p++ = (Py_UNICODE)ch;
-            break;
-
-        case 4:
-            if ((s[1] & 0xc0) != 0x80 ||
-                (s[2] & 0xc0) != 0x80 ||
-                (s[3] & 0xc0) != 0x80 ||
-                ((unsigned char)s[0] == 0xF0 &&
-                 (unsigned char)s[1] < 0x90) ||
-                ((unsigned char)s[0] == 0xF4 &&
-                 (unsigned char)s[1] > 0x8F)) {
-                goto surrogateescape;
-            }
-            ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
-                 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
-            assert ((ch > 0xFFFF) && (ch <= 0x10ffff));
-
-#if SIZEOF_WCHAR_T == 4
-            *p++ = (wchar_t)ch;
-#else
-            /*  compute and append the two surrogates: */
-
-            /*  translate from 10000..10FFFF to 0..FFFF */
-            ch -= 0x10000;
-
-            /*  high surrogate = top 10 bits added to D800 */
-            *p++ = (wchar_t)(0xD800 + (ch >> 10));
-
-            /*  low surrogate = bottom 10 bits added to DC00 */
-            *p++ = (wchar_t)(0xDC00 + (ch & 0x03FF));
-#endif
-            break;
-        }
-        s += n;
-        continue;
-
-      surrogateescape:
-        *p++ = 0xDC00 + ch;
-        s++;
-    }
-    *p = L'\0';
-    return unicode;
-}
-
-#endif /* __APPLE__ */
 
 /* Allocation strategy:  if the string is short, convert into a stack buffer
    and allocate exactly as much space needed at the end.  Else allocate the
    maximum possible needed (4 result bytes per Unicode character), and return
    the excess memory at the end.
 */
 PyObject *
 PyUnicode_EncodeUTF8(const Py_UNICODE *s,
                      Py_ssize_t size,
                      const char *errors)
@@ skipping 151 matching lines @@
     return result;
  error:
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     Py_XDECREF(result);
     return NULL;
 
 #undef MAX_SHORT_UNICHARS
 }
 
-PyObject *
-PyUnicode_AsUTF8String(PyObject *unicode)
-{
-    PyObject *utf8;
+PyObject *PyUnicode_AsUTF8String(PyObject *unicode)
+{
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
-    utf8 = _PyUnicode_AsDefaultEncodedString(unicode);
-    if (utf8 == NULL)
-        return NULL;
-    Py_INCREF(utf8);
-    return utf8;
+    return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
+                                PyUnicode_GET_SIZE(unicode),
+                                NULL);
 }
 
 /* --- UTF-32 Codec ------------------------------------------------------- */
 
 PyObject *
 PyUnicode_DecodeUTF32(const char *s,
                       Py_ssize_t size,
                       const char *errors,
                       int *byteorder)
 {
@@ skipping 23 matching lines @@
     int bo = 0;       /* assume native ordering by default */
     const char *errmsg = "";
     /* Offsets from q for retrieving bytes in the right order. */
 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
     int iorder[] = {0, 1, 2, 3};
 #else
     int iorder[] = {3, 2, 1, 0};
 #endif
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
-
+    
     q = (unsigned char *)s;
     e = q + size;
 
     if (byteorder)
         bo = *byteorder;
 
     /* Check for BOM marks (U+FEFF) in the input and adjust current
        byte order setting accordingly. In native mode, the leading BOM
        mark is skipped, in all other modes, it is copied to the output
        stream as-is (giving a ZWNBSP character). */
@@ skipping 201 matching lines @@
         }
 #endif
         STORECHAR(ch);
     }
 
   done:
     return v;
 #undef STORECHAR
 }
 
-PyObject *
-PyUnicode_AsUTF32String(PyObject *unicode)
+PyObject *PyUnicode_AsUTF32String(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(unicode),
                                  PyUnicode_GET_SIZE(unicode),
                                  NULL,
                                  0);
 }
@@ skipping 369 matching lines @@
         STORECHAR(ch);
         if (ch2)
             STORECHAR(ch2);
     }
 
   done:
     return v;
 #undef STORECHAR
 }
 
-PyObject *
-PyUnicode_AsUTF16String(PyObject *unicode)
+PyObject *PyUnicode_AsUTF16String(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode),
                                  PyUnicode_GET_SIZE(unicode),
                                  NULL,
                                  0);
 }
 
 /* --- Unicode Escape Codec ----------------------------------------------- */
 
 static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
 
-PyObject *
-PyUnicode_DecodeUnicodeEscape(const char *s,
-                              Py_ssize_t size,
-                              const char *errors)
+PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
+                                        Py_ssize_t size,
+                                        const char *errors)
 {
     const char *starts = s;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     int i;
     PyUnicodeObject *v;
     Py_UNICODE *p;
     const char *end;
     char* message;
@@ skipping 84 matching lines @@
                 if (unicode_decode_call_errorhandler(
                         errors, &errorHandler,
                         "unicodeescape", "end of string in escape sequence",
                         &starts, &end, &startinpos, &endinpos, &exc, &s,
                         &v, &outpos, &p))
                     goto onError;
                 goto nextByte;
             }
             for (i = 0; i < digits; ++i) {
                 c = (unsigned char) s[i];
-                if (!Py_ISXDIGIT(c)) {
+                if (!ISXDIGIT(c)) {
                     endinpos = (s+i+1)-starts;
                     if (unicode_decode_call_errorhandler(
                             errors, &errorHandler,
                             "unicodeescape", message,
                             &starts, &end, &startinpos, &endinpos, &exc, &s,
                             &v, &outpos, &p))
                         goto onError;
                     goto nextByte;
                 }
                 chr = (chr<<4) & ~0xF;
@@ skipping 130 matching lines @@
         if (*s == ch)
             return s;
         s++;
     }
 
     return NULL;
 }
 
 static const char *hexdigits = "0123456789abcdef";
 
-PyObject *
-PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
-                              Py_ssize_t size)
+PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
+                                        Py_ssize_t size)
 {
     PyObject *repr;
     char *p;
 
 #ifdef Py_UNICODE_WIDE
     const Py_ssize_t expandsize = 10;
 #else
     const Py_ssize_t expandsize = 6;
 #endif
 
@@ skipping 119 matching lines @@
         else
             *p++ = (char) ch;
     }
 
     assert(p - PyBytes_AS_STRING(repr) > 0);
     if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0)
         return NULL;
     return repr;
 }
 
-PyObject *
-PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
+PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
 {
     PyObject *s;
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     s = PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
                                       PyUnicode_GET_SIZE(unicode));
     return s;
 }
 
 /* --- Raw Unicode Escape Codec ------------------------------------------- */
 
-PyObject *
-PyUnicode_DecodeRawUnicodeEscape(const char *s,
-                                 Py_ssize_t size,
-                                 const char *errors)
+PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
+                                           Py_ssize_t size,
+                                           const char *errors)
 {
     const char *starts = s;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     PyUnicodeObject *v;
     Py_UNICODE *p;
     const char *end;
     const char *bs;
     PyObject *errorHandler = NULL;
@@ skipping 37 matching lines @@
             continue;
         }
         p--;
         count = *s=='u' ? 4 : 8;
         s++;
 
         /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */
         outpos = p-PyUnicode_AS_UNICODE(v);
         for (x = 0, i = 0; i < count; ++i, ++s) {
             c = (unsigned char)*s;
-            if (!Py_ISXDIGIT(c)) {
+            if (!ISXDIGIT(c)) {
                 endinpos = s-starts;
                 if (unicode_decode_call_errorhandler(
                         errors, &errorHandler,
                         "rawunicodeescape", "truncated \\uXXXX",
                         &starts, &end, &startinpos, &endinpos, &exc, &s,
                         &v, &outpos, &p))
                     goto onError;
                 goto nextByte;
             }
             x = (x<<4) & ~0xF;
@@ skipping 36 matching lines @@
     Py_XDECREF(exc);
     return (PyObject *)v;
 
   onError:
     Py_XDECREF(v);
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     return NULL;
 }
 
-PyObject *
-PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
-                                 Py_ssize_t size)
+PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
+                                           Py_ssize_t size)
 {
     PyObject *repr;
     char *p;
     char *q;
 
 #ifdef Py_UNICODE_WIDE
     const Py_ssize_t expandsize = 10;
 #else
     const Py_ssize_t expandsize = 6;
 #endif
@@ skipping 66 matching lines @@
             *p++ = (char) ch;
     }
     size = p - q;
 
     assert(size > 0);
     if (_PyBytes_Resize(&repr, size) < 0)
         return NULL;
     return repr;
 }
 
-PyObject *
-PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
+PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
 {
     PyObject *s;
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     s = PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
                                          PyUnicode_GET_SIZE(unicode));
 
     return s;
 }
 
 /* --- Unicode Internal Codec ------------------------------------------- */
 
-PyObject *
-_PyUnicode_DecodeUnicodeInternal(const char *s,
-                                 Py_ssize_t size,
-                                 const char *errors)
+PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s,
+                                           Py_ssize_t size,
+                                           const char *errors)
 {
     const char *starts = s;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     PyUnicodeObject *v;
     Py_UNICODE *p;
     const char *end;
     const char *reason;
     PyObject *errorHandler = NULL;
@@ skipping 55 matching lines @@
 
   onError:
     Py_XDECREF(v);
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     return NULL;
 }
 
 /* --- Latin-1 Codec ------------------------------------------------------ */
 
-PyObject *
-PyUnicode_DecodeLatin1(const char *s,
-                       Py_ssize_t size,
-                       const char *errors)
+PyObject *PyUnicode_DecodeLatin1(const char *s,
+                                 Py_ssize_t size,
+                                 const char *errors)
 {
     PyUnicodeObject *v;
     Py_UNICODE *p;
     const char *e, *unrolled_end;
 
     /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
     if (size == 1) {
         Py_UNICODE r = *(unsigned char*)s;
         return PyUnicode_FromUnicode(&r, 1);
     }
@@ skipping 19 matching lines @@
     while (s < e)
         *p++ = (unsigned char) *s++;
     return (PyObject *)v;
 
   onError:
     Py_XDECREF(v);
     return NULL;
 }
 
 /* create or adjust a UnicodeEncodeError */
-static void
-make_encode_exception(PyObject **exceptionObject,
-                      const char *encoding,
-                      const Py_UNICODE *unicode, Py_ssize_t size,
-                      Py_ssize_t startpos, Py_ssize_t endpos,
-                      const char *reason)
+static void make_encode_exception(PyObject **exceptionObject,
+                                  const char *encoding,
+                                  const Py_UNICODE *unicode, Py_ssize_t size,
+                                  Py_ssize_t startpos, Py_ssize_t endpos,
+                                  const char *reason)
 {
     if (*exceptionObject == NULL) {
         *exceptionObject = PyUnicodeEncodeError_Create(
             encoding, unicode, size, startpos, endpos, reason);
     }
     else {
         if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos))
             goto onError;
         if (PyUnicodeEncodeError_SetEnd(*exceptionObject, endpos))
             goto onError;
         if (PyUnicodeEncodeError_SetReason(*exceptionObject, reason))
             goto onError;
         return;
       onError:
         Py_DECREF(*exceptionObject);
         *exceptionObject = NULL;
     }
 }
 
 /* raises a UnicodeEncodeError */
-static void
-raise_encode_exception(PyObject **exceptionObject,
-                       const char *encoding,
-                       const Py_UNICODE *unicode, Py_ssize_t size,
-                       Py_ssize_t startpos, Py_ssize_t endpos,
-                       const char *reason)
+static void raise_encode_exception(PyObject **exceptionObject,
+                                   const char *encoding,
+                                   const Py_UNICODE *unicode, Py_ssize_t size,
+                                   Py_ssize_t startpos, Py_ssize_t endpos,
+                                   const char *reason)
 {
     make_encode_exception(exceptionObject,
                           encoding, unicode, size, startpos, endpos, reason);
     if (*exceptionObject != NULL)
         PyCodec_StrictErrors(*exceptionObject);
 }
 
 /* error handling callback helper:
    build arguments, call the callback and check the arguments,
    put the result into newpos and return the replacement string, which
    has to be freed by the caller */
-static PyObject *
-unicode_encode_call_errorhandler(const char *errors,
-                                 PyObject **errorHandler,
-                                 const char *encoding, const char *reason,
-                                 const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
-                                 Py_ssize_t startpos, Py_ssize_t endpos,
-                                 Py_ssize_t *newpos)
+static PyObject *unicode_encode_call_errorhandler(const char *errors,
+                                                  PyObject **errorHandler,
+                                                  const char *encoding, const char *reason,
+                                                  const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
+                                                  Py_ssize_t startpos, Py_ssize_t endpos,
+                                                  Py_ssize_t *newpos)
 {
     static char *argparse = "On;encoding error handler must return (str/bytes, int) tuple";
 
     PyObject *restuple;
     PyObject *resunicode;
 
     if (*errorHandler == NULL) {
         *errorHandler = PyCodec_LookupError(errors);
         if (*errorHandler == NULL)
             return NULL;
@@ skipping 28 matching lines @@
     if (*newpos<0 || *newpos>size) {
         PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
         Py_DECREF(restuple);
         return NULL;
     }
     Py_INCREF(resunicode);
     Py_DECREF(restuple);
     return resunicode;
 }
 
-static PyObject *
-unicode_encode_ucs1(const Py_UNICODE *p,
-                    Py_ssize_t size,
-                    const char *errors,
-                    int limit)
+static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
+                                     Py_ssize_t size,
+                                     const char *errors,
+                                     int limit)
 {
     /* output object */
     PyObject *res;
     /* pointers to the beginning and end+1 of input */
     const Py_UNICODE *startp = p;
     const Py_UNICODE *endp = p + size;
     /* pointer to the beginning of the unencodable characters */
     /* const Py_UNICODE *badp = NULL; */
     /* pointer into the output */
     char *str;
@@ skipping 172 matching lines @@
     Py_XDECREF(exc);
     return res;
 
   onError:
     Py_XDECREF(res);
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     return NULL;
 }
 
-PyObject *
-PyUnicode_EncodeLatin1(const Py_UNICODE *p,
-                       Py_ssize_t size,
-                       const char *errors)
+PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
+                                 Py_ssize_t size,
+                                 const char *errors)
 {
     return unicode_encode_ucs1(p, size, errors, 256);
 }
 
-PyObject *
-PyUnicode_AsLatin1String(PyObject *unicode)
+PyObject *PyUnicode_AsLatin1String(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode),
                                   PyUnicode_GET_SIZE(unicode),
                                   NULL);
 }
 
 /* --- 7-bit ASCII Codec -------------------------------------------------- */
 
-PyObject *
-PyUnicode_DecodeASCII(const char *s,
-                      Py_ssize_t size,
-                      const char *errors)
+PyObject *PyUnicode_DecodeASCII(const char *s,
+                                Py_ssize_t size,
+                                const char *errors)
 {
     const char *starts = s;
     PyUnicodeObject *v;
     Py_UNICODE *p;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     const char *e;
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
@@ skipping 36 matching lines @@
     Py_XDECREF(exc);
     return (PyObject *)v;
 
   onError:
     Py_XDECREF(v);
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     return NULL;
 }
 
-PyObject *
-PyUnicode_EncodeASCII(const Py_UNICODE *p,
-                      Py_ssize_t size,
-                      const char *errors)
+PyObject *PyUnicode_EncodeASCII(const Py_UNICODE *p,
+                                Py_ssize_t size,
+                                const char *errors)
 {
     return unicode_encode_ucs1(p, size, errors, 128);
 }
 
-PyObject *
-PyUnicode_AsASCIIString(PyObject *unicode)
+PyObject *PyUnicode_AsASCIIString(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode),
                                  PyUnicode_GET_SIZE(unicode),
                                  NULL);
 }
 
 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
 
 /* --- MBCS codecs for Windows -------------------------------------------- */
 
 #if SIZEOF_INT < SIZEOF_SIZE_T
 #define NEED_RETRY
 #endif
 
 /* XXX This code is limited to "true" double-byte encodings, as
    a) it assumes an incomplete character consists of a single byte, and
    b) IsDBCSLeadByte (probably) does not work for non-DBCS multi-byte
    encodings, see IsDBCSLeadByteEx documentation. */
 
-static int
-is_dbcs_lead_byte(const char *s, int offset)
+static int is_dbcs_lead_byte(const char *s, int offset)
 {
     const char *curr = s + offset;
 
     if (IsDBCSLeadByte(*curr)) {
         const char *prev = CharPrev(s, curr);
         return (prev == curr) || !IsDBCSLeadByte(*prev) || (curr - prev == 2);
     }
     return 0;
 }
 
 /*
  * Decode MBCS string into unicode object. If 'final' is set, converts
  * trailing lead-byte too. Returns consumed size if succeed, -1 otherwise.
  */
-static int
-decode_mbcs(PyUnicodeObject **v,
-            const char *s, /* MBCS string */
-            int size, /* sizeof MBCS string */
-            int final,
-            const char *errors)
+static int decode_mbcs(PyUnicodeObject **v,
+                       const char *s, /* MBCS string */
+                       int size, /* sizeof MBCS string */
+                       int final)
 {
     Py_UNICODE *p;
-    Py_ssize_t n;
-    DWORD usize;
-    DWORD flags;
+    Py_ssize_t n = 0;
+    int usize = 0;
 
     assert(size >= 0);
-
-    /* check and handle 'errors' arg */
-    if (errors==NULL || strcmp(errors, "strict")==0)
-        flags = MB_ERR_INVALID_CHARS;
-    else if (strcmp(errors, "ignore")==0)
-        flags = 0;
-    else {
-        PyErr_Format(PyExc_ValueError,
-                     "mbcs encoding does not support errors='%s'",
-                     errors);
-        return -1;
-    }
 
     /* Skip trailing lead-byte unless 'final' is set */
     if (!final && size >= 1 && is_dbcs_lead_byte(s, size - 1))
         --size;
 
     /* First get the size of the result */
     if (size > 0) {
-        usize = MultiByteToWideChar(CP_ACP, flags, s, size, NULL, 0);
-        if (usize==0)
-            goto mbcs_decode_error;
-    } else
-        usize = 0;
+        usize = MultiByteToWideChar(CP_ACP, 0, s, size, NULL, 0);
+        if (usize == 0) {
+            PyErr_SetFromWindowsErrWithFilename(0, NULL);
+            return -1;
+        }
+    }
 
     if (*v == NULL) {
         /* Create unicode object */
         *v = _PyUnicode_New(usize);
         if (*v == NULL)
             return -1;
-        n = 0;
     }
     else {
         /* Extend unicode object */
         n = PyUnicode_GET_SIZE(*v);
         if (_PyUnicode_Resize(v, n + usize) < 0)
             return -1;
     }
 
     /* Do the conversion */
-    if (usize > 0) {
+    if (size > 0) {
         p = PyUnicode_AS_UNICODE(*v) + n;
-        if (0 == MultiByteToWideChar(CP_ACP, flags, s, size, p, usize)) {
-            goto mbcs_decode_error;
-        }
-    }
+        if (0 == MultiByteToWideChar(CP_ACP, 0, s, size, p, usize)) {
+            PyErr_SetFromWindowsErrWithFilename(0, NULL);
+            return -1;
+        }
+    }
+
     return size;
-
-mbcs_decode_error:
-    /* If the last error was ERROR_NO_UNICODE_TRANSLATION, then
-       we raise a UnicodeDecodeError - else it is a 'generic'
-       windows error
-     */
-    if (GetLastError()==ERROR_NO_UNICODE_TRANSLATION) {
-        /* Ideally, we should get reason from FormatMessage - this
-           is the Windows 2000 English version of the message
-        */
-        PyObject *exc = NULL;
-        const char *reason = "No mapping for the Unicode character exists "
-                             "in the target multi-byte code page.";
-        make_decode_exception(&exc, "mbcs", s, size, 0, 0, reason);
-        if (exc != NULL) {
-            PyCodec_StrictErrors(exc);
-            Py_DECREF(exc);
-        }
-    } else {
-        PyErr_SetFromWindowsErrWithFilename(0, NULL);
-    }
-    return -1;
-}
-
-PyObject *
-PyUnicode_DecodeMBCSStateful(const char *s,
-                             Py_ssize_t size,
-                             const char *errors,
-                             Py_ssize_t *consumed)
+}
+
+PyObject *PyUnicode_DecodeMBCSStateful(const char *s,
+                                       Py_ssize_t size,
+                                       const char *errors,
+                                       Py_ssize_t *consumed)
 {
     PyUnicodeObject *v = NULL;
     int done;
 
     if (consumed)
         *consumed = 0;
 
 #ifdef NEED_RETRY
   retry:
     if (size > INT_MAX)
-        done = decode_mbcs(&v, s, INT_MAX, 0, errors);
+        done = decode_mbcs(&v, s, INT_MAX, 0);
     else
 #endif
-        done = decode_mbcs(&v, s, (int)size, !consumed, errors);
+        done = decode_mbcs(&v, s, (int)size, !consumed);
 
     if (done < 0) {
         Py_XDECREF(v);
         return NULL;
     }
 
     if (consumed)
         *consumed += done;
 
 #ifdef NEED_RETRY
     if (size > INT_MAX) {
         s += done;
         size -= done;
         goto retry;
     }
 #endif
 
     return (PyObject *)v;
 }
 
-PyObject *
-PyUnicode_DecodeMBCS(const char *s,
-                     Py_ssize_t size,
-                     const char *errors)
+PyObject *PyUnicode_DecodeMBCS(const char *s,
+                               Py_ssize_t size,
+                               const char *errors)
 {
     return PyUnicode_DecodeMBCSStateful(s, size, errors, NULL);
 }
 
 /*
  * Convert unicode into string object (MBCS).
  * Returns 0 if succeed, -1 otherwise.
  */
-static int
-encode_mbcs(PyObject **repr,
-            const Py_UNICODE *p, /* unicode */
-            int size, /* size of unicode */
-            const char* errors)
-{
-    BOOL usedDefaultChar = FALSE;
-    BOOL *pusedDefaultChar;
-    int mbcssize;
-    Py_ssize_t n;
-    PyObject *exc = NULL;
-    DWORD flags;
+static int encode_mbcs(PyObject **repr,
+                       const Py_UNICODE *p, /* unicode */
+                       int size) /* size of unicode */
+{
+    int mbcssize = 0;
+    Py_ssize_t n = 0;
 
     assert(size >= 0);
-
-    /* check and handle 'errors' arg */
-    if (errors==NULL || strcmp(errors, "strict")==0) {
-        flags = WC_NO_BEST_FIT_CHARS;
-        pusedDefaultChar = &usedDefaultChar;
-    } else if (strcmp(errors, "replace")==0) {
-        flags = 0;
-        pusedDefaultChar = NULL;
-    } else {
-         PyErr_Format(PyExc_ValueError,
-                      "mbcs encoding does not support errors='%s'",
-                      errors);
-         return -1;
-    }
 
     /* First get the size of the result */
     if (size > 0) {
-        mbcssize = WideCharToMultiByte(CP_ACP, flags, p, size, NULL, 0,
-                                       NULL, pusedDefaultChar);
+        mbcssize = WideCharToMultiByte(CP_ACP, 0, p, size, NULL, 0, NULL, NULL);
         if (mbcssize == 0) {
             PyErr_SetFromWindowsErrWithFilename(0, NULL);
             return -1;
         }
-        /* If we used a default char, then we failed! */
-        if (pusedDefaultChar && *pusedDefaultChar)
-            goto mbcs_encode_error;
-    } else {
-        mbcssize = 0;
     }
 
     if (*repr == NULL) {
         /* Create string object */
         *repr = PyBytes_FromStringAndSize(NULL, mbcssize);
         if (*repr == NULL)
             return -1;
-        n = 0;
     }
     else {
         /* Extend string object */
         n = PyBytes_Size(*repr);
         if (_PyBytes_Resize(repr, n + mbcssize) < 0)
             return -1;
     }
 
     /* Do the conversion */
     if (size > 0) {
         char *s = PyBytes_AS_STRING(*repr) + n;
-        if (0 == WideCharToMultiByte(CP_ACP, flags, p, size, s, mbcssize,
-                                     NULL, pusedDefaultChar)) {
+        if (0 == WideCharToMultiByte(CP_ACP, 0, p, size, s, mbcssize, NULL, NULL)) {
             PyErr_SetFromWindowsErrWithFilename(0, NULL);
             return -1;
         }
-        if (pusedDefaultChar && *pusedDefaultChar)
-            goto mbcs_encode_error;
-    }
+    }
+
     return 0;
-
-mbcs_encode_error:
-    raise_encode_exception(&exc, "mbcs", p, size, 0, 0, "invalid character");
-    Py_XDECREF(exc);
-    return -1;
-}
-
-PyObject *
-PyUnicode_EncodeMBCS(const Py_UNICODE *p,
-                     Py_ssize_t size,
-                     const char *errors)
+}
+
+PyObject *PyUnicode_EncodeMBCS(const Py_UNICODE *p,
+                               Py_ssize_t size,
+                               const char *errors)
 {
     PyObject *repr = NULL;
     int ret;
 
 #ifdef NEED_RETRY
   retry:
     if (size > INT_MAX)
-        ret = encode_mbcs(&repr, p, INT_MAX, errors);
+        ret = encode_mbcs(&repr, p, INT_MAX);
     else
 #endif
-        ret = encode_mbcs(&repr, p, (int)size, errors);
+        ret = encode_mbcs(&repr, p, (int)size);
 
     if (ret < 0) {
         Py_XDECREF(repr);
         return NULL;
     }
 
 #ifdef NEED_RETRY
     if (size > INT_MAX) {
         p += INT_MAX;
         size -= INT_MAX;
         goto retry;
     }
 #endif
 
     return repr;
 }
 
-PyObject *
-PyUnicode_AsMBCSString(PyObject *unicode)
+PyObject *PyUnicode_AsMBCSString(PyObject *unicode)
 {
     if (!PyUnicode_Check(unicode)) {
         PyErr_BadArgument();
         return NULL;
     }
     return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode),
                                 PyUnicode_GET_SIZE(unicode),
                                 NULL);
 }
 
 #undef NEED_RETRY
 
 #endif /* MS_WINDOWS */
 
 /* --- Character Mapping Codec -------------------------------------------- */
 
-PyObject *
-PyUnicode_DecodeCharmap(const char *s,
-                        Py_ssize_t size,
-                        PyObject *mapping,
-                        const char *errors)
+PyObject *PyUnicode_DecodeCharmap(const char *s,
+                                  Py_ssize_t size,
+                                  PyObject *mapping,
+                                  const char *errors)
 {
     const char *starts = s;
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     Py_ssize_t outpos;
     const char *e;
     PyUnicodeObject *v;
     Py_UNICODE *p;
     Py_ssize_t extrachars = 0;
     PyObject *errorHandler = NULL;
@@ skipping 139 matching lines @@
 
   onError:
     Py_XDECREF(errorHandler);
     Py_XDECREF(exc);
     Py_XDECREF(v);
     return NULL;
 }
 
 /* Charmap encoding: the lookup table */
 
-struct encoding_map {
+struct encoding_map{
     PyObject_HEAD
     unsigned char level1[32];
     int count2, count3;
     unsigned char level23[1];
 };
 
 static PyObject*
 encoding_map_size(PyObject *obj, PyObject* args)
 {
     struct encoding_map *map = (struct encoding_map*)obj;
@@ skipping 106 matching lines @@
 
     if (count2 >= 0xFF || count3 >= 0xFF)
         need_dict = 1;
 
     if (need_dict) {
         PyObject *result = PyDict_New();
         PyObject *key, *value;
         if (!result)
             return NULL;
         for (i = 0; i < 256; i++) {
+            key = value = NULL;
             key = PyLong_FromLong(decode[i]);
             value = PyLong_FromLong(i);
             if (!key || !value)
                 goto failed1;
             if (PyDict_SetItem(result, key, value) == -1)
                 goto failed1;
             Py_DECREF(key);
             Py_DECREF(value);
         }
         return result;
@@ skipping 67 matching lines @@
     i = map->level23[16*map->count2 + 128*i + l3];
     if (i == 0) {
         return -1;
     }
     return i;
 }
 
 /* Lookup the character ch in the mapping. If the character
    can't be found, Py_None is returned (or NULL, if another
    error occurred). */
-static PyObject *
-charmapencode_lookup(Py_UNICODE c, PyObject *mapping)
+static PyObject *charmapencode_lookup(Py_UNICODE c, PyObject *mapping)
 {
     PyObject *w = PyLong_FromLong((long)c);
     PyObject *x;
 
     if (w == NULL)
         return NULL;
     x = PyObject_GetItem(mapping, w);
     Py_DECREF(w);
     if (x == NULL) {
         if (PyErr_ExceptionMatches(PyExc_LookupError)) {
@@ skipping 36 matching lines @@
     /* exponentially overallocate to minimize reallocations */
     if (requiredsize < 2*outsize)
         requiredsize = 2*outsize;
     if (_PyBytes_Resize(outobj, requiredsize))
         return -1;
     return 0;
 }
 
 typedef enum charmapencode_result {
     enc_SUCCESS, enc_FAILED, enc_EXCEPTION
-} charmapencode_result;
+}charmapencode_result;
 /* lookup the character, put the result in the output string and adjust
    various state variables. Resize the output bytes object if not enough
    space is available. Return a new reference to the object that
    was put in the output buffer, or Py_None, if the mapping was undefined
    (in which case no character was written) or NULL, if a
    reallocation error occurred. The caller must decref the result */
-static charmapencode_result
-charmapencode_output(Py_UNICODE c, PyObject *mapping,
-                     PyObject **outobj, Py_ssize_t *outpos)
+static
+charmapencode_result charmapencode_output(Py_UNICODE c, PyObject *mapping,
+                                          PyObject **outobj, Py_ssize_t *outpos)
 {
     PyObject *rep;
     char *outstart;
     Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj);
 
     if (Py_TYPE(mapping) == &EncodingMapType) {
         int res = encoding_map_lookup(c, mapping);
         Py_ssize_t requiredsize = *outpos+1;
         if (res == -1)
             return enc_FAILED;
@@ skipping 35 matching lines @@
             memcpy(outstart + *outpos, repchars, repsize);
             *outpos += repsize;
         }
     }
     Py_DECREF(rep);
     return enc_SUCCESS;
 }
 
 /* handle an error in PyUnicode_EncodeCharmap
    Return 0 on success, -1 on error */
-static int
-charmap_encoding_error(
+static
+int charmap_encoding_error(
     const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping,
     PyObject **exceptionObject,
     int *known_errorHandler, PyObject **errorHandler, const char *errors,
     PyObject **res, Py_ssize_t *respos)
 {
     PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
     Py_ssize_t repsize;
     Py_ssize_t newpos;
     Py_UNICODE *uni2;
     /* startpos for collecting unencodable chars */
@@ skipping 113 matching lines @@
                 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
                 return -1;
             }
         }
         *inpos = newpos;
         Py_DECREF(repunicode);
     }
     return 0;
 }
 
-PyObject *
-PyUnicode_EncodeCharmap(const Py_UNICODE *p,
-                        Py_ssize_t size,
-                        PyObject *mapping,
-                        const char *errors)
+PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
+                                  Py_ssize_t size,
+                                  PyObject *mapping,
+                                  const char *errors)
 {
     /* output object */
     PyObject *res = NULL;
     /* current input position */
     Py_ssize_t inpos = 0;
     /* current output position */
     Py_ssize_t respos = 0;
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
     /* the following variable is used for caching string comparisons
@@ skipping 40 matching lines @@
     Py_XDECREF(errorHandler);
     return res;
 
   onError:
     Py_XDECREF(res);
     Py_XDECREF(exc);
     Py_XDECREF(errorHandler);
     return NULL;
 }
 
-PyObject *
-PyUnicode_AsCharmapString(PyObject *unicode,
-                          PyObject *mapping)
+PyObject *PyUnicode_AsCharmapString(PyObject *unicode,
+                                    PyObject *mapping)
 {
     if (!PyUnicode_Check(unicode) || mapping == NULL) {
         PyErr_BadArgument();
         return NULL;
     }
     return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode),
                                    PyUnicode_GET_SIZE(unicode),
                                    mapping,
                                    NULL);
 }
 
 /* create or adjust a UnicodeTranslateError */
-static void
-make_translate_exception(PyObject **exceptionObject,
-                         const Py_UNICODE *unicode, Py_ssize_t size,
-                         Py_ssize_t startpos, Py_ssize_t endpos,
-                         const char *reason)
+static void make_translate_exception(PyObject **exceptionObject,
+                                     const Py_UNICODE *unicode, Py_ssize_t size,
+                                     Py_ssize_t startpos, Py_ssize_t endpos,
+                                     const char *reason)
 {
     if (*exceptionObject == NULL) {
         *exceptionObject = PyUnicodeTranslateError_Create(
             unicode, size, startpos, endpos, reason);
     }
     else {
         if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos))
             goto onError;
         if (PyUnicodeTranslateError_SetEnd(*exceptionObject, endpos))
             goto onError;
         if (PyUnicodeTranslateError_SetReason(*exceptionObject, reason))
             goto onError;
         return;
       onError:
         Py_DECREF(*exceptionObject);
         *exceptionObject = NULL;
     }
 }
 
 /* raises a UnicodeTranslateError */
-static void
-raise_translate_exception(PyObject **exceptionObject,
-                          const Py_UNICODE *unicode, Py_ssize_t size,
-                          Py_ssize_t startpos, Py_ssize_t endpos,
-                          const char *reason)
+static void raise_translate_exception(PyObject **exceptionObject,
+                                      const Py_UNICODE *unicode, Py_ssize_t size,
+                                      Py_ssize_t startpos, Py_ssize_t endpos,
+                                      const char *reason)
 {
     make_translate_exception(exceptionObject,
                              unicode, size, startpos, endpos, reason);
     if (*exceptionObject != NULL)
         PyCodec_StrictErrors(*exceptionObject);
 }
 
 /* error handling callback helper:
    build arguments, call the callback and check the arguments,
    put the result into newpos and return the replacement string, which
    has to be freed by the caller */
-static PyObject *
-unicode_translate_call_errorhandler(const char *errors,
-                                    PyObject **errorHandler,
-                                    const char *reason,
-                                    const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
-                                    Py_ssize_t startpos, Py_ssize_t endpos,
-                                    Py_ssize_t *newpos)
+static PyObject *unicode_translate_call_errorhandler(const char *errors,
+                                                     PyObject **errorHandler,
+                                                     const char *reason,
+                                                     const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
+                                                     Py_ssize_t startpos, Py_ssize_t endpos,
+                                                     Py_ssize_t *newpos)
 {
     static char *argparse = "O!n;translating error handler must return (str, int) tuple";
 
     Py_ssize_t i_newpos;
     PyObject *restuple;
     PyObject *resunicode;
 
     if (*errorHandler == NULL) {
         *errorHandler = PyCodec_LookupError(errors);
         if (*errorHandler == NULL)
@@ skipping 29 matching lines @@
         return NULL;
     }
     Py_INCREF(resunicode);
     Py_DECREF(restuple);
     return resunicode;
 }
 
 /* Lookup the character ch in the mapping and put the result in result,
    which must be decrefed by the caller.
    Return 0 on success, -1 on error */
-static int
-charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result)
+static
+int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result)
 {
     PyObject *w = PyLong_FromLong((long)c);
     PyObject *x;
 
     if (w == NULL)
         return -1;
     x = PyObject_GetItem(mapping, w);
     Py_DECREF(w);
     if (x == NULL) {
         if (PyErr_ExceptionMatches(PyExc_LookupError)) {
@@ skipping 28 matching lines @@
         /* wrong return value */
         PyErr_SetString(PyExc_TypeError,
                         "character mapping must return integer, None or str");
         Py_DECREF(x);
         return -1;
     }
 }
 /* ensure that *outobj is at least requiredsize characters long,
    if not reallocate and adjust various state variables.
    Return 0 on success, -1 on error */
-static int
-charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp,
+static
+int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp,
                                Py_ssize_t requiredsize)
 {
     Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj);
     if (requiredsize > oldsize) {
         /* remember old output position */
         Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj);
         /* exponentially overallocate to minimize reallocations */
         if (requiredsize < 2 * oldsize)
             requiredsize = 2 * oldsize;
         if (PyUnicode_Resize(outobj, requiredsize) < 0)
             return -1;
         *outp = PyUnicode_AS_UNICODE(*outobj) + outpos;
     }
     return 0;
 }
 /* lookup the character, put the result in the output string and adjust
    various state variables. Return a new reference to the object that
    was put in the output buffer in *result, or Py_None, if the mapping was
    undefined (in which case no character was written).
    The called must decref result.
    Return 0 on success, -1 on error. */
-static int
-charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp,
-                        Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp,
-                        PyObject **res)
+static
+int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp,
+                            Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp,
+                            PyObject **res)
 {
     if (charmaptranslate_lookup(*curinp, mapping, res))
         return -1;
     if (*res==NULL) {
         /* not found => default to 1:1 mapping */
         *(*outp)++ = *curinp;
     }
     else if (*res==Py_None)
         ;
     else if (PyLong_Check(*res)) {
@@ skipping 15 matching lines @@
                 return -1;
             memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize);
             *outp += repsize;
         }
     }
     else
         return -1;
     return 0;
 }
 
-PyObject *
-PyUnicode_TranslateCharmap(const Py_UNICODE *p,
-                           Py_ssize_t size,
-                           PyObject *mapping,
-                           const char *errors)
+PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p,
+                                     Py_ssize_t size,
+                                     PyObject *mapping,
+                                     const char *errors)
 {
     /* output object */
     PyObject *res = NULL;
     /* pointers to the beginning and end+1 of input */
     const Py_UNICODE *startp = p;
     const Py_UNICODE *endp = p + size;
     /* pointer into the output */
     Py_UNICODE *str;
     /* current output position */
     Py_ssize_t respos = 0;
@@ skipping 118 matching lines @@
     Py_XDECREF(errorHandler);
     return res;
 
   onError:
     Py_XDECREF(res);
     Py_XDECREF(exc);
     Py_XDECREF(errorHandler);
     return NULL;
 }
 
-PyObject *
-PyUnicode_Translate(PyObject *str,
-                    PyObject *mapping,
-                    const char *errors)
+PyObject *PyUnicode_Translate(PyObject *str,
+                              PyObject *mapping,
+                              const char *errors)
 {
     PyObject *result;
 
     str = PyUnicode_FromObject(str);
     if (str == NULL)
         goto onError;
     result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str),
                                         PyUnicode_GET_SIZE(str),
                                         mapping,
                                         errors);
     Py_DECREF(str);
     return result;
 
   onError:
     Py_XDECREF(str);
     return NULL;
 }
 
-PyObject *
-PyUnicode_TransformDecimalToASCII(Py_UNICODE *s,
-                                  Py_ssize_t length)
-{
-    PyObject *result;
-    Py_UNICODE *p; /* write pointer into result */
-    Py_ssize_t i;
-    /* Copy to a new string */
-    result = (PyObject *)_PyUnicode_New(length);
-    Py_UNICODE_COPY(PyUnicode_AS_UNICODE(result), s, length);
-    if (result == NULL)
-        return result;
-    p = PyUnicode_AS_UNICODE(result);
-    /* Iterate over code points */
-    for (i = 0; i < length; i++) {
-        Py_UNICODE ch =s[i];
-        if (ch > 127) {
-            int decimal = Py_UNICODE_TODECIMAL(ch);
-            if (decimal >= 0)
-                p[i] = '0' + decimal;
-        }
-    }
-    return result;
-}
 /* --- Decimal Encoder ---------------------------------------------------- */
 
-int
-PyUnicode_EncodeDecimal(Py_UNICODE *s,
-                        Py_ssize_t length,
-                        char *output,
-                        const char *errors)
+int PyUnicode_EncodeDecimal(Py_UNICODE *s,
+                            Py_ssize_t length,
+                            char *output,
+                            const char *errors)
 {
     Py_UNICODE *p, *end;
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
     const char *encoding = "decimal";
     const char *reason = "invalid decimal Unicode string";
     /* the following variable is used for caching string comparisons
      * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
     int known_errorHandler = -1;
 
@@ skipping 115 matching lines @@
   onError:
     Py_XDECREF(exc);
     Py_XDECREF(errorHandler);
     return -1;
 }
 
 /* --- Helpers ------------------------------------------------------------ */
 
 #include "stringlib/unicodedefs.h"
 #include "stringlib/fastsearch.h"
-
 #include "stringlib/count.h"
+/* Include _ParseTupleFinds from find.h */
+#define FROM_UNICODE
 #include "stringlib/find.h"
 #include "stringlib/partition.h"
-#include "stringlib/split.h"
 
 #define _Py_InsertThousandsGrouping _PyUnicode_InsertThousandsGrouping
 #define _Py_InsertThousandsGroupingLocale _PyUnicode_InsertThousandsGroupingLocale
 #include "stringlib/localeutil.h"
 
 /* helper macro to fixup start/end slice values */
-#define ADJUST_INDICES(start, end, len)         \
-    if (end > len)                              \
-        end = len;                              \
-    else if (end < 0) {                         \
-        end += len;                             \
-        if (end < 0)                            \
-            end = 0;                            \
-    }                                           \
-    if (start < 0) {                            \
-        start += len;                           \
-        if (start < 0)                          \
-            start = 0;                          \
-    }
-
-Py_ssize_t
-PyUnicode_Count(PyObject *str,
-                PyObject *substr,
-                Py_ssize_t start,
-                Py_ssize_t end)
+#define FIX_START_END(obj)                      \
+    if (start < 0)                              \
+        start += (obj)->length;                 \
+    if (start < 0)                              \
+        start = 0;                              \
+    if (end > (obj)->length)                    \
+        end = (obj)->length;                    \
+    if (end < 0)                                \
+        end += (obj)->length;                   \
+    if (end < 0)                                \
+        end = 0;
+
+Py_ssize_t PyUnicode_Count(PyObject *str,
+                           PyObject *substr,
+                           Py_ssize_t start,
+                           Py_ssize_t end)
 {
     Py_ssize_t result;
     PyUnicodeObject* str_obj;
     PyUnicodeObject* sub_obj;
 
     str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str);
     if (!str_obj)
         return -1;
     sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr);
     if (!sub_obj) {
         Py_DECREF(str_obj);
         return -1;
     }
 
-    ADJUST_INDICES(start, end, str_obj->length);
+    FIX_START_END(str_obj);
+
     result = stringlib_count(
-        str_obj->str + start, end - start, sub_obj->str, sub_obj->length,
-        PY_SSIZE_T_MAX
+        str_obj->str + start, end - start, sub_obj->str, sub_obj->length
         );
 
     Py_DECREF(sub_obj);
     Py_DECREF(str_obj);
 
     return result;
 }
 
-Py_ssize_t
-PyUnicode_Find(PyObject *str,
-               PyObject *sub,
-               Py_ssize_t start,
-               Py_ssize_t end,
-               int direction)
+Py_ssize_t PyUnicode_Find(PyObject *str,
+                          PyObject *sub,
+                          Py_ssize_t start,
+                          Py_ssize_t end,
+                          int direction)
 {
     Py_ssize_t result;
 
     str = PyUnicode_FromObject(str);
     if (!str)
         return -2;
     sub = PyUnicode_FromObject(sub);
     if (!sub) {
         Py_DECREF(str);
         return -2;
@@ skipping 11 matching lines @@
             PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub),
             start, end
             );
 
     Py_DECREF(str);
     Py_DECREF(sub);
 
     return result;
 }
 
-static int
-tailmatch(PyUnicodeObject *self,
-          PyUnicodeObject *substring,
-          Py_ssize_t start,
-          Py_ssize_t end,
-          int direction)
+static
+int tailmatch(PyUnicodeObject *self,
+              PyUnicodeObject *substring,
+              Py_ssize_t start,
+              Py_ssize_t end,
+              int direction)
 {
     if (substring->length == 0)
         return 1;
 
-    ADJUST_INDICES(start, end, self->length);
+    FIX_START_END(self);
+
     end -= substring->length;
     if (end < start)
         return 0;
 
     if (direction > 0) {
         if (Py_UNICODE_MATCH(self, end, substring))
             return 1;
     } else {
         if (Py_UNICODE_MATCH(self, start, substring))
             return 1;
     }
 
     return 0;
 }
 
-Py_ssize_t
-PyUnicode_Tailmatch(PyObject *str,
-                    PyObject *substr,
-                    Py_ssize_t start,
-                    Py_ssize_t end,
-                    int direction)
+Py_ssize_t PyUnicode_Tailmatch(PyObject *str,
+                               PyObject *substr,
+                               Py_ssize_t start,
+                               Py_ssize_t end,
+                               int direction)
 {
     Py_ssize_t result;
 
     str = PyUnicode_FromObject(str);
     if (str == NULL)
         return -1;
     substr = PyUnicode_FromObject(substr);
     if (substr == NULL) {
         Py_DECREF(str);
         return -1;
     }
 
     result = tailmatch((PyUnicodeObject *)str,
                        (PyUnicodeObject *)substr,
                        start, end, direction);
     Py_DECREF(str);
     Py_DECREF(substr);
     return result;
 }
 
 /* Apply fixfct filter to the Unicode object self and return a
    reference to the modified object */
 
-static PyObject *
-fixup(PyUnicodeObject *self,
-      int (*fixfct)(PyUnicodeObject *s))
+static
+PyObject *fixup(PyUnicodeObject *self,
+                int (*fixfct)(PyUnicodeObject *s))
 {
 
     PyUnicodeObject *u;
 
     u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length);
     if (u == NULL)
         return NULL;
 
     Py_UNICODE_COPY(u->str, self->str, self->length);
 
     if (!fixfct(u) && PyUnicode_CheckExact(self)) {
         /* fixfct should return TRUE if it modified the buffer. If
            FALSE, return a reference to the original buffer instead
            (to save space, not time) */
         Py_INCREF(self);
         Py_DECREF(u);
         return (PyObject*) self;
     }
     return (PyObject*) u;
 }
 
-static int
-fixupper(PyUnicodeObject *self)
+static
+int fixupper(PyUnicodeObject *self)
 {
     Py_ssize_t len = self->length;
     Py_UNICODE *s = self->str;
     int status = 0;
 
     while (len-- > 0) {
         register Py_UNICODE ch;
 
         ch = Py_UNICODE_TOUPPER(*s);
         if (ch != *s) {
             status = 1;
             *s = ch;
         }
         s++;
     }
 
     return status;
 }
 
-static int
-fixlower(PyUnicodeObject *self)
+static
+int fixlower(PyUnicodeObject *self)
 {
     Py_ssize_t len = self->length;
     Py_UNICODE *s = self->str;
     int status = 0;
 
     while (len-- > 0) {
         register Py_UNICODE ch;
 
         ch = Py_UNICODE_TOLOWER(*s);
         if (ch != *s) {
             status = 1;
             *s = ch;
         }
         s++;
     }
 
     return status;
 }
 
-static int
-fixswapcase(PyUnicodeObject *self)
+static
+int fixswapcase(PyUnicodeObject *self)
 {
     Py_ssize_t len = self->length;
     Py_UNICODE *s = self->str;
     int status = 0;
 
     while (len-- > 0) {
         if (Py_UNICODE_ISUPPER(*s)) {
             *s = Py_UNICODE_TOLOWER(*s);
             status = 1;
         } else if (Py_UNICODE_ISLOWER(*s)) {
             *s = Py_UNICODE_TOUPPER(*s);
             status = 1;
         }
         s++;
     }
 
     return status;
 }
 
-static int
-fixcapitalize(PyUnicodeObject *self)
+static
+int fixcapitalize(PyUnicodeObject *self)
 {
     Py_ssize_t len = self->length;
     Py_UNICODE *s = self->str;
     int status = 0;
 
     if (len == 0)
         return 0;
     if (Py_UNICODE_ISLOWER(*s)) {
         *s = Py_UNICODE_TOUPPER(*s);
         status = 1;
     }
     s++;
     while (--len > 0) {
         if (Py_UNICODE_ISUPPER(*s)) {
             *s = Py_UNICODE_TOLOWER(*s);
             status = 1;
         }
         s++;
     }
     return status;
 }
 
-static int
-fixtitle(PyUnicodeObject *self)
+static
+int fixtitle(PyUnicodeObject *self)
 {
     register Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
     register Py_UNICODE *e;
     int previous_is_cased;
 
     /* Shortcut for single character strings */
     if (PyUnicode_GET_SIZE(self) == 1) {
         Py_UNICODE ch = Py_UNICODE_TOTITLE(*p);
         if (*p != ch) {
             *p = ch;
@@ skipping 129 matching lines @@
   Done:
     Py_DECREF(fseq);
     return (PyObject *)res;
 
   onError:
     Py_DECREF(fseq);
     Py_XDECREF(res);
     return NULL;
 }
 
-static PyUnicodeObject *
-pad(PyUnicodeObject *self,
-    Py_ssize_t left,
-    Py_ssize_t right,
-    Py_UNICODE fill)
+static
+PyUnicodeObject *pad(PyUnicodeObject *self,
+                     Py_ssize_t left,
+                     Py_ssize_t right,
+                     Py_UNICODE fill)
 {
     PyUnicodeObject *u;
 
     if (left < 0)
         left = 0;
     if (right < 0)
         right = 0;
 
     if (left == 0 && right == 0 && PyUnicode_CheckExact(self)) {
         Py_INCREF(self);
@@ skipping 10 matching lines @@