diff -r 3b94a4ef244e Lib/test/test_codecs.py --- a/Lib/test/test_codecs.py Fri Feb 07 17:53:13 2014 +0100 +++ b/Lib/test/test_codecs.py Fri Feb 07 19:46:46 2014 +0200 @@ -124,8 +124,6 @@ "".join(codecs.iterdecode([bytes([c]) for c in encoded], self.encoding)) ) - # Temporary skip, see http://bugs.python.org/issue20542 - @unittest.skip def test_readline(self): def getreader(input): stream = io.BytesIO(input.encode(self.encoding)) @@ -899,13 +897,40 @@ def test_partial(self): self.check_partial( - "a+-b", + 'a+-b\x00c\x80d\u0100e\U00010000f', [ - "a", - "a", - "a+", - "a+-", - "a+-b", + 'a', + 'a', + 'a+', + 'a+-', + 'a+-b', + 'a+-b', + 'a+-b', + 'a+-b', + 'a+-b', + 'a+-b\x00', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c', + 'a+-b\x00c\x80', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d', + 'a+-b\x00c\x80d\u0100', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e', + 'a+-b\x00c\x80d\u0100e\U00010000', + 'a+-b\x00c\x80d\u0100e\U00010000f', ] ) diff -r 3b94a4ef244e Objects/unicodeobject.c --- a/Objects/unicodeobject.c Fri Feb 07 17:53:13 2014 +0100 +++ b/Objects/unicodeobject.c Fri Feb 07 19:46:46 2014 +0200 @@ -4459,8 +4459,16 @@ /* return state */ if (consumed) { if (inShift) { + *consumed = startinpos; + if (writer.pos != shiftOutStart && writer.maxchar > 127) { + PyObject *result = PyUnicode_FromKindAndData( + writer.kind, writer.data, shiftOutStart); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + _PyUnicodeWriter_Dealloc(&writer); + return result; + } writer.pos = shiftOutStart; /* back off output */ - *consumed = startinpos; } else { *consumed = s-starts;