diff -r 03a55e207720 Lib/email/charset.py --- a/Lib/email/charset.py Fri Nov 22 20:17:24 2013 -0500 +++ b/Lib/email/charset.py Sat Nov 23 22:29:27 2013 +0800 @@ -386,10 +386,11 @@ string using the ascii codec produces the correct string version of the content. """ - # 7bit/8bit encodings return the string unchanged (module conversions) + if not string: + return string if self.body_encoding is BASE64: if isinstance(string, str): - string = string.encode(self.output_charset) + string = string.encode(self.output_charset, 'surrogateescape') return email.base64mime.body_encode(string) elif self.body_encoding is QP: # quopromime.body_encode takes a string, but operates on it as if @@ -398,13 +399,9 @@ # character set, then, we must turn it into pseudo bytes via the # latin1 charset, which will encode any byte as a single code point # between 0 and 255, which is what body_encode is expecting. - # - # Note that this clause doesn't handle the case of a _payload that - # is already bytes. It never did, and the semantics of _payload - # being bytes has never been nailed down, so fixing that is a - # longer term TODO. if isinstance(string, str): - string = string.encode(self.output_charset).decode('latin1') + string = string.encode(self.output_charset) + string = string.decode('latin1') return email.quoprimime.body_encode(string) else: if isinstance(string, str): diff -r 03a55e207720 Lib/email/generator.py --- a/Lib/email/generator.py Fri Nov 22 20:17:24 2013 -0500 +++ b/Lib/email/generator.py Sat Nov 23 22:29:27 2013 +0800 @@ -15,7 +15,7 @@ from io import StringIO, BytesIO from email._policybase import compat32 from email.header import Header -from email.utils import _has_surrogates +from email.utils import _has_surrogates, collapse_rfc2231_value import email.charset as _charset UNDERSCORE = '_' @@ -172,19 +172,37 @@ # Do The Right Thing, and can still modify the Content-Type: header if # necessary. oldfp = self._fp + orig_payload = msg._payload + if 'content-transfer-encoding' in msg: + orig_cte = msg['content-transfer-encoding'] + else: + orig_cte = None try: - self._fp = sfp = self._new_buffer() - self._dispatch(msg) + try: + self._fp = sfp = self._new_buffer() + # Use base64 encoding to write 8bit payload with cte 8bit. + charset = msg.get_charset() + if orig_cte == '8bit' and charset is not None and\ + self.__class__ == Generator: + charset.body_encoding = _charset.BASE64 + msg._payload = charset.body_encode(msg._payload) + msg.replace_header('Content-Transfer-Encoding', + charset.get_body_encoding()) + self._dispatch(msg) + finally: + self._fp = oldfp + # Write the headers. First we see if the message object wants to + # handle that itself. If not, we'll do it generically. + meth = getattr(msg, '_write_headers', None) + if meth is None: + self._write_headers(msg) + else: + meth(self) + self._fp.write(sfp.getvalue()) finally: - self._fp = oldfp - # Write the headers. First we see if the message object wants to - # handle that itself. If not, we'll do it generically. - meth = getattr(msg, '_write_headers', None) - if meth is None: - self._write_headers(msg) - else: - meth(self) - self._fp.write(sfp.getvalue()) + msg._payload = orig_payload + if orig_cte is not None: + msg.replace_header('Content-Transfer-Encoding', orig_cte) def _dispatch(self, msg): # Get the Content-Type: for the message, then try to dispatch to @@ -390,9 +408,13 @@ # Bytes versions of this constant for use in manipulating data from # the BytesIO buffer. _encoded_EMPTY = b'' + _charset = None def write(self, s): - self._fp.write(s.encode('ascii', 'surrogateescape')) + if self._charset is not None: + self._fp.write(s.encode(self._charset)) + else: + self._fp.write(s.encode('ascii', 'surrogateescape')) def _new_buffer(self): return BytesIO() @@ -418,6 +440,7 @@ msg._payload = fcre.sub(">From ", msg._payload) self._write_lines(msg._payload) else: + self._charset = collapse_rfc2231_value(msg.get_param('charset')) super(BytesGenerator,self)._handle_text(msg) # Default body handler diff -r 03a55e207720 Lib/email/message.py --- a/Lib/email/message.py Fri Nov 22 20:17:24 2013 -0500 +++ b/Lib/email/message.py Sat Nov 23 22:29:27 2013 +0800 @@ -275,9 +275,16 @@ Optional charset sets the message's default character set. See set_charset() for details. """ - if isinstance(payload, bytes): - payload = payload.decode('ascii', 'surrogateescape') - self._payload = payload + if hasattr(payload, 'encode'): + if charset is None: + # We should check for ASCII-only here, but we can't do that + # for backward compatibility reasons. Fixed in 3.4. + self._payload = payload + return + if not isinstance(charset, Charset): + charset = Charset(charset) + payload = payload.encode(charset.output_charset) + self._payload = payload.decode('ascii', 'surrogateescape') if charset is not None: self.set_charset(charset) @@ -316,7 +323,15 @@ try: cte(self) except TypeError: - self._payload = charset.body_encode(self._payload) + # This if is for backward compatibility and will be removed + # in 3.4 when the ascii check is added to set_payload. + payload = self._payload + if payload: + try: + payload = payload.encode('ascii', 'surrogateescape') + except UnicodeError: + payload = payload.encode(charset.output_charset) + self._payload = charset.body_encode(payload) self.add_header('Content-Transfer-Encoding', cte) def get_charset(self): diff -r 03a55e207720 Lib/email/utils.py --- a/Lib/email/utils.py Fri Nov 22 20:17:24 2013 -0500 +++ b/Lib/email/utils.py Sat Nov 23 22:29:27 2013 +0800 @@ -331,6 +331,8 @@ def collapse_rfc2231_value(value, errors='replace', fallback_charset='us-ascii'): + if value is None: + return fallback_charset if not isinstance(value, tuple) or len(value) != 3: return unquote(value) # While value comes to us as a unicode string, we need it to be a bytes diff -r 03a55e207720 Lib/test/test_email/test_email.py --- a/Lib/test/test_email/test_email.py Fri Nov 22 20:17:24 2013 -0500 +++ b/Lib/test/test_email/test_email.py Sat Nov 23 22:29:27 2013 +0800 @@ -92,6 +92,49 @@ msg.set_payload('This is a string payload', charset) self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') + def test_set_payload_with_8bit_data_and_charset(self): + data = b'\xd0\x90\xd0\x91\xd0\x92' + charset = Charset('utf-8') + msg = Message() + msg.set_payload(data, charset) + self.assertEqual(msg['content-transfer-encoding'], 'base64') + self.assertEqual(msg.get_payload(decode=True), data) + self.assertEqual(msg.get_payload(), '0JDQkdCS\n') + + def test_set_payload_with_non_ascii_and_charset_body_encoding_none(self): + data = b'\xd0\x90\xd0\x91\xd0\x92' + charset = Charset('utf-8') + charset.body_encoding = None # Disable base64 encoding + msg = Message() + msg.set_payload(data.decode('utf-8'), charset) + self.assertEqual(msg['content-transfer-encoding'], '8bit') + self.assertEqual(msg.get_payload(decode=True), data) + + def test_set_payload_with_8bit_data_and_charset_body_encoding_none(self): + data = b'\xd0\x90\xd0\x91\xd0\x92' + charset = Charset('utf-8') + charset.body_encoding = None # Disable base64 encoding + msg = Message() + msg.set_payload(data, charset) + self.assertEqual(msg['content-transfer-encoding'], '8bit') + self.assertEqual(msg.get_payload(decode=True), data) + + def test_bytes_generator_after_calling_as_string(self): + data = b'\xd0\x90\xd0\x91\xd0\x92'.decode('utf-8') + cs = Charset('utf-8') + cs.body_encoding = None # Disable base64 encoding + msg = Message() + msg.set_payload(data, cs) + msg.as_string() # It will call set_payload implicitly! + fp = BytesIO() + g = BytesGenerator(fp) + g.flatten(msg) + self.assertEqual(fp.getvalue(), + (b'MIME-Version: 1.0\n' + b'Content-Type: text/plain; charset="utf-8"\n' + b'Content-Transfer-Encoding: 8bit\n\n' + b'\xd0\x90\xd0\x91\xd0\x92')) + def test_get_charsets(self): eq = self.assertEqual