--- a/Lib/smtpd.py 2014-06-11 08:34:13.049317102 +0200 +++ b/Lib/smtpd.py 2014-06-12 00:05:50.091680665 +0200 @@ -1,5 +1,5 @@ #! /usr/bin/env python3 -"""An RFC 5321 smtp proxy. +"""An RFC 5321 (opt. RFC 6532) smtp proxy. Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] @@ -25,6 +25,10 @@ Restrict the total size of the incoming message to "limit" number of bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. + --smtputf8 + -u + Enable the SMTPUTF8 extension and behave as RFC 6532 smtp proxy. + --debug -d Turn on debugging prints. @@ -98,7 +102,6 @@ DEBUGSTREAM = Devnull() NEWLINE = '\n' -EMPTYSTRING = '' COMMASPACE = ', ' DATA_SIZE_DEFAULT = 33554432 @@ -114,20 +117,47 @@ COMMAND = 0 DATA = 1 - command_size_limit = 512 - command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) - command_size_limits.update({ - 'MAIL': command_size_limit + 26, - }) - max_command_size_limit = max(command_size_limits.values()) + command_size_limits = collections.defaultdict(lambda : 512) + + @property + def max_command_size_limit(self): + try: + return max(self.command_size_limits.values()) + except ValueError: + # raised if command_size_limits has no values. 'MAIL' is just a + # random string here (wich would be added anyway). + return self.command_size_limits['MAIL'] def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, - map=None): + map=None, enable_SMTPUTF8=False, decode_data=None): asynchat.async_chat.__init__(self, conn, map=map) self.smtp_server = server self.conn = conn self.addr = addr self.data_size_limit = data_size_limit + self.enable_SMTPUTF8 = enable_SMTPUTF8 + if enable_SMTPUTF8: + if decode_data: + warn("Setting value of decode_data to False (enable_SMTPUTF8" + " implies this). Do not set decode_data to True to" + " suppress this warning.") + decode_data = False + if decode_data is None: + warn("The decode_data default of True will change to False in 3.6;" + " specify an explicit value for this keyword", + DeprecationWarning, 2) + decode_data = True + self._decode_data = decode_data + if decode_data: + self._emptystring = '' + self._linesep = '\r\n' + self._dotsep = '.' + self._newline = NEWLINE + else: + self._emptystring = b'' + self._linesep = b'\r\n' + self._dotsep = b'.' + self._newline = b'\n' self.received_lines = [] self.smtp_state = self.COMMAND self.seen_greeting = '' @@ -272,9 +302,10 @@ "set 'addr' instead", DeprecationWarning, 2) self.addr = value - # Overrides base class for convenience + # Overrides base class for convenience. def push(self, msg): - asynchat.async_chat.push(self, bytes(msg + '\r\n', 'ascii')) + asynchat.async_chat.push(self, bytes( + msg + '\r\n', 'utf-8' if self.enable_SMTPUTF8 else 'ascii')) # Implementation of base class abstract method def collect_incoming_data(self, data): @@ -287,11 +318,14 @@ return elif limit: self.num_bytes += len(data) - self.received_lines.append(str(data, "utf-8")) + if self._decode_data: + self.received_lines.append(str(data, 'utf-8')) + else: + self.received_lines.append(data) # Implementation of base class abstract method def found_terminator(self): - line = EMPTYSTRING.join(self.received_lines) + line = self._emptystring.join(self.received_lines) print('Data:', repr(line), file=DEBUGSTREAM) self.received_lines = [] if self.smtp_state == self.COMMAND: @@ -299,6 +333,15 @@ if not line: self.push('500 Error: bad syntax') return + if not self._decode_data: + line = str(line, 'utf-8') + #XXX: Should we check this? (breaks bug compatibility) + try: + line.encode('utf-8' if self.enable_SMTPUTF8 else 'ascii') + except UnicodeEncodeError: + self.push( + '500 Error: unknown characters in input.') + return method = None i = line.find(' ') if i < 0: @@ -307,8 +350,7 @@ else: command = line[:i].upper() arg = line[i+1:].strip() - max_sz = (self.command_size_limits[command] - if self.extended_smtp else self.command_size_limit) + max_sz = self.command_size_limits[command] if sz > max_sz: self.push('500 Error: line too long') return @@ -330,12 +372,12 @@ # Remove extraneous carriage returns and de-transparency according # to RFC 5321, Section 4.5.2. data = [] - for text in line.split('\r\n'): - if text and text[0] == '.': + for text in line.split(self._linesep): + if text and text[0] == self._dotsep: data.append(text[1:]) else: data.append(text) - self.received_data = NEWLINE.join(data) + self.received_data = self._newline.join(data) status = self.smtp_server.process_message(self.peer, self.mailfrom, self.rcpttos, @@ -371,9 +413,16 @@ else: self.seen_greeting = arg self.extended_smtp = True + self.command_size_limits.pop('MAIL') self.push('250-%s' % self.fqdn) if self.data_size_limit: self.push('250-SIZE %s' % self.data_size_limit) + self.command_size_limits['MAIL'] += 26 + if self.enable_SMTPUTF8: + self.push('250-8BITMIME') + self.push('250-SMTPUTF8') + self.command_size_limits['MAIL'] += 10 + self.push('250 HELP') def smtp_NOOP(self, arg): @@ -407,8 +456,8 @@ def _getparams(self, params): # Return any parameters that appear to be syntactically valid according # to RFC 1869, ignore all others. (Postel rule: accept what we can.) - params = [param.split('=', 1) for param in params.split() - if '=' in param] + params = [param.split('=', 1) if '=' in param else (param, True) for + param in params.split()] return {k: v for k, v in params if k.isalnum()} def smtp_HELP(self, arg): @@ -486,6 +535,22 @@ if params is None: self.push(syntaxerr) return + # XXX: I assume that the 8BITMIME parameter can be ignored since we are + # using 8-bit clean streams anyway. (The variable is used to validate + # SMTPUTF8 later.) + body = params.pop('BODY', '7BIT') + if params.pop('SMTPUTF8', False): + if not self.enable_SMTPUTF8: + self.push( + '555 MAIL FROM parameter SMTPUTF8 is not enabled.' + ) + return + elif body != '8BITMIME': + self.push( + '501 Syntax: MAIL FROM:
BODY=8BITMIME SMTPUTF8') + return + else: + self.require_SMTPUTF8 = True size = params.pop('SIZE', None) if size: if not size.isdigit(): @@ -577,13 +642,29 @@ channel_class = SMTPChannel def __init__(self, localaddr, remoteaddr, - data_size_limit=DATA_SIZE_DEFAULT, map=None): + data_size_limit=DATA_SIZE_DEFAULT, map=None, + enable_SMTPUTF8=False, decode_data=None): self._localaddr = localaddr self._remoteaddr = remoteaddr self.data_size_limit = data_size_limit + self.enable_SMTPUTF8 = enable_SMTPUTF8 + if enable_SMTPUTF8: + if decode_data: + warn("Setting value of decode_data to False (enable_SMTPUTF8" + " implies this). Do not set decode_data to True to" + " suppress this warning.") + decode_data = False + if decode_data is None: + warn("The decode_data default of True will change to False in 3.6;" + " specify an explicit value for this keyword", + DeprecationWarning, 2) + decode_data = True + self._decode_data = decode_data asyncore.dispatcher.__init__(self, map=map) try: - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + gai_results = socket.getaddrinfo(*localaddr, + type=socket.SOCK_STREAM) + self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible self.set_reuse_addr() self.bind(localaddr) @@ -598,8 +679,11 @@ def handle_accepted(self, conn, addr): print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) - channel = self.channel_class(self, conn, addr, self.data_size_limit, - self._map) + channel = self.channel_class( + self, conn, addr, self.data_size_limit, + self._map, + self.enable_SMTPUTF8, + self._decode_data) # API for "doing something useful with the message" def process_message(self, peer, mailfrom, rcpttos, data): @@ -761,17 +845,19 @@ class Options: - setuid = 1 + setuid = True classname = 'PureProxy' size_limit = None + enable_SMTPUTF8 = False def parseargs(): global DEBUGSTREAM try: opts, args = getopt.getopt( - sys.argv[1:], 'nVhc:s:d', - ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug']) + sys.argv[1:], 'nVhc:s:du', + ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', + 'smtputf8']) except getopt.error as e: usage(1, e) @@ -783,11 +869,13 @@ print(__version__) sys.exit(0) elif opt in ('-n', '--nosetuid'): - options.setuid = 0 + options.setuid = False elif opt in ('-c', '--class'): options.classname = arg elif opt in ('-d', '--debug'): DEBUGSTREAM = sys.stderr + elif opt in ('-u', '--smtputf8'): + options.enable_SMTPUTF8 = True elif opt in ('-s', '--size'): try: int_size = int(arg) @@ -842,7 +930,7 @@ class_ = getattr(mod, classname) proxy = class_((options.localhost, options.localport), (options.remotehost, options.remoteport), - options.size_limit) + options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) if options.setuid: try: import pwd --- a/Lib/test/test_smtpd.py 2014-06-11 08:34:13.049317102 +0200 +++ b/Lib/test/test_smtpd.py 2014-06-11 23:14:31.335310848 +0200 @@ -7,13 +7,18 @@ class DummyServer(smtpd.SMTPServer): - def __init__(self, localaddr, remoteaddr): - smtpd.SMTPServer.__init__(self, localaddr, remoteaddr) + def __init__(self, localaddr, remoteaddr, decode_data=True): + smtpd.SMTPServer.__init__(self, localaddr, remoteaddr, + decode_data=decode_data) self.messages = [] + if decode_data: + self.return_status = 'return status' + else: + self.return_status = b'return status' def process_message(self, peer, mailfrom, rcpttos, data): self.messages.append((peer, mailfrom, rcpttos, data)) - if data == 'return status': + if data == self.return_status: return '250 Okish' @@ -31,9 +36,10 @@ smtpd.socket = asyncore.socket = mock_socket def test_process_message_unimplemented(self): - server = smtpd.SMTPServer('a', 'b') + server = smtpd.SMTPServer((support.HOST, 0), ('b', 0), + decode_data=True) conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) def write_line(line): channel.socket.queue_recv(line) @@ -45,19 +51,44 @@ write_line(b'DATA') self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') + def test_decode_data_default_warns(self): + with self.assertWarns(DeprecationWarning): + smtpd.SMTPServer((support.HOST, 0), ('b', 0)) + def tearDown(self): asyncore.close_all() asyncore.socket = smtpd.socket = socket +class TestFamilyDetection(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") + def test_socket_uses_IPv6(self): + server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0), + decode_data=False) + self.assertEqual(server.socket.family, socket.AF_INET6) + + def test_socket_uses_IPv4(self): + server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0), + decode_data=False) + self.assertEqual(server.socket.family, socket.AF_INET) + + class SMTPDChannelTest(unittest.TestCase): def setUp(self): smtpd.socket = asyncore.socket = mock_socket self.old_debugstream = smtpd.DEBUGSTREAM self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer('a', 'b') + self.server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr) + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) def tearDown(self): asyncore.close_all() @@ -69,7 +100,9 @@ self.channel.handle_read() def test_broken_connect(self): - self.assertRaises(DummyDispatcherBroken, BrokenDummyServer, 'a', 'b') + self.assertRaises( + DummyDispatcherBroken, BrokenDummyServer, + (support.HOST, 0), ('b', 0)) def test_server_accept(self): self.server.handle_accept() @@ -195,24 +228,47 @@ def test_command_too_long(self): self.write_line(b'HELO example') self.write_line(b'MAIL from: ' + - b'a' * self.channel.command_size_limit + + b'a' * self.channel.command_size_limits['MAIL'] + b'@example') self.assertEqual(self.channel.socket.last, b'500 Error: line too long\r\n') def test_MAIL_command_limit_extended_with_SIZE(self): self.write_line(b'EHLO example') - fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') - self.write_line(b'MAIL from:<' + - b'a' * fill_len + - b'@example> SIZE=1234') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - + fill_len = 512 - len('MAIL from:<@example>') self.write_line(b'MAIL from:<' + b'a' * (fill_len + 26) + b'@example> SIZE=1234') self.assertEqual(self.channel.socket.last, b'500 Error: line too long\r\n') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_unicode_output_raises_error_by_default(self): + self.write_line('EHLO éxämplé'.encode('utf-8')) + self.assertEqual( + self.channel.socket.last, + b'500 Error: unknown characters in input.\r\n') + + def test_MAIL_command_rejects_SMTPUTF8_by_default(self): + self.write_line(b'EHLO example') + self.write_line( + b'MAIL from: BODY=8BITMIME SMTPUTF8') + self.assertEqual( + self.channel.socket.last, + b'555 MAIL FROM parameter SMTPUTF8 is not enabled.\r\n') + + def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): + self.channel.enable_SMTPUTF8 = True + self.write_line(b'EHLO example') + self.write_line( + 'MAIL from: BODY=8BITMIME SMTPUTF8'.encode( + 'utf-8') + ) + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.channel.enable_SMTPUTF8 = False def test_data_longer_than_default_data_size_limit(self): # Hack the default so we don't have to generate so much data. @@ -502,6 +558,22 @@ with support.check_warnings(('', DeprecationWarning)): self.channel._SMTPChannel__addr = 'spam' + def test_decode_data_default_warning(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = self.server.accept() + with self.assertWarns(DeprecationWarning): + smtpd.SMTPChannel(server, conn, addr) + +@unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") +class SMTPDChannelIPv6Test(SMTPDChannelTest): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOSTv6, 0), ('b', 0)) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): @@ -509,10 +581,11 @@ smtpd.socket = asyncore.socket = mock_socket self.old_debugstream = smtpd.DEBUGSTREAM self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer('a', 'b') + self.server = DummyServer((support.HOST, 0), ('b', 0)) conn, addr = self.server.accept() # Set DATA size limit to 32 bytes for easy testing - self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32) + self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, + decode_data=True) def tearDown(self): asyncore.close_all() @@ -553,5 +626,94 @@ b'552 Error: Too much mail data\r\n') +class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=False) + conn, addr = self.server.accept() + # Set decode_data to False + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=False) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, b'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' + b'and some plain ascii') + + +class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set decode_data to True + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, 'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + 'utf8 enriched text: żźć\nand some plain ascii') + + if __name__ == "__main__": unittest.main() --- a/Doc/library/smtpd.rst 2014-06-11 08:34:13.049317102 +0200 +++ b/Doc/library/smtpd.rst 2014-06-11 23:20:15.514631130 +0200 @@ -20,7 +20,8 @@ Additionally the SMTPChannel may be extended to implement very specific interaction behaviour with SMTP clients. -The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE extension. +The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531` +SMTPUTF8 extensions. SMTPServer Objects @@ -28,7 +29,11 @@ .. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ - map=None) +<<<<<<< local + map=None, enable_SMTPUTF8=False) +======= + map=None, decode_data=True) +>>>>>>> other Create a new :class:`SMTPServer` object, which binds to local address *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. It @@ -39,8 +44,16 @@ accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no limit. + *enable_SMTPUTF8* determins whether the SMTPUTF8 extension (as defined in + :RFC:`6531`) should be enabled. Default: ``False``. + A dictionary can be specified in *map* to avoid using a global socket map. + *decode_data* specifies whether the data portion of the SMTP transaction + should be decoded using UTF-8. The default is ``True`` for backward + compatibility reasons, but will change to ``False`` in Python 3.6. Specify + the keyword value explicitly to avoid the :exc:`DeprecationWarning`. + .. method:: process_message(peer, mailfrom, rcpttos, data) Raise :exc:`NotImplementedError` exception. Override this in subclasses to @@ -51,6 +64,10 @@ containing the contents of the e-mail (which should be in :rfc:`2822` format). + If the *decode_data* constructor keyword is set to ``True``, the *data* + argument will be a unicode string. If it is set to ``False``, it + will be a bytes object. + .. attribute:: channel_class Override this in subclasses to use a custom :class:`SMTPChannel` for @@ -59,6 +76,14 @@ .. versionchanged:: 3.4 The *map* argument was added. +<<<<<<< local + .. versionchanged:: 3.6 + The *enable_SMTPUTF8* argument was added. +======= + .. versionchanged:: 3.5 the *decode_data* argument was added, and *localaddr* + and *remoteaddr* may now contain IPv6 addresses. +>>>>>>> other + DebuggingServer Objects ----------------------- @@ -97,7 +122,7 @@ ------------------- .. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ - map=None)) + map=None, enable_SMTPUTF8=False, decode_data=True) Create a new :class:`SMTPChannel` object which manages the communication between the server and a single SMTP client. @@ -108,11 +133,23 @@ accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no limit. + *enable_SMTPUTF8* determins whether the SMTPUTF8 extension (as defined in + :RFC:`6531`) should be enabled. This will set *decode_data* + to ``False`` regardless of the given value. Default: ``False``. + A dictionary can be specified in *map* to avoid using a global socket map. + *decode_data* specifies whether the data portion of the SMTP transaction + should be decoded using UTF-8. The default is ``True`` for backward + compatibility reasons, but will change to ``False`` in Python 3.6. Specify + the keyword value explicitly to avoid the :exc:`DeprecationWarning`. + To use a custom SMTPChannel implementation you need to override the :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. + .. versionchanged:: 3.5 + the *decode_data* and *enable_SMTPUTF8* arguments were added. + The :class:`SMTPChannel` has the following instance variables: .. attribute:: smtp_server