Index: Misc/NEWS =================================================================== --- Misc/NEWS (revision 86110) +++ Misc/NEWS (working copy) @@ -205,6 +205,9 @@ - Issue #9948: Fixed problem of losing filename case information. +- Issue #1926: Added nntp SSL on port 563 and STARTTLS support to + nntplib. Patch by Andrew Vant. + Extensions ---------- Index: Doc/library/nntplib.rst =================================================================== --- Doc/library/nntplib.rst (revision 86110) +++ Doc/library/nntplib.rst (working copy) @@ -69,6 +69,18 @@ *readermode* defaults to ``None``. *usenetrc* defaults to ``True``. +.. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=True, [timeout]) + + Return a new instance of the :class:`NNTP_SSL` class. + :class:`NNTP_SSL` objects have the same methods as :class:`NNTP` objects. + If *port* is omitted, port 563 (NNTPS) is used. *ssl_context* is also + optional, and is an ssl.SSLContext object. All other parameters + behave the same as for :class:`NNTP`. + + Note that SSL-on-563 is discouraged per RFC 4642, in favor of + STARTTLS as described below. However, some servers only support the + former. + .. exception:: NNTPError Derived from the standard exception :exc:`Exception`, this is the base @@ -157,6 +169,32 @@ .. versionadded:: 3.2 +.. method:: NNTP.login(user=None, password=None, usenetrc=True) + + Send ``AUTHINFO`` commands with the user name and password. If *user* + and *password* are None and *usenetrc* is True, credentials from + ~/.netrc will be used if possible. + + Unless intentionally delayed, login is normally performed during an NNTP + object initialization and separately calling this function is unnecessary. + + +.. method:: NNTP.starttls(ssl_context=None) + + Send a ``STARTTLS`` command. The *ssl_context* argument is optional + and should be a :class:`ssl.SSLContext` object. This will enable + encryption on the NNTP connection. + + Note that this should not be done after authentication information has + been transmitted, and authentication occurs by default if possible during a + :class:`NNTP` object initialization. To force authentication to be + delayed, you must not set *user* or *password* when creating the + object, and must set *usenetrc* to False. + + After calling :meth:`NNTP.starttls`, :meth:`NNTP.login` must be + called separately. + + .. method:: NNTP.newgroups(date, *, file=None) Send a ``NEWGROUPS`` command. The *date* argument should be a Index: Lib/nntplib.py =================================================================== --- Lib/nntplib.py (revision 86110) +++ Lib/nntplib.py (working copy) @@ -69,6 +69,14 @@ import datetime import warnings + +try: + import ssl +except ImportError: + _have_ssl = False +else: + _have_ssl = True + from email.header import decode_header as _email_decode_header from socket import _GLOBAL_DEFAULT_TIMEOUT @@ -111,6 +119,7 @@ # Standard port used by NNTP servers NNTP_PORT = 119 +NNTP_SSL_PORT = 563 # Response numbers that are followed by additional text (e.g. article) @@ -193,7 +202,7 @@ overview = [] for line in lines: fields = {} - article_number, *tokens = line.split('\t') + article_number, *tokens = line.split("\t") article_number = int(article_number) for i, token in enumerate(tokens): if i >= len(fmt): @@ -261,7 +270,22 @@ date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) return date_str, time_str +if _have_ssl: + def _encrypt_on(sock, context): + """Wrap a socket in SSL/TLS. Arguments: + - sock: Socket to wrap + - context: SSL context to use for the encrypted connection + Returns: + - sock: New, encrypted socket. + """ + #Generate a default SSL context if none was passed. + if context is None: + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.options |= ssl.OP_NO_SSLv2 + #SSLv2 considered harmful. + return context.wrap_socket(sock) + # The classes themselves class _NNTPBase: # UTF-8 is the character set for all NNTP commands and responses: they @@ -279,18 +303,14 @@ encoding = 'utf-8' errors = 'surrogateescape' - def __init__(self, file, host, user=None, password=None, - readermode=None, usenetrc=True, + def __init__(self, file, host, readermode=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Initialize an instance. Arguments: - - file: file-like object (open for read/write in binary mode) - - host: hostname of the server (used if `usenetrc` is True) - - user: username to authenticate with - - password: password to use with username + - host: nntp server hostname + - port: nntp server port + - sock: socket used for the connection - readermode: if true, send 'mode reader' command after connecting. - - usenetrc: allow loading username and password from ~/.netrc file - if not specified explicitly - timeout: timeout (in seconds) used for socket connections readermode is sometimes necessary if you are connecting to an @@ -298,30 +318,92 @@ reader-specific comamnds, such as `group'. If you get unexpected NNTPPermanentErrors, you might need to set readermode. + + Since the order in which certain operations need to be done varies + between normal, SSL, and STARTTLS connections varies, some + initialization must be done in the subclasses. """ + self.host = host self.file = file self.debugging = 0 self.welcome = self._getresp() - + # 'mode reader' is sometimes necessary to enable 'reader' mode. # However, the order in which 'mode reader' and 'authinfo' need to - # arrive differs between some NNTP servers. Try to send - # 'mode reader', and if it fails with an authorization failed - # error, try again after sending authinfo. - readermode_afterauth = 0 + # arrive differs between some NNTP servers. If setreadermode() fails + # with an authorization failed error, it will set this to true; + # the login() routine will interpret that as a request to try again + # after performing its normal function. + self.readermode_afterauth = False if readermode: + self.setreadermode() + + #RFC4642 2.2.2: Both the client and the server MUST know if there is + #a TLS session active. A client MUST NOT attempt to start a TLS + #session if a TLS session is already active. + self.tls_on = False + + # Inquire about capabilities (RFC 3977). + self._caps = None + self.getcapabilities() + # Log in and encryption setup order is left to subclasses. + + def getwelcome(self): + """Get the welcome message from the server + (this is read and squirreled away by __init__()). + If the response code is 200, posting is allowed; + if it 201, posting is not allowed.""" + + if self.debugging: print('*welcome*', repr(self.welcome)) + return self.welcome + + def getcapabilities(self): + """Get the server capabilities. + If the CAPABILITIES command is not supported, an empty dict is + returned.""" + if self._caps is None: + self.nntp_version = 1 try: - self.welcome = self._shortcmd('mode reader') + resp, caps = self.capabilities() except NNTPPermanentError: - # error 500, probably 'not implemented' - pass - except NNTPTemporaryError as e: - if user and e.response.startswith('480'): - # Need authorization before 'mode reader' - readermode_afterauth = 1 - else: - raise - # If no login/password was specified, try to get them from ~/.netrc + # Server doesn't support capabilities + self._caps = {} + else: + self._caps = caps + if 'VERSION' in caps: + self.nntp_version = int(caps['VERSION'][0]) + return self._caps + + def capabilities(self): + """Process a CAPABILITIES command. Not supported by all servers. + Return: + - resp: server response if successful + - caps: a dictionary mapping capability names to lists of tokens + (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) + """ + caps = {} + resp, lines = self._longcmdstring("CAPABILITIES") + for line in lines: + name, *tokens = line.split() + caps[name] = tokens + return resp, caps + + def setreadermode(self): + try: + self.welcome = self._shortcmd('mode reader') + except NNTPPermanentError: + # error 500, probably 'not implemented' + pass + except NNTPTemporaryError as e: + if user and e.response.startswith('480'): + # Need authorization before 'mode reader' + self.readermode_afterauth = True + else: + raise + + def login(self, user=None, password=None, usenetrc=True): + # If no login/password was specified but netrc was requested, + # try to get them from ~/.netrc # Presume that if .netc has an entry, NNRP authentication is required. try: if usenetrc and not user: @@ -344,40 +426,10 @@ 'authinfo pass '+password) if not resp.startswith('281'): raise NNTPPermanentError(resp) - if readermode_afterauth: - try: - self.welcome = self._shortcmd('mode reader') - except NNTPPermanentError: - # error 500, probably 'not implemented' - pass + #Attempt to send mode reader if it was requested after login. + if self.readermode_afterauth: + self.setreadermode() - # Inquire about capabilities (RFC 3977) - self.nntp_version = 1 - try: - resp, caps = self.capabilities() - except NNTPPermanentError: - # Server doesn't support capabilities - self._caps = {} - else: - self._caps = caps - if 'VERSION' in caps: - self.nntp_version = int(caps['VERSION'][0]) - - def getwelcome(self): - """Get the welcome message from the server - (this is read and squirreled away by __init__()). - If the response code is 200, posting is allowed; - if it 201, posting is not allowed.""" - - if self.debugging: print('*welcome*', repr(self.welcome)) - return self.welcome - - def getcapabilities(self): - """Get the server capabilities, as read by __init__(). - If the CAPABILITIES command is not supported, an empty dict is - returned.""" - return self._caps - def set_debuglevel(self, level): """Set the debugging level. Argument 'level' means: 0: no debugging output (default) @@ -522,20 +574,6 @@ # Parse lines into "group last first flag" return [GroupInfo(*line.split()) for line in lines] - def capabilities(self): - """Process a CAPABILITIES command. Not supported by all servers. - Return: - - resp: server response if successful - - caps: a dictionary mapping capability names to lists of tokens - (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) - """ - caps = {} - resp, lines = self._longcmdstring("CAPABILITIES") - for line in lines: - name, *tokens = line.split() - caps[name] = tokens - return resp, caps - def newgroups(self, date, *, file=None): """Process a NEWGROUPS command. Arguments: - date: a date or datetime object @@ -893,6 +931,26 @@ - resp: server response if successful Note that if the server refuses the article an exception is raised.""" return self._post('IHAVE {0}'.format(message_id), data) + + if _have_ssl: + def starttls(self, context=None): + """Process a STARTTLS command. Arguments: + - context: SSL context to use for the encrypted connection + """ + resp = self._shortcmd('STARTTLS') + if resp.startswith('382'): + self.file.close() + self.sock = _encrypt_on(self.sock, context) + self.file = self.sock.makefile("rwb") + self.tls_on = True + #Per RFC4642 capabilities may change after TLS starts up, so + #ask for them again. + self._caps = None + self.getcapabilities() + else: + #Either the server doesn't support STARTTLS, or it's already + #active + raise NNTPPermanentError("TLS unavailable or not supported.") def _close(self): self.file.close() @@ -930,12 +988,11 @@ unexpected NNTPPermanentErrors, you might need to set readermode. """ - self.host = host - self.port = port self.sock = socket.create_connection((host, port), timeout) file = self.sock.makefile("rwb") - _NNTPBase.__init__(self, file, host, user, password, - readermode, usenetrc, timeout) + _NNTPBase.__init__(self, file, host, + readermode=readermode, timeout=timeout) + self.login(user, password, usenetrc) def _close(self): try: @@ -944,11 +1001,36 @@ self.sock.close() +if _have_ssl: + + class NNTP_SSL(_NNTPBase): + + def __init__(self, host, port=NNTP_SSL_PORT, + user=None, password=None, ssl_context=None, + readermode=None, usenetrc=True, + timeout=_GLOBAL_DEFAULT_TIMEOUT): + """This works identically to NNTP.__init__, except for the change + in default port and the context argument for SSL connections. + """ + self.sock = socket.create_connection((host, port), timeout) + self.sock = _encrypt_on(self.sock, ssl_context) + file = self.sock.makefile("rwb") + _NNTPBase.__init__(self, file, host, + readermode=readermode, timeout=timeout) + self.login(user, password, usenetrc) + + def _close(self): + try: + _NNTPBase._close(self) + finally: + self.sock.close() + + __all__.append("NNTP_SSL") + + # Test retrieval when run as a script. -if __name__ == '__main__': +if __name__ == "__main__": import argparse - from email.utils import parsedate - parser = argparse.ArgumentParser(description="""\ nntplib built-in demo - display the latest articles in a newsgroup""") parser.add_argument('-g', '--group', default='gmane.comp.python.general', @@ -959,25 +1041,52 @@ help='NNTP port number (default: %(default)s)') parser.add_argument('-n', '--nb-articles', default=10, type=int, help='number of articles to fetch (default: %(default)s)') + parser.add_argument('-ss', '--ssl_server', default='snews.gmane.org', + help='NNTPS server hostname (default: %(default)s)') + parser.add_argument('-sp', '--ssl_port', default=NNTP_SSL_PORT, type=int, + help='NNTPS port number (default: %(default)s)') + parser.add_argument('-c', '--ssl_context', default=None, + help='SSL context to use for connection (default: %(default)s)') + parser.add_argument('-u', '--user', default=None, + help='Login username (default: %(default)s)') + parser.add_argument('-pw', '--password', default=None, + help='Login password (default: %(default)s)') args = parser.parse_args() - - s = NNTP(host=args.server, port=args.port) - resp, count, first, last, name = s.group(args.group) - print('Group', name, 'has', count, 'articles, range', first, 'to', last) - + def cut(s, lim): if len(s) > lim: s = s[:lim - 4] + "..." return s - first = str(int(last) - args.nb_articles + 1) - resp, overviews = s.xover(first, last) - for artnum, over in overviews: - author = decode_header(over['from']).split('<', 1)[0] - subject = decode_header(over['subject']) - lines = int(over[':lines']) - print("{:7} {:20} {:42} ({})".format( - artnum, cut(author, 20), cut(subject, 42), lines) - ) + def nntpcheck(s): + resp, count, first, last, name = s.group(args.group) + print('Group', name, 'has', count, 'articles, range', first, 'to', last) + first = str(int(last) - args.nb_articles + 1) + resp, overviews = s.xover(first, last) + for artnum, over in overviews: + author = decode_header(over['from']).split('<', 1)[0] + subject = decode_header(over['subject']) + lines = int(over[':lines']) + print("{:7} {:20} {:42} ({})".format( + artnum, cut(author, 20), cut(subject, 42), lines)) + s = NNTP(host=args.server, port=args.port, + user=args.user, password=args.password) + nntpcheck(s) s.quit() + + if _have_ssl: + s = NNTP_SSL(host=args.ssl_server, port=args.ssl_port, + user=args.user, password=args.password, + ssl_context=args.ssl_context) + nntpcheck(s) + s.quit() + try: + s = NNTP(host=args.server, port=args.port, + user=None, password=None, usenetrc=False) + s.starttls(args.ssl_context) + s.login(args.user, args.password) + nntpcheck(s) + s.quit() + except NNTPPermanentError: + pass #STARTTLS probably not supported by server. Fail silently. Index: Lib/test/test_nntplib.py =================================================================== --- Lib/test/test_nntplib.py (revision 86110) +++ Lib/test/test_nntplib.py (working copy) @@ -3,9 +3,9 @@ import textwrap import unittest import contextlib +import nntplib from test import support -from nntplib import NNTP, GroupInfo -import nntplib +from nntplib import NNTP, NNTP_SSL, _have_ssl, GroupInfo TIMEOUT = 30 @@ -156,7 +156,35 @@ self.server.quit() self.server = None + def test_login(self): + baduser = "notarealuser" + badpw = "notarealpassword" + #Check that bogus credentials cause failure + self.assertRaises(nntplib.NNTPError, self.server.login, + user=baduser, password=badpw, usenetrc=False) + #FIXME: We should check that correct credentials succeed, but that + #would require valid details for some server somewhere to be in the + #test suite, I think. Gmane is anonymous, at least as used for the + #other tests. + + if _have_ssl: + def test_starttls(self): + file = self.server.file + sock = self.server.sock + try: + self.server.starttls() + except nntplib.NNTPPermanentError: + #Not supported by server + pass + else: + #Check that the socket and internal psuedo-file really were + #changed. + assertNotEqual(file, self.server.file) + assertNotEqual(sock, self.server.sock) + #Check that the new socket really is an SSL one + assertIsInstance(self.server.sock, ssl.SSLSocket) + class NetworkedNNTPTests(NetworkedNNTPTestsMixin, unittest.TestCase): NNTP_HOST = 'news.gmane.org' GROUP_NAME = 'gmane.comp.python.devel' @@ -165,7 +193,8 @@ def setUp(self): support.requires("network") with support.transient_internet(self.NNTP_HOST): - self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) + self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT, + usenetrc=False) def tearDown(self): if self.server is not None: @@ -188,6 +217,16 @@ resp, caps = self.server.capabilities() _check_caps(caps) +if _have_ssl: + class NetworkedNNTP_SSLTests(NetworkedNNTPTests): + NNTP_HOST = 'snews.gmane.org' + def setUp(self): + support.requires("network") + with support.transient_internet(self.NNTP_HOST): + self.server = NNTP_SSL(self.NNTP_HOST, timeout=TIMEOUT, + usenetrc=False) + #disabled as the connection will already be encrypted. + test_starttls = None # # Non-networked tests using a local server (or something mocking it). @@ -255,7 +294,7 @@ # Using BufferedRWPair instead of BufferedRandom ensures the file # isn't seekable. file = io.BufferedRWPair(self.sio, self.sio) - kwargs.setdefault('usenetrc', False) + #kwargs.setdefault('usenetrc', False) self.server = nntplib._NNTPBase(file, 'test.server', *args, **kwargs) return self.server @@ -912,8 +951,8 @@ self.server.ihave("", self.sample_post) self.assertEqual(cm.exception.response, "435 Article not wanted") + - class NNTPv1Tests(NNTPv1v2TestsMixin, MockedNNTPTestsMixin, unittest.TestCase): """Tests an NNTP v1 server (no capabilities).""" @@ -1084,9 +1123,10 @@ def test_main(): - support.run_unittest(MiscTests, NNTPv1Tests, NNTPv2Tests, - NetworkedNNTPTests - ) + tests = [MiscTests, NNTPv1Tests, NNTPv2Tests, NetworkedNNTPTests] + if _have_ssl: + tests.append(NetworkedNNTP_SSLTests) + support.run_unittest(*tests) if __name__ == "__main__":