diff -r ad7e1f97752f Lib/nntplib.py --- a/Lib/nntplib.py Mon Sep 13 20:15:33 2010 +0200 +++ b/Lib/nntplib.py Mon Sep 13 21:12:02 2010 +0200 @@ -1,4 +1,5 @@ -"""An NNTP client class based on RFC 977: Network News Transfer Protocol. +"""An NNTP client class based on RFC 977: Network News Transfer Protocol +and RFC 3977: Network News Transfer Protocol. Example: @@ -31,11 +32,14 @@ are strings, not numbers, since they are # Imports import re import socket +import collections -__all__ = ["NNTP","NNTPReplyError","NNTPTemporaryError", - "NNTPPermanentError","NNTPProtocolError","NNTPDataError", - "error_reply","error_temp","error_perm","error_proto", - "error_data",] +from socket import _GLOBAL_DEFAULT_TIMEOUT + +__all__ = ["NNTP", + "NNTPReplyError", "NNTPTemporaryError", "NNTPPermanentError", + "NNTPProtocolError", "NNTPDataError", + ] # Exceptions raised when an error or invalid response is received class NNTPError(Exception): @@ -67,32 +71,36 @@ class NNTPDataError(NNTPError): """Error in response data""" pass -# for backwards compatibility -error_reply = NNTPReplyError -error_temp = NNTPTemporaryError -error_perm = NNTPPermanentError -error_proto = NNTPProtocolError -error_data = NNTPDataError - - # Standard port used by NNTP servers NNTP_PORT = 119 # Response numbers that are followed by additional text (e.g. article) -LONGRESP = [b'100', b'215', b'220', b'221', b'222', b'224', b'230', b'231', b'282'] +LONGRESP = {'100', '215', '220', '221', '222', '224', '230', '231', '282'} # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) CRLF = b'\r\n' +GroupInfo = collections.namedtuple('GroupInfo', + ['group', 'last', 'first', 'flag']) +ArticleInfo = collections.namedtuple('ArticleInfo', + ['resp', 'nr', 'id', 'lines']) + +XoverInfo = collections.namedtuple('XoverInfo', + ['articleno', 'subject', 'poster', 'date', + 'id', 'references', 'size', 'lines']) # The class itself class NNTP: + encoding = 'utf-8' + errors = 'surrogateescape' + def __init__(self, host, port=NNTP_PORT, user=None, password=None, - readermode=None, usenetrc=True): + readermode=None, usenetrc=True, + timeout=_GLOBAL_DEFAULT_TIMEOUT): """Initialize an instance. Arguments: - host: hostname to connect to - port: port to connect to (default the standard NNTP port) @@ -100,6 +108,9 @@ class NNTP: - password: password to use with username - readermode: if true, send 'mode reader' command after connecting. + - usenetrc: allow loading username and password from ~/.netrc file + if not specified explicitly + - timeout: timeout (in seconds) used for socket connections readermode is sometimes necessary if you are connecting to an NNTP server on the local machine and intend to call @@ -109,7 +120,7 @@ class NNTP: """ self.host = host self.port = port - self.sock = socket.create_connection((host, port)) + self.sock = socket.create_connection((host, port), timeout) self.file = self.sock.makefile('rb') self.debugging = 0 self.welcome = self.getresp() @@ -127,7 +138,7 @@ class NNTP: # error 500, probably 'not implemented' pass except NNTPTemporaryError as e: - if user and e.response.startswith(b'480'): + if user and e.response.startswith('480'): # Need authorization before 'mode reader' readermode_afterauth = 1 else: @@ -147,13 +158,13 @@ class NNTP: # Perform NNRP authentication if needed. if user: resp = self.shortcmd('authinfo user '+user) - if resp.startswith(b'381'): + if resp.startswith('381'): if not password: raise NNTPReplyError(resp) else: resp = self.shortcmd( 'authinfo pass '+password) - if not resp.startswith(b'281'): + if not resp.startswith('281'): raise NNTPPermanentError(resp) if readermode_afterauth: try: @@ -162,12 +173,6 @@ class NNTP: # error 500, probably 'not implemented' pass - - # Get the welcome message from the server - # (this is read and squirreled away by __init__()). - # If the response code is 200, posting is allowed; - # if it 201, posting is not allowed - def getwelcome(self): """Get the welcome message from the server (this is read and squirreled away by __init__()). @@ -187,20 +192,23 @@ class NNTP: debug = set_debuglevel def putline(self, line): - """Internal: send one line to the server, appending CRLF.""" + """Internal: send one line to the server, appending CRLF. + The `line` must be a bytes-like object.""" line = line + CRLF if self.debugging > 1: print('*put*', repr(line)) self.sock.sendall(line) def putcmd(self, line): - """Internal: send one command to the server (through putline()).""" + """Internal: send one command to the server (through putline()). + The `line` must be an unicode string.""" if self.debugging: print('*cmd*', repr(line)) - line = bytes(line, "ASCII") + line = line.encode(self.encoding, self.errors) self.putline(line) def getline(self): """Internal: return one line from the server, stripping CRLF. - Raise EOFError if the connection is closed.""" + Raise EOFError if the connection is closed. + Returns a bytes object.""" line = self.file.readline() if self.debugging > 1: print('*get*', repr(line)) @@ -213,27 +221,34 @@ class NNTP: def getresp(self): """Internal: get a response from the server. - Raise various errors if the response indicates an error.""" + Raise various errors if the response indicates an error. + Returns an unicode string.""" resp = self.getline() if self.debugging: print('*resp*', repr(resp)) + resp = resp.decode(self.encoding, self.errors) c = resp[:1] - if c == b'4': + if c == '4': raise NNTPTemporaryError(resp) - if c == b'5': + if c == '5': raise NNTPPermanentError(resp) - if c not in b'123': + if c not in '123': raise NNTPProtocolError(resp) return resp def getlongresp(self, file=None): """Internal: get a response plus following text from the server. - Raise various errors if the response indicates an error.""" + Raise various errors if the response indicates an error. + + Returns a (response, lines) tuple where `response` is an unicode + string and `lines` is a list of bytes objects. + If `file` is a file-like object, it must be open in binary mode. + """ openedFile = None try: # If a string was passed then open a file with that name - if isinstance(file, str): - openedFile = file = open(file, "w") + if isinstance(file, (str, bytes)): + openedFile = file = open(file, "wb") resp = self.getresp() if resp[:3] not in LONGRESP: @@ -257,24 +272,41 @@ class NNTP: return resp, list def shortcmd(self, line): - """Internal: send a command and get the response.""" + """Internal: send a command and get the response. + Same return value as getresp().""" self.putcmd(line) return self.getresp() def longcmd(self, line, file=None): - """Internal: send a command and get the response plus following text.""" + """Internal: send a command and get the response plus following text. + Same return value as getlongresp().""" self.putcmd(line) return self.getlongresp(file) + def _longcmdstring(self, line, file=None): + """Internal: send a command and get the response plus following text. + Same as longcmd() and getlongresp(), except that the returned `lines` + are unicode strings rather than bytes objects. + """ + self.putcmd(line) + resp, list = self.getlongresp(file) + return resp, (line.decode(self.encoding, self.errors) + for line in list) + + def _grouplist(self, lines): + # Parse lines into "group last first flag" + return [GroupInfo(*line.split()) for line in lines] + def newgroups(self, date, time, file=None): """Process a NEWGROUPS command. Arguments: - date: string 'yymmdd' indicating the date - time: string 'hhmmss' indicating the time Return: - resp: server response if successful - - list: list of newsgroup names""" - - return self.longcmd('NEWGROUPS ' + date + ' ' + time, file) + - list: list of newsgroup names + """ + resp, list = self._longcmdstring('NEWGROUPS ' + date + ' ' + time, file) + return resp, self._grouplist(list) def newnews(self, group, date, time, file=None): """Process a NEWNEWS command. Arguments: @@ -283,24 +315,22 @@ class NNTP: - time: string 'hhmmss' indicating the time Return: - resp: server response if successful - - list: list of message ids""" - + - list: list of message ids + """ cmd = 'NEWNEWS ' + group + ' ' + date + ' ' + time - return self.longcmd(cmd, file) + return self._longcmdstring(cmd, file) def list(self, file=None): - """Process a LIST command. Return: + """Process a LIST command. Argument: + - file: Filename string or file object to store the result in + Returns: - resp: server response if successful - - list: list of (group, last, first, flag) (strings)""" - - resp, list = self.longcmd('LIST', file) - for i in range(len(list)): - # Parse lines into "group last first flag" - list[i] = tuple(list[i].split()) - return resp, list + - list: list of (group, last, first, flag) (strings) + """ + resp, list = self._longcmdstring('LIST', file) + return resp, self._grouplist(list) def description(self, group): - """Get a description for a single group. If more than one group matches ('group' is a pattern), return the first. If no group matches, return an empty string. @@ -311,23 +341,22 @@ class NNTP: NOTE: This neither checks for a wildcard in 'group' nor does it check whether the group actually exists.""" - resp, lines = self.descriptions(group) if len(lines) == 0: - return b'' + return '' else: return lines[0][1] def descriptions(self, group_pattern): """Get descriptions for a range of groups.""" - line_pat = re.compile(b'^(?P[^ \t]+)[ \t]+(.*)$') + line_pat = re.compile('^(?P[^ \t]+)[ \t]+(.*)$') # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first - resp, raw_lines = self.longcmd('LIST NEWSGROUPS ' + group_pattern) - if not resp.startswith(b'215'): + resp, raw_lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) + if not resp.startswith('215'): # Now the deprecated XGTITLE. This either raises an error # or succeeds with the same output structure as LIST # NEWSGROUPS. - resp, raw_lines = self.longcmd('XGTITLE ' + group_pattern) + resp, raw_lines = self._longcmdstring('XGTITLE ' + group_pattern) lines = [] for raw_line in raw_lines: match = line_pat.search(raw_line.strip()) @@ -343,10 +372,10 @@ class NNTP: - count: number of articles (string) - first: first article number (string) - last: last article number (string) - - name: the group name""" - + - name: the group name + """ resp = self.shortcmd('GROUP ' + name) - if not resp.startswith(b'211'): + if not resp.startswith('211'): raise NNTPReplyError(resp) words = resp.split() count = first = last = 0 @@ -362,19 +391,22 @@ class NNTP: return resp, count, first, last, name def help(self, file=None): - """Process a HELP command. Returns: + """Process a HELP command. Argument: + - file: Filename string or file object to store the result in + Returns: - resp: server response if successful - - list: list of strings""" - - return self.longcmd('HELP',file) + - list: list of strings returned by the server in response to the + HELP command + """ + return self._longcmdstring('HELP',file) def statparse(self, resp): """Internal: parse the response of a STAT, NEXT or LAST command.""" - if not resp.startswith(b'22'): + if not resp.startswith('22'): raise NNTPReplyError(resp) words = resp.split() nr = 0 - id = b'' + id = '' n = len(words) if n > 1: nr = words[1] @@ -409,7 +441,7 @@ class NNTP: """Internal: process a HEAD, BODY or ARTICLE command.""" resp, list = self.longcmd(line, file) resp, nr, id = self.statparse(resp) - return resp, nr, id, list + return ArticleInfo(resp, nr, id, list) def head(self, id): """Process a HEAD command. Argument: @@ -431,8 +463,8 @@ class NNTP: - nr: article number - id: message id - list: the lines of the article's body or an empty list - if file was used""" - + if file was used + """ return self.artcmd('BODY {0}'.format(id), file) def article(self, id): @@ -442,58 +474,59 @@ class NNTP: - resp: server response if successful - nr: article number - id: message id - - list: the lines of the article""" - + - list: the lines of the article + """ return self.artcmd('ARTICLE {0}'.format(id)) def slave(self): """Process a SLAVE command. Returns: - - resp: server response if successful""" - + - resp: server response if successful + """ return self.shortcmd('SLAVE') def xhdr(self, hdr, str, file=None): """Process an XHDR command (optional server extension). Arguments: - hdr: the header type (e.g. 'subject') - str: an article nr, a message id, or a range nr1-nr2 + - file: Filename string or file object to store the result in Returns: - resp: server response if successful - - list: list of (nr, value) strings""" - - pat = re.compile(b'^([0-9]+) ?(.*)\n?') - resp, lines = self.longcmd('XHDR {0} {1}'.format(hdr, str), file) - for i in range(len(lines)): - line = lines[i] + - list: list of (nr, value) strings + """ + pat = re.compile('^([0-9]+) ?(.*)\n?') + resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) + def remove_number(line): m = pat.match(line) - if m: - lines[i] = m.group(1, 2) - return resp, lines + return m.group(1, 2) if m else line + return resp, [remove_number(line) for line in lines] def xover(self, start, end, file=None): """Process an XOVER command (optional server extension) Arguments: - start: start of range - end: end of range + - file: Filename string or file object to store the result in Returns: - resp: server response if successful - - list: list of (art-nr, subject, poster, date, - id, references, size, lines)""" - - resp, lines = self.longcmd('XOVER {0}-{1}'.format(start, end), file) + - list: list of XoverInfo named tuples, with elements (art-nr, subject, + poster, date, id, references, size, lines) + """ + resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), + file) xover_lines = [] for line in lines: - elem = line.split(b'\t') + elem = line.split('\t') try: - xover_lines.append((elem[0], - elem[1], - elem[2], - elem[3], - elem[4], - elem[5].split(), - elem[6], - elem[7])) + xover_lines.append(XoverInfo(elem[0], + elem[1], + elem[2], + elem[3], + elem[4], + elem[5].split(), + elem[6], + elem[7])) except IndexError: raise NNTPDataError(line) - return resp,xover_lines + return resp, xover_lines def xgtitle(self, group, file=None): """Process an XGTITLE command (optional server extension) Arguments: @@ -501,9 +534,11 @@ class NNTP: Returns: - resp: server response if successful - list: list of (name,title) strings""" - - line_pat = re.compile(b'^([^ \t]+)[ \t]+(.*)$') - resp, raw_lines = self.longcmd('XGTITLE ' + group, file) + from warnings import warn + warn("The XGTITLE extension is not actively used, use the description()" + "function instead", PendingDeprecationWarning, 2) + line_pat = re.compile('^([^ \t]+)[ \t]+(.*)$') + resp, raw_lines = self._longcmdstring('XGTITLE ' + group, file) lines = [] for raw_line in raw_lines: match = line_pat.search(raw_line.strip()) @@ -511,15 +546,19 @@ class NNTP: lines.append(match.group(1, 2)) return resp, lines - def xpath(self,id): + def xpath(self, id): """Process an XPATH command (optional server extension) Arguments: - id: Message id of article Returns: resp: server response if successful - path: directory path to article""" + path: directory path to article + """ + from warnings import warn + warn("The XPATH extension is not actively used", + PendingDeprecationWarning, 2) resp = self.shortcmd('XPATH {0}'.format(id)) - if not resp.startswith(b'223'): + if not resp.startswith('223'): raise NNTPReplyError(resp) try: [resp_num, path] = resp.split() @@ -528,16 +567,15 @@ class NNTP: else: return resp, path - def date (self): - """Process the DATE command. Arguments: - None + def date(self): + """Process the DATE command. Returns: resp: server response if successful date: Date suitable for newnews/newgroups commands etc. - time: Time suitable for newnews/newgroups commands etc.""" - + time: Time suitable for newnews/newgroups commands etc. + """ resp = self.shortcmd("DATE") - if not resp.startswith(b'111'): + if not resp.startswith('111'): raise NNTPReplyError(resp) elem = resp.split() if len(elem) != 2: @@ -550,8 +588,8 @@ class NNTP: def _post(self, command, f): resp = self.shortcmd(command) - # Raises error_??? if posting is not allowed - if not resp.startswith(b'3'): + # Raises a specific exception if posting is not allowed + if not resp.startswith('3'): raise NNTPReplyError(resp) while 1: line = f.readline() diff -r ad7e1f97752f Lib/test/test_nntplib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_nntplib.py Mon Sep 13 21:12:02 2010 +0200 @@ -0,0 +1,97 @@ +import datetime +import unittest +from test import support +from nntplib import NNTP, GroupInfo, XoverInfo + +TIMEOUT = 60 + +class BasicNNTPTests: + + def setUp(self): + self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT) + + def tearDown(self): + if self.server is not None: + self.server.quit() + + def test_welcome(self): + welcome = self.server.getwelcome() + self.assertEqual(str, type(welcome)) + + def test_help(self): + resp, list = self.server.help() + for line in list: + self.assertEqual(str, type(line)) + + def test_list(self): + resp, list = self.server.list() + if len(list) > 0: + self.assertEqual(GroupInfo, type(list[0])) + self.assertEqual(str, type(list[0].group)) + + def test_newgroups(self): + # gmane gets a constant influx of new groups. In order not to stress + # the server too much, we choose a recent date in the past. + dt = datetime.date.today() - datetime.timedelta(days=7) + dt = "%02d%02d%02d" % (dt.year % 100, dt.month, dt.day) + resp, list = self.server.newgroups(dt, "000000") + if len(list) > 0: + self.assertEqual(GroupInfo, type(list[0])) + self.assertEqual(str, type(list[0].group)) + + def test_description(self): + descr = self.server.description(self.GROUP_NAME) + self.assertEqual(str, type(descr)) + + def test_group(self): + result = self.server.group(self.GROUP_NAME) + self.assertEqual(5, len(result)) + self.assertEqual(str, type(result[4])) + + def test_date(self): + result = self.server.date() + self.assertEqual(3, len(result)) + self.assertEqual(str, type(result[1])) + + def test_head(self): + resp, count, first, last, name = self.server.group(self.GROUP_NAME) + result = self.server.head(last) + for line in result.lines: + self.assertEqual(bytes, type(line)) + + def test_xover(self): + resp, count, first, last, name = self.server.group(self.GROUP_NAME) + resp, lines = self.server.xover(last, last) + self.assertEqual(XoverInfo, type(lines[0])) + self.assertEqual(str, type(lines[0].subject)) + + def test_xhdr(self): + resp, count, first, last, name = self.server.group(self.GROUP_NAME) + resp, lines = self.server.xhdr('subject', last) + for line in lines: + self.assertEqual(str, type(line[1])) + + def test_quit(self): + self.server.quit() + self.server = None + + +class NetworkedNNTPTests(BasicNNTPTests, unittest.TestCase): + NNTP_HOST = 'news.gmane.org' + GROUP_NAME = 'gmane.comp.python.devel' + + def setUp(self): + support.requires("network") + with support.transient_internet(self.NNTP_HOST): + BasicNNTPTests.setUp(self) + + # Disabled with gmane as it produces too much data + test_list = None + + +def test_main(): + support.run_unittest(NetworkedNNTPTests) + + +if __name__ == "__main__": + test_main()