# HG changeset patch # Parent 1756beed417ce1d44f05b973b2af9d2c99bbb4ad Issue #28971: Override _MAXLINE when reading OVER responses diff -r 1756beed417c Lib/nntplib.py --- a/Lib/nntplib.py Sat Dec 17 09:19:11 2016 +0100 +++ b/Lib/nntplib.py Sat Dec 24 06:52:36 2016 +0000 @@ -86,11 +86,15 @@ ] # maximal line length when calling readline(). This is to prevent -# reading arbitrary length lines. RFC 3977 limits NNTP line length to -# 512 characters, including CRLF. We have selected 2048 just to be on -# the safe side. +# reading lines of arbitrary length. RFC 3977 limits the length of the +# initial response line for standard commands to 512 bytes, including CRLF. +# We have selected 2048 just to be on the safe side, and because the RFC +# limit does not apply to multi-line data blocks. _MAXLINE = 2048 +# Enough for hundreds of message id's, each being hundreds of bytes +_MAX_OVER_LINE = 64000 + # Exceptions raised when an error or invalid response is received class NNTPError(Exception): @@ -425,12 +429,12 @@ line = line.encode(self.encoding, self.errors) self._putline(line) - def _getline(self, strip_crlf=True): + def _getline(self, strip_crlf=True, *, maxline=_MAXLINE): """Internal: return one line from the server, stripping _CRLF. Raise EOFError if the connection is closed. Returns a bytes object.""" - line = self.file.readline(_MAXLINE +1) - if len(line) > _MAXLINE: + line = self.file.readline(maxline + 1) + if len(line) > maxline: raise NNTPDataError('line too long') if self.debugging > 1: print('*get*', repr(line)) @@ -458,7 +462,7 @@ raise NNTPProtocolError(resp) return resp - def _getlongresp(self, file=None): + def _getlongresp(self, file=None, **kw): """Internal: get a response plus following text from the server. Raise various errors if the response indicates an error. @@ -482,7 +486,7 @@ # XXX lines = None instead? terminators = (b'.' + _CRLF, b'.\n') while 1: - line = self._getline(False) + line = self._getline(False, **kw) if line in terminators: break if line.startswith(b'..'): @@ -491,7 +495,7 @@ else: terminator = b'.' while 1: - line = self._getline() + line = self._getline(**kw) if line == terminator: break if line.startswith(b'..'): @@ -516,13 +520,13 @@ self._putcmd(line) return self._getlongresp(file) - def _longcmdstring(self, line, file=None): + def _longcmdstring(self, line, file=None, **kw): """Internal: send a command and get the response plus following text. Same as _longcmd() and _getlongresp(), except that the returned `lines` are unicode strings rather than bytes objects. """ self._putcmd(line) - resp, list = self._getlongresp(file) + resp, list = self._getlongresp(file, **kw) return resp, [line.decode(self.encoding, self.errors) for line in list] @@ -801,7 +805,7 @@ - list: list of dicts containing the response fields """ resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), - file) + file, maxline=_MAX_OVER_LINE) fmt = self._getoverviewfmt() return resp, _parse_overview(lines, fmt) @@ -828,7 +832,7 @@ cmd += ' {0}-{1}'.format(start, end or '') elif message_spec is not None: cmd = cmd + ' ' + message_spec - resp, lines = self._longcmdstring(cmd, file) + resp, lines = self._longcmdstring(cmd, file, maxline=_MAX_OVER_LINE) fmt = self._getoverviewfmt() return resp, _parse_overview(lines, fmt) diff -r 1756beed417c Lib/test/test_nntplib.py --- a/Lib/test/test_nntplib.py Sat Dec 17 09:19:11 2016 +0100 +++ b/Lib/test/test_nntplib.py Sat Dec 24 06:52:36 2016 +0000 @@ -132,8 +132,6 @@ self.assertLessEqual(art_num, last) self._check_art_dict(art_dict) - @unittest.skipIf(True, 'temporarily skipped until a permanent solution' - ' is found for issue #28971') def test_over(self): resp, count, first, last, name = self.server.group(self.GROUP_NAME) start = last - 10 @@ -620,6 +618,20 @@ "\tXref: saria.nerim.net fr.comp.lang.python:1265" "\n" ".\n") + elif message_spec == "1000-1000": + ref = "" + self.push_lit( + "224 Overview information follows\n" + "1000\t" + "Re: Dummy Subject\t" + "Author \t" + "Sat, 24 Dec 2016 14:19:29 +1100\t" + "\t" + + " ".join(ref for i in range(100)) + "\t" + "100\t" + "3\t" + "Xref: saria.nerim.net fr.comp.lang.python:1000\r\n" + ".\r\n") else: self.push_lit("""\ 224 No articles @@ -1105,6 +1117,28 @@ resp, overviews = self.server.over((57, 59)) self.check_over_xover_resp(resp, overviews) + def test_over_large(self): + response = self.server.group('fr.comp.lang.python') + [response, count, first, last, name] = response + self.assertLessEqual(first, 1000) + message_spec = (1000, 1000) + for [method, args] in ( + (self.server.over, (message_spec,)), + (self.server.xover, message_spec), + ): + [response, overviews] = method(*args) + [[article, overview]] = overviews + self.assertEqual(article, 1000) + references = overview['references'] + self.assertGreater(len(references), nntplib._MAXLINE) + expected = "" + references = references.split() + self.assertEqual(len(references), 100) + for ref in references: + self.assertEqual(ref, expected) + expected = "saria.nerim.net fr.comp.lang.python:1000" + self.assertEqual(overview["xref"], expected) + sample_post = ( b'From: "Demo User" \r\n' b'Subject: I am just a test article\r\n'