# HG changeset patch # Parent 673191aa172421c8f47b7bc37e5471ed7d1731dc diff -r 673191aa1724 Doc/library/http.client.rst --- a/Doc/library/http.client.rst Sun Feb 01 22:53:41 2015 -0800 +++ b/Doc/library/http.client.rst Mon Feb 02 11:31:49 2015 +0000 @@ -93,7 +93,7 @@ parameter. -.. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) +.. class:: HTTPResponse(fp, debuglevel=0, method=None, url=None) Class whose instances are returned upon successful connection. Not instantiated directly by user. diff -r 673191aa1724 Lib/http/client.py --- a/Lib/http/client.py Sun Feb 01 22:53:41 2015 -0800 +++ b/Lib/http/client.py Mon Feb 02 11:31:49 2015 +0000 @@ -171,7 +171,7 @@ # text following RFC 2047. The basic status line parsing only # accepts iso-8859-1. - def __init__(self, sock, debuglevel=0, method=None, url=None): + def __init__(self, fp, debuglevel=0, method=None, url=None): # If the response includes a content-length header, we need to # make sure that the client doesn't read more than the # specified number of bytes. If it does, it will block until @@ -179,7 +179,7 @@ # happen if a self.fp.read() is done (without a size) whether # self.fp is buffered or not. So, no self.fp.read() by # clients unless they know what they are doing. - self.fp = sock.makefile("rb") + self.fp = fp self.debuglevel = debuglevel self._method = method @@ -199,7 +199,7 @@ self.chunked = _UNKNOWN # is "chunked" being used? self.chunk_left = _UNKNOWN # bytes left to read in current chunk self.length = _UNKNOWN # number of bytes left in response - self.will_close = _UNKNOWN # conn will close at end of response + self.will_close = _UNKNOWN # fp not shared with HTTPConnection def _read_status(self): line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") @@ -346,7 +346,8 @@ def _close_conn(self): fp = self.fp self.fp = None - fp.close() + if self.will_close: + fp.close() def close(self): super().close() # set "closed" flag @@ -358,11 +359,6 @@ # XXX This class should probably be revised to act more like # the "raw stream" that BufferedReader expects. - def flush(self): - super().flush() - if self.fp: - self.fp.flush() - def readable(self): return True @@ -688,7 +684,8 @@ self.source_address = source_address self.sock = None self._buffer = [] - self.__response = None + self._reader = None + self.__response = None # Response object when connection is shared self.__state = _CS_IDLE self._method = None self._tunnel_host = None @@ -759,7 +756,8 @@ self.send(header_bytes) self.send(b'\r\n') - response = self.response_class(self.sock, method=self._method) + self._reader = self.sock.makefile("rb") + response = self.response_class(self._reader, method=self._method) (version, code, message) = response._read_status() if code != http.HTTPStatus.OK: @@ -787,14 +785,25 @@ def close(self): """Close the connection to the HTTP server.""" + if self._reader: + self._reader.close() + self._reader = None if self.sock: - self.sock.close() # close it manually... there may be other refs + self.sock.close() self.sock = None if self.__response: self.__response.close() self.__response = None self.__state = _CS_IDLE + def _disown(self): + """Pass ownership of the connection to the response object.""" + if self.__response: + self.__response.will_close = True + self.__response = None + self._reader = None + self.close() + def send(self, data): """Send `data' to the server. ``data`` can be a string object, a bytes object, an array object, a @@ -879,8 +888,8 @@ self.__response = None - # in certain cases, we cannot issue another request on this connection. - # this occurs when: + # In certain cases, we cannot issue another request on a previously- + # used connection. This occurs when: # 1) we are in the process of sending a request. (_CS_REQ_STARTED) # 2) a response to a previous request has signalled that it is going # to close the connection upon completion. @@ -1074,8 +1083,8 @@ If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then - it will be closed before the response is returned. When the - connection is closed, the underlying socket is closed. + the HTTPConnection object will detach itself from this connection, + and a subsequent request will trigger a new connection. """ # if a prior response has been completed, then forget about it. @@ -1100,11 +1109,13 @@ if self.__state != _CS_REQ_SENT or self.__response: raise ResponseNotReady(self.__state) + if self._reader is None: + self._reader = self.sock.makefile("rb") if self.debuglevel > 0: - response = self.response_class(self.sock, self.debuglevel, + response = self.response_class(self._reader, self.debuglevel, method=self._method) else: - response = self.response_class(self.sock, method=self._method) + response = self.response_class(self._reader, method=self._method) try: response.begin() @@ -1112,8 +1123,7 @@ self.__state = _CS_IDLE if response.will_close: - # this effectively passes the connection to the response - self.close() + self._disown() else: # remember this, so we can tell when it is complete self.__response = response diff -r 673191aa1724 Lib/test/test_httplib.py --- a/Lib/test/test_httplib.py Sun Feb 01 22:53:41 2015 -0800 +++ b/Lib/test/test_httplib.py Mon Feb 02 11:31:49 2015 +0000 @@ -4,6 +4,7 @@ import os import array import socket +from threading import Thread import unittest TestCase = unittest.TestCase @@ -20,23 +21,23 @@ # constants for testing chunked encoding chunked_start = ( - 'HTTP/1.1 200 OK\r\n' - 'Transfer-Encoding: chunked\r\n\r\n' - 'a\r\n' - 'hello worl\r\n' - '3\r\n' - 'd! \r\n' - '8\r\n' - 'and now \r\n' - '22\r\n' - 'for something completely different\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'a\r\n' + b'hello worl\r\n' + b'3\r\n' + b'd! \r\n' + b'8\r\n' + b'and now \r\n' + b'22\r\n' + b'for something completely different\r\n' ) chunked_expected = b'hello world! and now for something completely different' -chunk_extension = ";foo=bar" -last_chunk = "0\r\n" -last_chunk_extended = "0" + chunk_extension + "\r\n" -trailers = "X-Dummy: foo\r\nX-Dumm2: bar\r\n" -chunked_end = "\r\n" +chunk_extension = b";foo=bar" +last_chunk = b"0\r\n" +last_chunk_extended = b"0" + chunk_extension + b"\r\n" +trailers = b"X-Dummy: foo\r\nX-Dumm2: bar\r\n" +chunked_end = b"\r\n" HOST = support.HOST @@ -106,6 +107,12 @@ raise AssertionError('caller tried to read past EOF') return data +class StringHTTPResponse(client.HTTPResponse): + """HTTPResponse reading from a byte string rather than a socket""" + def __init__(self, string, *pos, **kw): + reader = io.BytesIO(string) + super().__init__(reader, *pos, **kw) + class HeaderTests(TestCase): def test_auto_headers(self): # Some headers are added automatically, but should not be added by @@ -192,9 +199,8 @@ def test_malformed_headers_coped_with(self): # Issue 19996 - body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + resp = StringHTTPResponse(body) resp.begin() self.assertEqual(resp.getheader('First'), 'val') @@ -205,9 +211,8 @@ def test_status_lines(self): # Test HTTP status lines - body = "HTTP/1.1 200 Ok\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() self.assertEqual(resp.read(0), b'') # Issue #20007 self.assertFalse(resp.isclosed()) @@ -218,9 +223,8 @@ resp.close() self.assertTrue(resp.closed) - body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 400.100 Not Ok\r\n\r\nText" + resp = StringHTTPResponse(body) self.assertRaises(client.BadStatusLine, resp.begin) def test_bad_status_repr(self): @@ -230,9 +234,8 @@ def test_partial_reads(self): # if we have a length, the system knows when to close itself # same behaviour than when we read the whole thing with read() - body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() self.assertEqual(resp.read(2), b'Te') self.assertFalse(resp.isclosed()) @@ -245,9 +248,8 @@ def test_partial_readintos(self): # if we have a length, the system knows when to close itself # same behaviour than when we read the whole thing with read() - body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() b = bytearray(2) n = resp.readinto(b) @@ -265,9 +267,8 @@ def test_partial_reads_no_content_length(self): # when no length is present, the socket should be gracefully closed when # all data was read - body = "HTTP/1.1 200 Ok\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() self.assertEqual(resp.read(2), b'Te') self.assertFalse(resp.isclosed()) @@ -281,9 +282,8 @@ def test_partial_readintos_no_content_length(self): # when no length is present, the socket should be gracefully closed when # all data was read - body = "HTTP/1.1 200 Ok\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() b = bytearray(2) n = resp.readinto(b) @@ -300,9 +300,8 @@ def test_partial_reads_incomplete_body(self): # if the server shuts down the connection before the whole # content-length is delivered, the socket is gracefully closed - body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() self.assertEqual(resp.read(2), b'Te') self.assertFalse(resp.isclosed()) @@ -313,9 +312,8 @@ def test_partial_readintos_incomplete_body(self): # if the server shuts down the connection before the whole # content-length is delivered, the socket is gracefully closed - body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + resp = StringHTTPResponse(body) resp.begin() b = bytearray(2) n = resp.readinto(b) @@ -351,18 +349,17 @@ def test_response_headers(self): # test response with multiple message headers with the same field name. - text = ('HTTP/1.1 200 OK\r\n' - 'Set-Cookie: Customer="WILE_E_COYOTE"; ' - 'Version="1"; Path="/acme"\r\n' - 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' - ' Path="/acme"\r\n' - '\r\n' - 'No body\r\n') + text = (b'HTTP/1.1 200 OK\r\n' + b'Set-Cookie: Customer="WILE_E_COYOTE"; ' + b'Version="1"; Path="/acme"\r\n' + b'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + b' Path="/acme"\r\n' + b'\r\n' + b'No body\r\n') hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' ', ' 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') - s = FakeSocket(text) - r = client.HTTPResponse(s) + r = StringHTTPResponse(text) r.begin() cookies = r.getheader("Set-Cookie") self.assertEqual(cookies, hdr) @@ -370,12 +367,11 @@ def test_read_head(self): # Test that the library doesn't attempt to read any data # from a HEAD request. (Tickles SF bug #622042.) - sock = FakeSocket( - 'HTTP/1.1 200 OK\r\n' - 'Content-Length: 14432\r\n' - '\r\n', - NoEOFBytesIO) - resp = client.HTTPResponse(sock, method="HEAD") + reader = NoEOFBytesIO( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 14432\r\n' + b'\r\n') + resp = client.HTTPResponse(reader, method="HEAD") resp.begin() if resp.read(): self.fail("Did not expect response from HEAD request") @@ -383,12 +379,11 @@ def test_readinto_head(self): # Test that the library doesn't attempt to read any data # from a HEAD request. (Tickles SF bug #622042.) - sock = FakeSocket( - 'HTTP/1.1 200 OK\r\n' - 'Content-Length: 14432\r\n' - '\r\n', - NoEOFBytesIO) - resp = client.HTTPResponse(sock, method="HEAD") + reader = NoEOFBytesIO( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 14432\r\n' + b'\r\n') + resp = client.HTTPResponse(reader, method="HEAD") resp.begin() b = bytearray(5) if resp.readinto(b) != 0: @@ -396,11 +391,10 @@ self.assertEqual(bytes(b), b'\x00'*5) def test_too_many_headers(self): - headers = '\r\n'.join('Header%d: foo' % i - for i in range(client._MAXHEADERS + 1)) + '\r\n' - text = ('HTTP/1.1 200 OK\r\n' + headers) - s = FakeSocket(text) - r = client.HTTPResponse(s) + headers = b'\r\n'.join(('Header%d: foo' % i).encode('ascii') + for i in range(client._MAXHEADERS + 1)) + b'\r\n' + text = (b'HTTP/1.1 200 OK\r\n' + headers) + r = StringHTTPResponse(text) self.assertRaisesRegex(client.HTTPException, r"got more than \d+ headers", r.begin) @@ -476,23 +470,23 @@ def test_chunked(self): expected = chunked_expected - sock = FakeSocket(chunked_start + last_chunk + chunked_end) - resp = client.HTTPResponse(sock, method="GET") + input = chunked_start + last_chunk + chunked_end + resp = StringHTTPResponse(input, method="GET") resp.begin() self.assertEqual(resp.read(), expected) resp.close() # Various read sizes for n in range(1, 12): - sock = FakeSocket(chunked_start + last_chunk + chunked_end) - resp = client.HTTPResponse(sock, method="GET") + input = chunked_start + last_chunk + chunked_end + resp = StringHTTPResponse(input, method="GET") resp.begin() self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected) resp.close() - for x in ('', 'foo\r\n'): - sock = FakeSocket(chunked_start + x) - resp = client.HTTPResponse(sock, method="GET") + for x in (b'', b'foo\r\n'): + input = chunked_start + x + resp = StringHTTPResponse(input, method="GET") resp.begin() try: resp.read() @@ -512,8 +506,8 @@ nexpected = len(expected) b = bytearray(128) - sock = FakeSocket(chunked_start + last_chunk + chunked_end) - resp = client.HTTPResponse(sock, method="GET") + input = chunked_start + last_chunk + chunked_end + resp = StringHTTPResponse(input, method="GET") resp.begin() n = resp.readinto(b) self.assertEqual(b[:nexpected], expected) @@ -522,8 +516,8 @@ # Various read sizes for n in range(1, 12): - sock = FakeSocket(chunked_start + last_chunk + chunked_end) - resp = client.HTTPResponse(sock, method="GET") + input = chunked_start + last_chunk + chunked_end + resp = StringHTTPResponse(input, method="GET") resp.begin() m = memoryview(b) i = resp.readinto(m[0:n]) @@ -533,9 +527,9 @@ self.assertEqual(i, nexpected) resp.close() - for x in ('', 'foo\r\n'): - sock = FakeSocket(chunked_start + x) - resp = client.HTTPResponse(sock, method="GET") + for x in (b'', b'foo\r\n'): + input = chunked_start + x + resp = StringHTTPResponse(input, method="GET") resp.begin() try: n = resp.readinto(b) @@ -551,15 +545,15 @@ def test_chunked_head(self): chunked_start = ( - 'HTTP/1.1 200 OK\r\n' - 'Transfer-Encoding: chunked\r\n\r\n' - 'a\r\n' - 'hello world\r\n' - '1\r\n' - 'd\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'a\r\n' + b'hello world\r\n' + b'1\r\n' + b'd\r\n' ) - sock = FakeSocket(chunked_start + last_chunk + chunked_end) - resp = client.HTTPResponse(sock, method="HEAD") + input = chunked_start + last_chunk + chunked_end + resp = StringHTTPResponse(input, method="HEAD") resp.begin() self.assertEqual(resp.read(), b'') self.assertEqual(resp.status, 200) @@ -571,15 +565,15 @@ def test_readinto_chunked_head(self): chunked_start = ( - 'HTTP/1.1 200 OK\r\n' - 'Transfer-Encoding: chunked\r\n\r\n' - 'a\r\n' - 'hello world\r\n' - '1\r\n' - 'd\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'a\r\n' + b'hello world\r\n' + b'1\r\n' + b'd\r\n' ) - sock = FakeSocket(chunked_start + last_chunk + chunked_end) - resp = client.HTTPResponse(sock, method="HEAD") + input = chunked_start + last_chunk + chunked_end + resp = StringHTTPResponse(input, method="HEAD") resp.begin() b = bytearray(5) n = resp.readinto(b) @@ -593,16 +587,15 @@ self.assertTrue(resp.closed) def test_negative_content_length(self): - sock = FakeSocket( - 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n') - resp = client.HTTPResponse(sock, method="GET") + input = b'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n' + resp = StringHTTPResponse(input, method="GET") resp.begin() self.assertEqual(resp.read(), b'Hello\r\n') self.assertTrue(resp.isclosed()) def test_incomplete_read(self): - sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') - resp = client.HTTPResponse(sock, method="GET") + input = b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n' + resp = StringHTTPResponse(input, method="GET") resp.begin() try: resp.read() @@ -634,36 +627,35 @@ # Test lines overflowing the max line size (_MAXLINE in http.client) def test_overflowing_status_line(self): - body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" - resp = client.HTTPResponse(FakeSocket(body)) + body = b"HTTP/1.1 200 Ok" + b"k" * 65536 + b"\r\n" + resp = StringHTTPResponse(body) self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin) def test_overflowing_header_line(self): body = ( - 'HTTP/1.1 200 OK\r\n' - 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + b'HTTP/1.1 200 OK\r\n' + b'X-Foo: bar' + b'r' * 65536 + b'\r\n\r\n' ) - resp = client.HTTPResponse(FakeSocket(body)) + resp = StringHTTPResponse(body) self.assertRaises(client.LineTooLong, resp.begin) def test_overflowing_chunked_line(self): body = ( - 'HTTP/1.1 200 OK\r\n' - 'Transfer-Encoding: chunked\r\n\r\n' - + '0' * 65536 + 'a\r\n' - 'hello world\r\n' - '0\r\n' - '\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + + b'0' * 65536 + b'a\r\n' + b'hello world\r\n' + b'0\r\n' + b'\r\n' ) - resp = client.HTTPResponse(FakeSocket(body)) + resp = StringHTTPResponse(body) resp.begin() self.assertRaises(client.LineTooLong, resp.read) def test_early_eof(self): # Test httpresponse with no \r\n termination, - body = "HTTP/1.1 200 Ok" - sock = FakeSocket(body) - resp = client.HTTPResponse(sock) + body = b"HTTP/1.1 200 Ok" + resp = StringHTTPResponse(body) resp.begin() self.assertEqual(resp.read(), b'') self.assertTrue(resp.isclosed()) @@ -688,11 +680,11 @@ self.assertTrue(conn.sock.file_closed) def test_chunked_extension(self): - extra = '3;foo=bar\r\n' + 'abc\r\n' + extra = b'3;foo=bar\r\n' + b'abc\r\n' expected = chunked_expected + b'abc' - sock = FakeSocket(chunked_start + extra + last_chunk_extended + chunked_end) - resp = client.HTTPResponse(sock, method="GET") + input = chunked_start + extra + last_chunk_extended + chunked_end + resp = StringHTTPResponse(input, method="GET") resp.begin() self.assertEqual(resp.read(), expected) resp.close() @@ -700,8 +692,8 @@ def test_chunked_missing_end(self): """some servers may serve up a short chunked encoding stream""" expected = chunked_expected - sock = FakeSocket(chunked_start + last_chunk) #no terminating crlf - resp = client.HTTPResponse(sock, method="GET") + input = chunked_start + last_chunk #no terminating crlf + resp = StringHTTPResponse(input, method="GET") resp.begin() self.assertEqual(resp.read(), expected) resp.close() @@ -709,36 +701,36 @@ def test_chunked_trailers(self): """See that trailers are read and ignored""" expected = chunked_expected - sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end) - resp = client.HTTPResponse(sock, method="GET") + reader = io.BytesIO(chunked_start + last_chunk + trailers + chunked_end) + resp = client.HTTPResponse(reader, method="GET") resp.begin() self.assertEqual(resp.read(), expected) # we should have reached the end of the file - self.assertEqual(sock.file.read(100), b"") #we read to the end + self.assertEqual(reader.read(100), b"") #we read to the end resp.close() def test_chunked_sync(self): """Check that we don't read past the end of the chunked-encoding stream""" expected = chunked_expected - extradata = "extradata" - sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata) - resp = client.HTTPResponse(sock, method="GET") + extradata = b"extradata" + reader = io.BytesIO(chunked_start + last_chunk + trailers + chunked_end + extradata) + resp = client.HTTPResponse(reader, method="GET") resp.begin() self.assertEqual(resp.read(), expected) # the file should now have our extradata ready to be read - self.assertEqual(sock.file.read(100), extradata.encode("ascii")) #we read to the end + self.assertEqual(reader.read(100), extradata) #we read to the end resp.close() def test_content_length_sync(self): """Check that we don't read past the end of the Content-Length stream""" - extradata = "extradata" + extradata = b"extradata" expected = b"Hello123\r\n" - sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello123\r\n' + extradata) - resp = client.HTTPResponse(sock, method="GET") + reader = io.BytesIO(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello123\r\n' + extradata) + resp = client.HTTPResponse(reader, method="GET") resp.begin() self.assertEqual(resp.read(), expected) # the file should now have our extradata ready to be read - self.assertEqual(sock.file.read(100), extradata.encode("ascii")) #we read to the end + self.assertEqual(reader.read(100), extradata) #we read to the end resp.close() class ExtendedReadTest(TestCase): @@ -746,34 +738,17 @@ Test peek(), read1(), readline() """ lines = ( - 'HTTP/1.1 200 OK\r\n' - '\r\n' - 'hello world!\n' - 'and now \n' - 'for something completely different\n' - 'foo' + b'HTTP/1.1 200 OK\r\n' + b'\r\n' + b'hello world!\n' + b'and now \n' + b'for something completely different\n' + b'foo' ) - lines_expected = lines[lines.find('hello'):].encode("ascii") - lines_chunked = ( - 'HTTP/1.1 200 OK\r\n' - 'Transfer-Encoding: chunked\r\n\r\n' - 'a\r\n' - 'hello worl\r\n' - '3\r\n' - 'd!\n\r\n' - '9\r\n' - 'and now \n\r\n' - '23\r\n' - 'for something completely different\n\r\n' - '3\r\n' - 'foo\r\n' - '0\r\n' # terminating chunk - '\r\n' # end of trailers - ) + lines_expected = lines[lines.find(b'hello'):] def setUp(self): - sock = FakeSocket(self.lines) - resp = client.HTTPResponse(sock, method="GET") + resp = StringHTTPResponse(self.lines, method="GET") resp.begin() resp.fp = io.BufferedReader(resp.fp) self.resp = resp @@ -870,20 +845,20 @@ Test peek(), read1(), readline() in chunked mode """ lines = ( - 'HTTP/1.1 200 OK\r\n' - 'Transfer-Encoding: chunked\r\n\r\n' - 'a\r\n' - 'hello worl\r\n' - '3\r\n' - 'd!\n\r\n' - '9\r\n' - 'and now \n\r\n' - '23\r\n' - 'for something completely different\n\r\n' - '3\r\n' - 'foo\r\n' - '0\r\n' # terminating chunk - '\r\n' # end of trailers + b'HTTP/1.1 200 OK\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'a\r\n' + b'hello worl\r\n' + b'3\r\n' + b'd!\n\r\n' + b'9\r\n' + b'and now \n\r\n' + b'23\r\n' + b'for something completely different\n\r\n' + b'3\r\n' + b'foo\r\n' + b'0\r\n' # terminating chunk + b'\r\n' # end of trailers ) @@ -956,12 +931,12 @@ # for an ssl_wrapped connect() to actually return from. -class TimeoutTest(TestCase): - PORT = None +class ServerTest(TestCase): + """Tests that use a real HTTP server socket""" def setUp(self): self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - TimeoutTest.PORT = support.bind_port(self.serv) + self.PORT = support.bind_port(self.serv) self.serv.listen() def tearDown(self): @@ -976,7 +951,7 @@ self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: - httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn = client.HTTPConnection(HOST, self.PORT) httpConn.connect() finally: socket.setdefaulttimeout(None) @@ -987,7 +962,7 @@ self.assertIsNone(socket.getdefaulttimeout()) socket.setdefaulttimeout(30) try: - httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, + httpConn = client.HTTPConnection(HOST, self.PORT, timeout=None) httpConn.connect() finally: @@ -996,11 +971,64 @@ httpConn.close() # a value - httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn = client.HTTPConnection(HOST, self.PORT, timeout=30) httpConn.connect() self.assertEqual(httpConn.sock.gettimeout(), 30) httpConn.close() + def testDoubleResponse(self): + # Test when a server's response is followed by another in the same + # send() call + httpConn = client.HTTPConnection(HOST, self.PORT) + background = None + try: + serverSocket = None + serverReader = None + def startFirstResponse(): + nonlocal serverSocket, serverReader + [serverSocket, _] = self.serv.accept() + serverReader = serverSocket.makefile('rb') + while serverReader.readline().rstrip(b'\r\n'): + pass + serverSocket.sendall( + b'HTTP/1.1 200 First response\r\n' + b'Content-Length: 5\r\n' + b'\r\n' + ) + background = Thread(target=startFirstResponse) + background.start() + httpConn.request('GET', '/first') + with httpConn.getresponse() as response: + background.join() + self.assertEqual(response.reason, 'First response') + + def finishBothResponses(): + with serverSocket: + with serverReader: + while serverReader.readline().rstrip(b'\r\n'): + pass + serverSocket.sendall( + b'ABC\r\n' # First response's body + + b'HTTP/1.1 200 Second response\r\n' + b'Content-Length: 6\r\n' + b'\r\n' + b'Done\r\n' + ) + background = Thread(target=finishBothResponses) + background.start() + # Pipeline second request before reading first body, so that + # server can immediately follow it with the second response + httpConn.request('GET', '/second') + self.assertEqual(response.read(), b'ABC\r\n') + with httpConn.getresponse() as response: + self.assertEqual(response.reason, 'Second response') + self.assertEqual(response.read(), b'Done\r\n') + finally: + httpConn.close() + if background: + background.join() + class HTTPSTest(TestCase): @@ -1014,7 +1042,7 @@ def test_attributes(self): # simple test to check it's storing the timeout - h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + h = client.HTTPSConnection(HOST, 80, timeout=30) self.assertEqual(h.timeout, 30) def test_networked(self): @@ -1037,6 +1065,7 @@ context=context) h.request('GET', '/') resp = h.getresponse() + h.close() self.assertIn('nginx', resp.getheader('server')) @support.system_must_validate_cert @@ -1047,6 +1076,7 @@ h = client.HTTPSConnection('www.python.org', 443) h.request('GET', '/') resp = h.getresponse() + h.close() content_type = resp.getheader('content-type') self.assertIn('text/html', content_type) @@ -1061,6 +1091,7 @@ h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) h.request('GET', '/') resp = h.getresponse() + h.close() server_string = resp.getheader('server') self.assertIn('nginx', server_string) @@ -1230,10 +1261,9 @@ class HTTPResponseTest(TestCase): def setUp(self): - body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ + body = b"HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ second-value\r\n\r\nText" - sock = FakeSocket(body) - self.resp = client.HTTPResponse(sock) + self.resp = StringHTTPResponse(body) self.resp.begin() def test_getting_header(self): @@ -1324,7 +1354,7 @@ @support.reap_threads def test_main(verbose=None): - support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, + support.run_unittest(HeaderTests, OfflineTest, BasicTest, ServerTest, HTTPSTest, RequestBodyTest, SourceAddressTest, HTTPResponseTest, ExtendedReadTest, ExtendedReadTestChunked, TunnelTests) diff -r 673191aa1724 Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py Sun Feb 01 22:53:41 2015 -0800 +++ b/Lib/test/test_urllib2.py Mon Feb 02 11:31:49 2015 +0000 @@ -329,6 +329,8 @@ def close(self): pass + def _disown(self): + pass class MockHandler: # useful for testing handler machinery diff -r 673191aa1724 Lib/urllib/request.py --- a/Lib/urllib/request.py Sun Feb 01 22:53:41 2015 -0800 +++ b/Lib/urllib/request.py Mon Feb 02 11:31:49 2015 +0000 @@ -1205,9 +1205,7 @@ # If the server does not send us a 'Connection: close' header, # HTTPConnection assumes the socket should be left open. Manually # mark the socket to be closed when this response object goes away. - if h.sock: - h.sock.close() - h.sock = None + h._disown() r.url = req.get_full_url() # This line replaces the .msg attribute of the HTTPResponse