# HG changeset patch # Parent 7846aadbd4f5532a993866118f2b62e7b14b6f27 diff -r 7846aadbd4f5 Doc/library/http.client.rst --- a/Doc/library/http.client.rst Thu Jun 25 23:39:53 2015 +0300 +++ b/Doc/library/http.client.rst Fri Jun 26 04:36:51 2015 +0000 @@ -303,6 +303,9 @@ .. versionadded:: 3.2 + .. versionchanged:: 3.6 + The CONNECT request now uses HTTP 1.1 and conforms to :rfc:`2817`. + .. method:: HTTPConnection.connect() @@ -315,6 +318,16 @@ Close the connection to the server. +.. method:: HTTPConnection.detach() + + Detach the underlying socket connection without closing it, and leave + the :class:`HTTPConnection` and current :class:`HTTPResponse` in + a closed state. The return value is a pair ``(sock, reader)``, + where *sock* is a :class:`~socket.socket` instance, and *reader* is + a :class:`~io.BufferedReader`. + + .. versionadded:: 3.6 + As an alternative to using the :meth:`request` method described above, you can also send your request step by step, by using the four functions below. diff -r 7846aadbd4f5 Lib/http/client.py --- a/Lib/http/client.py Thu Jun 25 23:39:53 2015 +0300 +++ b/Lib/http/client.py Fri Jun 26 04:36:51 2015 +0000 @@ -306,8 +306,8 @@ self.headers = self.msg = parse_headers(self.fp) if self.debuglevel > 0: - for hdr in self.headers: - print("header:", hdr, end=" ") + for name, value in self.headers.items(): + print("header: {}: {}".format(name, value)) # are we using the chunked-style of transfer encoding? tr_enc = self.headers.get("transfer-encoding") @@ -790,35 +790,31 @@ self.debuglevel = level def _tunnel(self): - connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, - self._tunnel_port) - connect_bytes = connect_str.encode("ascii") - self.send(connect_bytes) - for header, value in self._tunnel_headers.items(): - header_str = "%s: %s\r\n" % (header, value) - header_bytes = header_str.encode("latin-1") - self.send(header_bytes) - self.send(b'\r\n') + tunnel_state = self.__state + try: + self.__state = _CS_IDLE # Temporarily reset state to do CONNECT - response = self.response_class(self.sock, method=self._method) - (version, code, message) = response._read_status() + authority = "%s:%d" % (self._tunnel_host, self._tunnel_port) + # Host field must be identical to authority. Add it manually + # because putrequest() may omit the port. + self.putrequest("CONNECT", authority, + skip_host=True, skip_accept_encoding=True) + if not any(header.lower() == "host" for + header in self._tunnel_headers.keys()): + self.putheader("Host", authority) + for header, value in self._tunnel_headers.items(): + self.putheader(header, value) + self.endheaders() - if code != http.HTTPStatus.OK: - self.close() - raise OSError("Tunnel connection failed: %d %s" % (code, - message.strip())) - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break - - if self.debuglevel > 0: - print('header:', line.decode()) + with self.getresponse() as response: + if response.code != http.HTTPStatus.OK: + self.close() + msg = "Tunnel connection failed: %d %s" + raise OSError(msg % (response.code, response.reason)) + self.sock, reader = self.detach() + reader.close() # Any buffered but unread data is lost! + finally: + self.__state = tunnel_state def connect(self): """Connect to the host and port specified in __init__.""" @@ -843,6 +839,16 @@ self.__response = None response.close() + def detach(self): + """Detach (sock, reader); set response and connection to closed""" + detached = (self.__detach_sock, self.__detach_response.fp) + self.__detach_response.fp = None + self.__detach_response.close() + self.sock = None + self.__response = None + self.__state = _CS_IDLE + return detached + def send(self, data): """Send `data' to the server. ``data`` can be a string object, a bytes object, an array object, a @@ -912,7 +918,8 @@ if message_body is not None: self.send(message_body) - def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): + def putrequest(self, method, url, + skip_host=False, skip_accept_encoding=False): """Send a request to the server. `method' specifies an HTTP request method, e.g. 'GET'. @@ -1178,6 +1185,10 @@ assert response.will_close != _UNKNOWN self.__state = _CS_IDLE + # Save connection objects for detach() + self.__detach_sock = self.sock + self.__detach_response = response + if response.will_close: # this effectively passes the connection to the response self.close() diff -r 7846aadbd4f5 Lib/test/test_httplib.py --- a/Lib/test/test_httplib.py Thu Jun 25 23:39:53 2015 +0300 +++ b/Lib/test/test_httplib.py Fri Jun 26 04:36:51 2015 +0000 @@ -49,7 +49,6 @@ self.fileclass = fileclass self.data = b'' self.sendall_calls = 0 - self.file_closed = False self.host = host self.port = port @@ -62,12 +61,8 @@ raise client.UnimplementedFileMode() # keep the file around so we can check how much was read from it self.file = self.fileclass(self.text) - self.file.close = self.file_close #nerf close () return self.file - def file_close(self): - self.file_closed = True - def close(self): pass @@ -798,7 +793,7 @@ conn.request('GET', '/') self.assertRaises(client.BadStatusLine, conn.getresponse) self.assertTrue(response.closed) - self.assertTrue(conn.sock.file_closed) + self.assertTrue(conn.sock.file.closed) def test_chunked_extension(self): extra = '3;foo=bar\r\n' + 'abc\r\n' @@ -825,6 +820,7 @@ sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end) resp = client.HTTPResponse(sock, method="GET") resp.begin() + sock.file.close = lambda: None # Prevent socket reader from closing self.assertEqual(resp.read(), expected) # we should have reached the end of the file self.assertEqual(sock.file.read(100), b"") #we read to the end @@ -836,6 +832,7 @@ extradata = "extradata" sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata) resp = client.HTTPResponse(sock, method="GET") + sock.file.close = lambda: None # Prevent socket reader from closing resp.begin() self.assertEqual(resp.read(), expected) # the file should now have our extradata ready to be read @@ -849,11 +846,64 @@ sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello123\r\n' + extradata) resp = client.HTTPResponse(sock, method="GET") resp.begin() + sock.file.close = lambda: None # Prevent socket reader from closing self.assertEqual(resp.read(), expected) # the file should now have our extradata ready to be read self.assertEqual(sock.file.read(100), extradata.encode("ascii")) #we read to the end resp.close() + def test_detach_connect(self): + """Test detach() with CONNECT request""" + conn = FakeSocketHTTPConnection( + b"HTTP/1.1 200 Connection established\r\n" + b"Proxy-agent: Netscape-Proxy/1.1\r\n" + b"\r\n" + b"" + ) + headers = {"Host": "server.example.com:80"} + conn.request("CONNECT", "server.example.com:80", headers=headers) + with conn.getresponse() as response: + self.assertFalse(response.closed) + sock, reader = conn.detach() + self.assertTrue(response.closed) + + # Connection object was marked as closed, so should be reusable + conn.request("GET", "/separate-request") + self.assertEqual(conn.connections, 2) + + self.assertEqual(response.version, 11) + self.assertEqual(response.status, 200) + self.assertEqual(response.reason, "Connection established") + fields = ( + ("Proxy-agent", "Netscape-Proxy/1.1"), + ) + self.assertSequenceEqual(response.getheaders(), fields) + self.assertIs(reader, sock.file) + self.assertEqual(reader.read(), b"") + + def test_detach_upgrade(self): + """Test detach() with Upgrade: h2c""" + conn = FakeSocketHTTPConnection( + b"HTTP/1.1 101 Switching Protocols\r\n" + b"Connection: Upgrade\r\n" + b"Upgrade: h2c\r\n" + b"\r\n" + b"" + ) + headers = { + "Connection": "Upgrade, HTTP2-Settings", + "Upgrade": "h2c", + "HTTP2-Settings": "", + } + conn.request("HEAD", "/", headers=headers) + response = conn.getresponse() + self.assertEqual(response.status, 101) + self.assertEqual(response.getheader("Connection"), "Upgrade") + self.assertEqual(response.getheader("Upgrade"), "h2c") + sock, reader = conn.detach() + self.assertIs(reader, sock.file) + self.assertEqual(reader.read(), b"") + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1569,12 +1619,12 @@ self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + connect_request = self.conn.sock.data.split(b'\r\n\r\n', 1)[0] + self.assertIn(b'CONNECT destination.com', connect_request) + self.assertIn(b' HTTP/1.1', connect_request) # issue22095 self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) - - # This test should be removed when CONNECT gets the HTTP/1.1 blessing + self.assertIn(b'Host: destination.com', connect_request) self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) def test_connect_put_request(self): @@ -1582,8 +1632,9 @@ self.conn.request('PUT', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) + connect_request = self.conn.sock.data.split(b'\r\n\r\n', 1)[0] + self.assertIn(b'CONNECT destination.com', connect_request) + self.assertIn(b'Host: destination.com', connect_request) def test_tunnel_debuglog(self): expected_header = 'X-Dummy: 1' diff -r 7846aadbd4f5 Lib/test/test_httpservers.py --- a/Lib/test/test_httpservers.py Thu Jun 25 23:39:53 2015 +0300 +++ b/Lib/test/test_httpservers.py Fri Jun 26 04:36:51 2015 +0000 @@ -254,7 +254,8 @@ with support.captured_stderr() as err: self.con.request('GET', '/') - self.con.getresponse() + with self.con.getresponse(): + pass self.assertTrue( err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n')) @@ -265,7 +266,8 @@ with support.captured_stderr() as err: self.con.request('ERROR', '/') - self.con.getresponse() + with self.con.getresponse(): + pass lines = err.getvalue().split('\n') self.assertTrue(lines[0].endswith('code 404, message File not found'))