# HG changeset patch # Parent faeeb8dbe43260001c5231593adb44412e3f92a0 Issue #17214: Percent-encode non-ASCII bytes in redirect targets Some servers send Location header fields with non-ASCII bytes, but "http. client" requires the request target to be ASCII-encodable, otherwise a UnicodeEncodeError is raised. Based on patch by Christian Heimes. Python 2 does not suffer any problem because it allows non-ASCII bytes in the HTTP request target. diff -r faeeb8dbe432 Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py Wed Oct 28 21:39:36 2015 +0200 +++ b/Lib/test/test_urllib2.py Fri Oct 30 23:31:04 2015 +0000 @@ -1133,6 +1133,41 @@ fp = o.open('http://www.example.com') self.assertEqual(fp.geturl(), redirected_url.strip()) + def test_redirect_encoding(self): + # Some characters in the redirect target may need special handling, + # but most ASCII characters should be treated as already encoded + class Handler(urllib.request.HTTPHandler): + def http_open(self, req): + result = self.do_open(self.connection, req) + self.last_buf = self.connection.buf + # Set up a normal response for the next request + self.connection = test_urllib.fakehttp( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 3\r\n' + b'\r\n' + b'123' + ) + return result + handler = Handler() + opener = urllib.request.build_opener(handler) + tests = ( + (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), + (b'/spaced%20path/', b'/spaced%20path/'), + (b'/spaced path/', b'/spaced%20path/'), + (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), + ) + for [location, result] in tests: + with self.subTest(repr(location)): + handler.connection = test_urllib.fakehttp( + b'HTTP/1.1 302 Redirect\r\n' + b'Location: ' + location + b'\r\n' + b'\r\n' + ) + response = opener.open('http://example.com/') + expected = b'GET ' + result + b' ' + request = handler.last_buf + self.assertTrue(request.startswith(expected), repr(request)) + def test_proxy(self): o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) diff -r faeeb8dbe432 Lib/urllib/request.py --- a/Lib/urllib/request.py Wed Oct 28 21:39:36 2015 +0200 +++ b/Lib/urllib/request.py Fri Oct 30 23:31:04 2015 +0000 @@ -91,6 +91,7 @@ import posixpath import re import socket +import string import sys import time import collections @@ -615,8 +616,6 @@ # from the user (of urllib.request, in this case). In practice, # essentially all clients do redirect in this case, so we do # the same. - # be conciliant with URIs containing a space - newurl = newurl.replace(' ', '%20') CONTENT_HEADERS = ("content-length", "content-type") newheaders = dict((k, v) for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS) @@ -656,6 +655,11 @@ urlparts[2] = "/" newurl = urlunparse(urlparts) + # http.client.parse_headers() decodes as ISO-8859-1. Recover the + # original bytes and percent-encode non-ASCII bytes, and any special + # characters such as the space. + newurl = quote( + newurl, encoding="iso-8859-1", safe=string.punctuation) newurl = urljoin(req.full_url, newurl) # XXX Probably want to forget about the state of the current diff -r faeeb8dbe432 Misc/NEWS --- a/Misc/NEWS Wed Oct 28 21:39:36 2015 +0200 +++ b/Misc/NEWS Fri Oct 30 23:31:04 2015 +0000 @@ -96,6 +96,12 @@ Library ------- +- Issue #17214: The "urllib.request" module now percent-encodes non-ASCII + bytes found in redirect target URLs. Some servers send Location header + fields with non-ASCII bytes, but "http.client" requires the request target + to be ASCII-encodable, otherwise a UnicodeEncodeError is raised. Based on + patch by Christian Heimes. + - Issue #21827: Fixed textwrap.dedent() for the case when largest common whitespace is a substring of smallest leading whitespace. Based on patch by Robert Li.