# HG changeset patch
# Parent  faeeb8dbe43260001c5231593adb44412e3f92a0
Issue #17214: Percent-encode non-ASCII bytes in redirect targets

Some servers send Location header fields with non-ASCII bytes, but "http.
client" requires the request target to be ASCII-encodable, otherwise a
UnicodeEncodeError is raised. Based on patch by Christian Heimes.

Python 2 does not suffer any problem because it allows non-ASCII bytes in the
HTTP request target.

diff -r faeeb8dbe432 Lib/test/test_urllib2.py
--- a/Lib/test/test_urllib2.py	Wed Oct 28 21:39:36 2015 +0200
+++ b/Lib/test/test_urllib2.py	Fri Oct 30 23:31:04 2015 +0000
@@ -1133,6 +1133,41 @@
         fp = o.open('http://www.example.com')
         self.assertEqual(fp.geturl(), redirected_url.strip())
 
+    def test_redirect_encoding(self):
+        # Some characters in the redirect target may need special handling,
+        # but most ASCII characters should be treated as already encoded
+        class Handler(urllib.request.HTTPHandler):
+            def http_open(self, req):
+                result = self.do_open(self.connection, req)
+                self.last_buf = self.connection.buf
+                # Set up a normal response for the next request
+                self.connection = test_urllib.fakehttp(
+                    b'HTTP/1.1 200 OK\r\n'
+                    b'Content-Length: 3\r\n'
+                    b'\r\n'
+                    b'123'
+                )
+                return result
+        handler = Handler()
+        opener = urllib.request.build_opener(handler)
+        tests = (
+            (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'),
+            (b'/spaced%20path/', b'/spaced%20path/'),
+            (b'/spaced path/', b'/spaced%20path/'),
+            (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'),
+        )
+        for [location, result] in tests:
+            with self.subTest(repr(location)):
+                handler.connection = test_urllib.fakehttp(
+                    b'HTTP/1.1 302 Redirect\r\n'
+                    b'Location: ' + location + b'\r\n'
+                    b'\r\n'
+                )
+                response = opener.open('http://example.com/')
+                expected = b'GET ' + result + b' '
+                request = handler.last_buf
+                self.assertTrue(request.startswith(expected), repr(request))
+
     def test_proxy(self):
         o = OpenerDirector()
         ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
diff -r faeeb8dbe432 Lib/urllib/request.py
--- a/Lib/urllib/request.py	Wed Oct 28 21:39:36 2015 +0200
+++ b/Lib/urllib/request.py	Fri Oct 30 23:31:04 2015 +0000
@@ -91,6 +91,7 @@
 import posixpath
 import re
 import socket
+import string
 import sys
 import time
 import collections
@@ -615,8 +616,6 @@
         # from the user (of urllib.request, in this case).  In practice,
         # essentially all clients do redirect in this case, so we do
         # the same.
-        # be conciliant with URIs containing a space
-        newurl = newurl.replace(' ', '%20')
         CONTENT_HEADERS = ("content-length", "content-type")
         newheaders = dict((k, v) for k, v in req.headers.items()
                           if k.lower() not in CONTENT_HEADERS)
@@ -656,6 +655,11 @@
             urlparts[2] = "/"
         newurl = urlunparse(urlparts)
 
+        # http.client.parse_headers() decodes as ISO-8859-1. Recover the
+        # original bytes and percent-encode non-ASCII bytes, and any special
+        # characters such as the space.
+        newurl = quote(
+            newurl, encoding="iso-8859-1", safe=string.punctuation)
         newurl = urljoin(req.full_url, newurl)
 
         # XXX Probably want to forget about the state of the current
diff -r faeeb8dbe432 Misc/NEWS
--- a/Misc/NEWS	Wed Oct 28 21:39:36 2015 +0200
+++ b/Misc/NEWS	Fri Oct 30 23:31:04 2015 +0000
@@ -96,6 +96,12 @@
 Library
 -------
 
+- Issue #17214: The "urllib.request" module now percent-encodes non-ASCII
+  bytes found in redirect target URLs. Some servers send Location header
+  fields with non-ASCII bytes, but "http.client" requires the request target
+  to be ASCII-encodable, otherwise a UnicodeEncodeError is raised. Based on
+  patch by Christian Heimes.
+
 - Issue #21827: Fixed textwrap.dedent() for the case when largest common
   whitespace is a substring of smallest leading whitespace.
   Based on patch by Robert Li.