Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(76627)

Delta Between Two Patch Sets: Lib/httplib.py

Issue 16037: httplib: header parsing is not delimited
Left Patch Set: Created 5 years, 9 months ago
Right Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | Lib/test/test_httplib.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 r"""HTTP/1.1 client library 1 r"""HTTP/1.1 client library
2 2
3 <intro stuff goes here> 3 <intro stuff goes here>
4 <other stuff, too> 4 <other stuff, too>
5 5
6 HTTPConnection goes through a number of "states", which define when a client 6 HTTPConnection goes through a number of "states", which define when a client
7 may legally make another request or fetch the response for a particular 7 may legally make another request or fetch the response for a particular
8 request. This diagram details these state transitions: 8 request. This diagram details these state transitions:
9 9
10 (null) 10 (null)
(...skipping 198 matching lines...) Expand 10 before | Expand all | Expand 10 after
209 505: 'HTTP Version Not Supported', 209 505: 'HTTP Version Not Supported',
210 } 210 }
211 211
212 # maximal amount of data to read at one time in _safe_read 212 # maximal amount of data to read at one time in _safe_read
213 MAXAMOUNT = 1048576 213 MAXAMOUNT = 1048576
214 214
215 # maximal line length when calling readline(). 215 # maximal line length when calling readline().
216 _MAXLINE = 65536 216 _MAXLINE = 65536
217 _MAXHEADERS = 100 217 _MAXHEADERS = 100
218 218
219
220 class HTTPMessage(mimetools.Message): 219 class HTTPMessage(mimetools.Message):
221 220
222 def addheader(self, key, value): 221 def addheader(self, key, value):
223 """Add header for field key handling repeats.""" 222 """Add header for field key handling repeats."""
224 prev = self.dict.get(key) 223 prev = self.dict.get(key)
225 if prev is None: 224 if prev is None:
226 self.dict[key] = value 225 self.dict[key] = value
227 else: 226 else:
228 combined = ", ".join((prev, value)) 227 combined = ", ".join((prev, value))
229 self.dict[key] = combined 228 self.dict[key] = combined
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
267 self.status = '' 266 self.status = ''
268 headerseen = "" 267 headerseen = ""
269 firstline = 1 268 firstline = 1
270 startofline = unread = tell = None 269 startofline = unread = tell = None
271 if hasattr(self.fp, 'unread'): 270 if hasattr(self.fp, 'unread'):
272 unread = self.fp.unread 271 unread = self.fp.unread
273 elif self.seekable: 272 elif self.seekable:
274 tell = self.fp.tell 273 tell = self.fp.tell
275 while True: 274 while True:
276 if len(hlist) > _MAXHEADERS: 275 if len(hlist) > _MAXHEADERS:
277 raise TooManyHeaders() 276 raise HTTPException("got more than %d headers" % _MAXHEADERS)
278 if tell: 277 if tell:
279 try: 278 try:
280 startofline = tell() 279 startofline = tell()
281 except IOError: 280 except IOError:
282 startofline = tell = None 281 startofline = tell = None
283 self.seekable = 0 282 self.seekable = 0
284 line = self.fp.readline(_MAXLINE + 1) 283 line = self.fp.readline(_MAXLINE + 1)
285 if len(line) > _MAXLINE: 284 if len(line) > _MAXLINE:
286 raise LineTooLong("header line") 285 raise LineTooLong("header line")
287 if not line: 286 if not line:
(...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after
562 561
563 if self.length is not None: 562 if self.length is not None:
564 if amt > self.length: 563 if amt > self.length:
565 # clip the read to the "end of response" 564 # clip the read to the "end of response"
566 amt = self.length 565 amt = self.length
567 566
568 # we do not use _safe_read() here because this may be a .will_close 567 # we do not use _safe_read() here because this may be a .will_close
569 # connection, and the user is reading more bytes than will be provided 568 # connection, and the user is reading more bytes than will be provided
570 # (for example, reading in 1k chunks) 569 # (for example, reading in 1k chunks)
571 s = self.fp.read(amt) 570 s = self.fp.read(amt)
572 if not s: 571 if not s and amt:
573 # Ideally, we would raise IncompleteRead if the content-length 572 # Ideally, we would raise IncompleteRead if the content-length
574 # wasn't satisfied, but it might break compatibility. 573 # wasn't satisfied, but it might break compatibility.
575 self.close() 574 self.close()
576 if self.length is not None: 575 if self.length is not None:
577 self.length -= len(s) 576 self.length -= len(s)
578 if not self.length: 577 if not self.length:
579 self.close() 578 self.close()
580 579
581 return s 580 return s
582 581
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after
697 self.timeout = timeout 696 self.timeout = timeout
698 self.source_address = source_address 697 self.source_address = source_address
699 self.sock = None 698 self.sock = None
700 self._buffer = [] 699 self._buffer = []
701 self.__response = None 700 self.__response = None
702 self.__state = _CS_IDLE 701 self.__state = _CS_IDLE
703 self._method = None 702 self._method = None
704 self._tunnel_host = None 703 self._tunnel_host = None
705 self._tunnel_port = None 704 self._tunnel_port = None
706 self._tunnel_headers = {} 705 self._tunnel_headers = {}
707
708 self._set_hostport(host, port)
709 if strict is not None: 706 if strict is not None:
710 self.strict = strict 707 self.strict = strict
711 708
709 (self.host, self.port) = self._get_hostport(host, port)
710
711 # This is stored as an instance variable to allow unittests
712 # to replace with a suitable mock
713 self._create_connection = socket.create_connection
714
712 def set_tunnel(self, host, port=None, headers=None): 715 def set_tunnel(self, host, port=None, headers=None):
713 """ Sets up the host and the port for the HTTP CONNECT Tunnelling. 716 """ Set up host and port for HTTP CONNECT tunnelling.
717
718 In a connection that uses HTTP Connect tunneling, the host passed to the
719 constructor is used as proxy server that relays all communication to the
720 endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
721 request to the proxy server when the connection is established.
722
723 This method must be called before the HTML connection has been
724 established.
714 725
715 The headers argument should be a mapping of extra HTTP headers 726 The headers argument should be a mapping of extra HTTP headers
716 to send with the CONNECT request. 727 to send with the CONNECT request.
717 """ 728 """
729 # Verify if this is required.
730 if self.sock:
731 raise RuntimeError("Can't setup tunnel for established connection.")
732
718 self._tunnel_host = host 733 self._tunnel_host = host
719 self._tunnel_port = port 734 self._tunnel_port = port
720 if headers: 735 if headers:
721 self._tunnel_headers = headers 736 self._tunnel_headers = headers
722 else: 737 else:
723 self._tunnel_headers.clear() 738 self._tunnel_headers.clear()
724 739
725 def _set_hostport(self, host, port): 740 def _get_hostport(self, host, port):
726 if port is None: 741 if port is None:
727 i = host.rfind(':') 742 i = host.rfind(':')
728 j = host.rfind(']') # ipv6 addresses have [...] 743 j = host.rfind(']') # ipv6 addresses have [...]
729 if i > j: 744 if i > j:
730 try: 745 try:
731 port = int(host[i+1:]) 746 port = int(host[i+1:])
732 except ValueError: 747 except ValueError:
733 if host[i+1:] == "": # http://foo.com:/ == http://foo.com/ 748 if host[i+1:] == "": # http://foo.com:/ == http://foo.com/
734 port = self.default_port 749 port = self.default_port
735 else: 750 else:
736 raise InvalidURL("nonnumeric port: '%s'" % host[i+1:]) 751 raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
737 host = host[:i] 752 host = host[:i]
738 else: 753 else:
739 port = self.default_port 754 port = self.default_port
740 if host and host[0] == '[' and host[-1] == ']': 755 if host and host[0] == '[' and host[-1] == ']':
741 host = host[1:-1] 756 host = host[1:-1]
742 self.host = host 757 return (host, port)
743 self.port = port
744 758
745 def set_debuglevel(self, level): 759 def set_debuglevel(self, level):
746 self.debuglevel = level 760 self.debuglevel = level
747 761
748 def _tunnel(self): 762 def _tunnel(self):
749 self._set_hostport(self._tunnel_host, self._tunnel_port) 763 (host, port) = self._get_hostport(self._tunnel_host, self._tunnel_port)
750 self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port)) 764 self.send("CONNECT %s:%d HTTP/1.0\r\n" % (host, port))
751 for header, value in self._tunnel_headers.iteritems(): 765 for header, value in self._tunnel_headers.iteritems():
752 self.send("%s: %s\r\n" % (header, value)) 766 self.send("%s: %s\r\n" % (header, value))
753 self.send("\r\n") 767 self.send("\r\n")
754 response = self.response_class(self.sock, strict = self.strict, 768 response = self.response_class(self.sock, strict = self.strict,
755 method = self._method) 769 method = self._method)
756 (version, code, message) = response._read_status() 770 (version, code, message) = response._read_status()
757 771
758 if code != 200: 772 if code != 200:
759 self.close() 773 self.close()
760 raise socket.error("Tunnel connection failed: %d %s" % (code, 774 raise socket.error("Tunnel connection failed: %d %s" % (code,
761 message.stri p())) 775 message.stri p()))
762 while True: 776 while True:
763 line = response.fp.readline(_MAXLINE + 1) 777 line = response.fp.readline(_MAXLINE + 1)
764 if len(line) > _MAXLINE: 778 if len(line) > _MAXLINE:
765 raise LineTooLong("header line") 779 raise LineTooLong("header line")
766 if not line: 780 if not line:
767 # for sites which EOF without sending trailer 781 # for sites which EOF without sending trailer
768 break 782 break
769 if line == '\r\n': 783 if line == '\r\n':
770 break 784 break
771 785
772 786
773 def connect(self): 787 def connect(self):
774 """Connect to the host and port specified in __init__.""" 788 """Connect to the host and port specified in __init__."""
775 self.sock = socket.create_connection((self.host,self.port), 789 self.sock = self._create_connection((self.host,self.port),
776 self.timeout, self.source_address) 790 self.timeout, self.source_address)
777 791
778 if self._tunnel_host: 792 if self._tunnel_host:
779 self._tunnel() 793 self._tunnel()
780 794
781 def close(self): 795 def close(self):
782 """Close the connection to the HTTP server.""" 796 """Close the connection to the HTTP server."""
783 if self.sock: 797 if self.sock:
784 self.sock.close() # close it manually... there may be other refs 798 self.sock.close() # close it manually... there may be other refs
785 self.sock = None 799 self.sock = None
786 if self.__response: 800 if self.__response:
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after
904 if url.startswith('http'): 918 if url.startswith('http'):
905 nil, netloc, nil, nil, nil = urlsplit(url) 919 nil, netloc, nil, nil, nil = urlsplit(url)
906 920
907 if netloc: 921 if netloc:
908 try: 922 try:
909 netloc_enc = netloc.encode("ascii") 923 netloc_enc = netloc.encode("ascii")
910 except UnicodeEncodeError: 924 except UnicodeEncodeError:
911 netloc_enc = netloc.encode("idna") 925 netloc_enc = netloc.encode("idna")
912 self.putheader('Host', netloc_enc) 926 self.putheader('Host', netloc_enc)
913 else: 927 else:
928 if self._tunnel_host:
929 host = self._tunnel_host
930 port = self._tunnel_port
931 else:
932 host = self.host
933 port = self.port
934
914 try: 935 try:
915 host_enc = self.host.encode("ascii") 936 host_enc = host.encode("ascii")
916 except UnicodeEncodeError: 937 except UnicodeEncodeError:
917 host_enc = self.host.encode("idna") 938 host_enc = host.encode("idna")
918 # Wrap the IPv6 Host Header with [] (RFC 2732) 939 # Wrap the IPv6 Host Header with [] (RFC 2732)
919 if host_enc.find(':') >= 0: 940 if host_enc.find(':') >= 0:
920 host_enc = "[" + host_enc + "]" 941 host_enc = "[" + host_enc + "]"
921 if self.port == self.default_port: 942 if port == self.default_port:
922 self.putheader('Host', host_enc) 943 self.putheader('Host', host_enc)
923 else: 944 else:
924 self.putheader('Host', "%s:%s" % (host_enc, self.port)) 945 self.putheader('Host', "%s:%s" % (host_enc, port))
925 946
926 # note: we are assuming that clients will not attempt to set these 947 # note: we are assuming that clients will not attempt to set these
927 # headers since *this* library must deal with the 948 # headers since *this* library must deal with the
928 # consequences. this also means that when the supporting 949 # consequences. this also means that when the supporting
929 # libraries are updated to recognize other forms, then this 950 # libraries are updated to recognize other forms, then this
930 # code should be changed (removed or updated). 951 # code should be changed (removed or updated).
931 952
932 # we only want a Content-Encoding of "identity" since we don't 953 # we only want a Content-Encoding of "identity" since we don't
933 # support encodings such as x-gzip or x-deflate. 954 # support encodings such as x-gzip or x-deflate.
934 if not skip_accept_encoding: 955 if not skip_accept_encoding:
(...skipping 230 matching lines...) Expand 10 before | Expand all | Expand 10 after
1165 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, 1186 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
1166 source_address=None): 1187 source_address=None):
1167 HTTPConnection.__init__(self, host, port, strict, timeout, 1188 HTTPConnection.__init__(self, host, port, strict, timeout,
1168 source_address) 1189 source_address)
1169 self.key_file = key_file 1190 self.key_file = key_file
1170 self.cert_file = cert_file 1191 self.cert_file = cert_file
1171 1192
1172 def connect(self): 1193 def connect(self):
1173 "Connect to a host on a given (SSL) port." 1194 "Connect to a host on a given (SSL) port."
1174 1195
1175 sock = socket.create_connection((self.host, self.port), 1196 sock = self._create_connection((self.host, self.port),
1176 self.timeout, self.source_address) 1197 self.timeout, self.source_address)
1177 if self._tunnel_host: 1198 if self._tunnel_host:
1178 self.sock = sock 1199 self.sock = sock
1179 self._tunnel() 1200 self._tunnel()
1180 self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) 1201 self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
1181 1202
1182 __all__.append("HTTPSConnection") 1203 __all__.append("HTTPSConnection")
1183 1204
1184 class HTTPS(HTTP): 1205 class HTTPS(HTTP):
1185 """Compatibility with 1.5 httplib interface 1206 """Compatibility with 1.5 httplib interface
1186 1207
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after
1266 def __init__(self, line): 1287 def __init__(self, line):
1267 if not line: 1288 if not line:
1268 line = repr(line) 1289 line = repr(line)
1269 self.args = line, 1290 self.args = line,
1270 self.line = line 1291 self.line = line
1271 1292
1272 class LineTooLong(HTTPException): 1293 class LineTooLong(HTTPException):
1273 def __init__(self, line_type): 1294 def __init__(self, line_type):
1274 HTTPException.__init__(self, "got more than %d bytes when reading %s" 1295 HTTPException.__init__(self, "got more than %d bytes when reading %s"
1275 % (_MAXLINE, line_type)) 1296 % (_MAXLINE, line_type))
1276
1277
1278 class TooManyHeaders(HTTPException):
1279 def __init__(self):
1280 HTTPException.__init__(self, "got more than %d headers" % _MAXHEADERS)
1281 1297
1282 # for backwards compatibility 1298 # for backwards compatibility
1283 error = HTTPException 1299 error = HTTPException
1284 1300
1285 class LineAndFileWrapper: 1301 class LineAndFileWrapper:
1286 """A limited file-like object for HTTP/0.9 responses.""" 1302 """A limited file-like object for HTTP/0.9 responses."""
1287 1303
1288 # The status-line parsing code calls readline(), which normally 1304 # The status-line parsing code calls readline(), which normally
1289 # get the HTTP status line. For a 0.9 response, however, this is 1305 # get the HTTP status line. For a 0.9 response, however, this is
1290 # actually the first line of the body! Clients need to get a 1306 # actually the first line of the body! Clients need to get a
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
1342 def readlines(self, size=None): 1358 def readlines(self, size=None):
1343 if self._line_consumed: 1359 if self._line_consumed:
1344 return self._file.readlines(size) 1360 return self._file.readlines(size)
1345 assert self._line_left 1361 assert self._line_left
1346 L = [self._line[self._line_offset:]] 1362 L = [self._line[self._line_offset:]]
1347 self._done() 1363 self._done()
1348 if size is None: 1364 if size is None:
1349 return L + self._file.readlines() 1365 return L + self._file.readlines()
1350 else: 1366 else:
1351 return L + self._file.readlines(size) 1367 return L + self._file.readlines(size)
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+