Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(38639)

Delta Between Two Patch Sets: Lib/http/client.py

Issue 22417: PEP 476: verify HTTPS certificates by default
Left Patch Set: Created 5 years ago
Right Patch Set: Created 4 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Doc/whatsnew/3.4.rst ('k') | Lib/ssl.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 """HTTP/1.1 client library 1 """HTTP/1.1 client library
2 2
3 <intro stuff goes here> 3 <intro stuff goes here>
4 <other stuff, too> 4 <other stuff, too>
5 5
6 HTTPConnection goes through a number of "states", which define when a client 6 HTTPConnection goes through a number of "states", which define when a client
7 may legally make another request or fetch the response for a particular 7 may legally make another request or fetch the response for a particular
8 request. This diagram details these state transitions: 8 request. This diagram details these state transitions:
9 9
10 (null) 10 (null)
(...skipping 252 matching lines...) Expand 10 before | Expand all | Expand 10 after
263 raise LineTooLong("header line") 263 raise LineTooLong("header line")
264 headers.append(line) 264 headers.append(line)
265 if len(headers) > _MAXHEADERS: 265 if len(headers) > _MAXHEADERS:
266 raise HTTPException("got more than %d headers" % _MAXHEADERS) 266 raise HTTPException("got more than %d headers" % _MAXHEADERS)
267 if line in (b'\r\n', b'\n', b''): 267 if line in (b'\r\n', b'\n', b''):
268 break 268 break
269 hstring = b''.join(headers).decode('iso-8859-1') 269 hstring = b''.join(headers).decode('iso-8859-1')
270 return email.parser.Parser(_class=_class).parsestr(hstring) 270 return email.parser.Parser(_class=_class).parsestr(hstring)
271 271
272 272
273 class HTTPResponse(io.RawIOBase): 273 class HTTPResponse(io.BufferedIOBase):
274 274
275 # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details. 275 # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
276 276
277 # The bytes from the socket object are iso-8859-1 strings. 277 # The bytes from the socket object are iso-8859-1 strings.
278 # See RFC 2616 sec 2.2 which notes an exception for MIME-encoded 278 # See RFC 2616 sec 2.2 which notes an exception for MIME-encoded
279 # text following RFC 2047. The basic status line parsing only 279 # text following RFC 2047. The basic status line parsing only
280 # accepts iso-8859-1. 280 # accepts iso-8859-1.
281 281
282 def __init__(self, sock, debuglevel=0, method=None, url=None): 282 def __init__(self, sock, debuglevel=0, method=None, url=None):
283 # If the response includes a content-length header, we need to 283 # If the response includes a content-length header, we need to
(...skipping 204 matching lines...) Expand 10 before | Expand all | Expand 10 after
488 488
489 def read(self, amt=None): 489 def read(self, amt=None):
490 if self.fp is None: 490 if self.fp is None:
491 return b"" 491 return b""
492 492
493 if self._method == "HEAD": 493 if self._method == "HEAD":
494 self._close_conn() 494 self._close_conn()
495 return b"" 495 return b""
496 496
497 if amt is not None: 497 if amt is not None:
498 # Amount is given, so call base class version 498 # Amount is given, implement using readinto
499 # (which is implemented in terms of self.readinto) 499 b = bytearray(amt)
500 return super(HTTPResponse, self).read(amt) 500 n = self.readinto(b)
501 return memoryview(b)[:n].tobytes()
501 else: 502 else:
502 # Amount is not given (unbounded read) so we must check self.length 503 # Amount is not given (unbounded read) so we must check self.length
503 # and self.chunked 504 # and self.chunked
504 505
505 if self.chunked: 506 if self.chunked:
506 return self._readall_chunked() 507 return self._readall_chunked()
507 508
508 if self.length is None: 509 if self.length is None:
509 s = self.fp.read() 510 s = self.fp.read()
510 else: 511 else:
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
570 line = self.fp.readline(_MAXLINE + 1) 571 line = self.fp.readline(_MAXLINE + 1)
571 if len(line) > _MAXLINE: 572 if len(line) > _MAXLINE:
572 raise LineTooLong("trailer line") 573 raise LineTooLong("trailer line")
573 if not line: 574 if not line:
574 # a vanishingly small number of sites EOF without 575 # a vanishingly small number of sites EOF without
575 # sending the trailer 576 # sending the trailer
576 break 577 break
577 if line in (b'\r\n', b'\n', b''): 578 if line in (b'\r\n', b'\n', b''):
578 break 579 break
579 580
581 def _get_chunk_left(self):
582 # return self.chunk_left, reading a new chunk if necessary.
583 # chunk_left == 0: at the end of the current chunk, need to close it
584 # chunk_left == None: No current chunk, should read next.
585 # This function returns non-zero or None if the last chunk has
586 # been read.
587 chunk_left = self.chunk_left
588 if not chunk_left: # Can be 0 or None
589 if chunk_left is not None:
590 # We are at the end of chunk. dicard chunk end
591 self._safe_read(2) # toss the CRLF at the end of the chunk
592 try:
593 chunk_left = self._read_next_chunk_size()
594 except ValueError:
595 raise IncompleteRead(b'')
596 if chunk_left == 0:
597 # last chunk: 1*("0") [ chunk-extension ] CRLF
598 self._read_and_discard_trailer()
599 # we read everything; close the "file"
600 self._close_conn()
601 chunk_left = None
602 self.chunk_left = chunk_left
603 return chunk_left
604
580 def _readall_chunked(self): 605 def _readall_chunked(self):
581 assert self.chunked != _UNKNOWN 606 assert self.chunked != _UNKNOWN
582 chunk_left = self.chunk_left
583 value = [] 607 value = []
584 while True: 608 try:
585 if chunk_left is None: 609 while True:
586 try: 610 chunk_left = self._get_chunk_left()
587 chunk_left = self._read_next_chunk_size() 611 if chunk_left is None:
588 if chunk_left == 0: 612 break
589 break 613 value.append(self._safe_read(chunk_left))
590 except ValueError: 614 self.chunk_left = 0
591 raise IncompleteRead(b''.join(value)) 615 return b''.join(value)
592 value.append(self._safe_read(chunk_left)) 616 except IncompleteRead:
593 617 raise IncompleteRead(b''.join(value))
594 # we read the whole chunk, get another
595 self._safe_read(2) # toss the CRLF at the end of the chunk
596 chunk_left = None
597
598 self._read_and_discard_trailer()
599
600 # we read everything; close the "file"
601 self._close_conn()
602
603 return b''.join(value)
604 618
605 def _readinto_chunked(self, b): 619 def _readinto_chunked(self, b):
606 assert self.chunked != _UNKNOWN 620 assert self.chunked != _UNKNOWN
607 chunk_left = self.chunk_left
608
609 total_bytes = 0 621 total_bytes = 0
610 mvb = memoryview(b) 622 mvb = memoryview(b)
611 while True: 623 try:
612 if chunk_left is None: 624 while True:
613 try: 625 chunk_left = self._get_chunk_left()
614 chunk_left = self._read_next_chunk_size() 626 if chunk_left is None:
615 if chunk_left == 0: 627 return total_bytes
616 break 628
617 except ValueError: 629 if len(mvb) <= chunk_left:
618 raise IncompleteRead(bytes(b[0:total_bytes])) 630 n = self._safe_readinto(mvb)
619 631 self.chunk_left = chunk_left - n
620 if len(mvb) < chunk_left: 632 return total_bytes + n
621 n = self._safe_readinto(mvb) 633
622 self.chunk_left = chunk_left - n 634 temp_mvb = mvb[:chunk_left]
623 return total_bytes + n
624 elif len(mvb) == chunk_left:
625 n = self._safe_readinto(mvb)
626 self._safe_read(2) # toss the CRLF at the end of the chunk
627 self.chunk_left = None
628 return total_bytes + n
629 else:
630 temp_mvb = mvb[0:chunk_left]
631 n = self._safe_readinto(temp_mvb) 635 n = self._safe_readinto(temp_mvb)
632 mvb = mvb[n:] 636 mvb = mvb[n:]
633 total_bytes += n 637 total_bytes += n
634 638 self.chunk_left = 0
635 # we read the whole chunk, get another 639
636 self._safe_read(2) # toss the CRLF at the end of the chunk 640 except IncompleteRead:
637 chunk_left = None 641 raise IncompleteRead(bytes(b[0:total_bytes]))
638
639 self._read_and_discard_trailer()
640
641 # we read everything; close the "file"
642 self._close_conn()
643
644 return total_bytes
645 642
646 def _safe_read(self, amt): 643 def _safe_read(self, amt):
647 """Read the number of bytes requested, compensating for partial reads. 644 """Read the number of bytes requested, compensating for partial reads.
648 645
649 Normally, we have a blocking socket, but a read() can be interrupted 646 Normally, we have a blocking socket, but a read() can be interrupted
650 by a signal (resulting in a partial read). 647 by a signal (resulting in a partial read).
651 648
652 Note that we cannot distinguish between EOF and an interrupt when zero 649 Note that we cannot distinguish between EOF and an interrupt when zero
653 bytes have been read. IncompleteRead() will be raised in this 650 bytes have been read. IncompleteRead() will be raised in this
654 situation. 651 situation.
(...skipping 19 matching lines...) Expand all
674 if MAXAMOUNT < len(mvb): 671 if MAXAMOUNT < len(mvb):
675 temp_mvb = mvb[0:MAXAMOUNT] 672 temp_mvb = mvb[0:MAXAMOUNT]
676 n = self.fp.readinto(temp_mvb) 673 n = self.fp.readinto(temp_mvb)
677 else: 674 else:
678 n = self.fp.readinto(mvb) 675 n = self.fp.readinto(mvb)
679 if not n: 676 if not n:
680 raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b)) 677 raise IncompleteRead(bytes(mvb[0:total_bytes]), len(b))
681 mvb = mvb[n:] 678 mvb = mvb[n:]
682 total_bytes += n 679 total_bytes += n
683 return total_bytes 680 return total_bytes
681
682 def read1(self, n=-1):
683 """Read with at most one underlying system call. If at least one
684 byte is buffered, return that instead.
685 """
686 if self.fp is None or self._method == "HEAD":
687 return b""
688 if self.chunked:
689 return self._read1_chunked(n)
690 try:
691 result = self.fp.read1(n)
692 except ValueError:
693 if n >= 0:
694 raise
695 # some implementations, like BufferedReader, don't support -1
696 # Read an arbitrarily selected largeish chunk.
697 result = self.fp.read1(16*1024)
698 if not result and n:
699 self._close_conn()
700 return result
701
702 def peek(self, n=-1):
703 # Having this enables IOBase.readline() to read more than one
704 # byte at a time
705 if self.fp is None or self._method == "HEAD":
706 return b""
707 if self.chunked:
708 return self._peek_chunked(n)
709 return self.fp.peek(n)
710
711 def readline(self, limit=-1):
712 if self.fp is None or self._method == "HEAD":
713 return b""
714 if self.chunked:
715 # Fallback to IOBase readline which uses peek() and read()
716 return super().readline(limit)
717 result = self.fp.readline(limit)
718 if not result and limit:
719 self._close_conn()
720 return result
721
722 def _read1_chunked(self, n):
723 # Strictly speaking, _get_chunk_left() may cause more than one read,
724 # but that is ok, since that is to satisfy the chunked protocol.
725 chunk_left = self._get_chunk_left()
726 if chunk_left is None or n == 0:
727 return b''
728 if not (0 <= n <= chunk_left):
729 n = chunk_left # if n is negative or larger than chunk_left
730 read = self.fp.read1(n)
731 self.chunk_left -= len(read)
732 if not read:
733 raise IncompleteRead(b"")
734 return read
735
736 def _peek_chunked(self, n):
737 # Strictly speaking, _get_chunk_left() may cause more than one read,
738 # but that is ok, since that is to satisfy the chunked protocol.
739 try:
740 chunk_left = self._get_chunk_left()
741 except IncompleteRead:
742 return b'' # peek doesn't worry about protocol
743 if chunk_left is None:
744 return b'' # eof
745 # peek is allowed to return more than requested. Just request the
746 # entire chunk, and truncate what we get.
747 return self.fp.peek(chunk_left)[:chunk_left]
684 748
685 def fileno(self): 749 def fileno(self):
686 return self.fp.fileno() 750 return self.fp.fileno()
687 751
688 def getheader(self, name, default=None): 752 def getheader(self, name, default=None):
689 if self.headers is None: 753 if self.headers is None:
690 raise ResponseNotReady() 754 raise ResponseNotReady()
691 headers = self.headers.get_all(name) or default 755 headers = self.headers.get_all(name) or default
692 if isinstance(headers, str) or not hasattr(headers, '__iter__'): 756 if isinstance(headers, str) or not hasattr(headers, '__iter__'):
693 return headers 757 return headers
(...skipping 569 matching lines...) Expand 10 before | Expand all | Expand 10 after
1263 class IncompleteRead(HTTPException): 1327 class IncompleteRead(HTTPException):
1264 def __init__(self, partial, expected=None): 1328 def __init__(self, partial, expected=None):
1265 self.args = partial, 1329 self.args = partial,
1266 self.partial = partial 1330 self.partial = partial
1267 self.expected = expected 1331 self.expected = expected
1268 def __repr__(self): 1332 def __repr__(self):
1269 if self.expected is not None: 1333 if self.expected is not None:
1270 e = ', %i more expected' % self.expected 1334 e = ', %i more expected' % self.expected
1271 else: 1335 else:
1272 e = '' 1336 e = ''
1273 return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e) 1337 return '%s(%i bytes read%s)' % (self.__class__.__name__,
1338 len(self.partial), e)
1274 def __str__(self): 1339 def __str__(self):
1275 return repr(self) 1340 return repr(self)
1276 1341
1277 class ImproperConnectionState(HTTPException): 1342 class ImproperConnectionState(HTTPException):
1278 pass 1343 pass
1279 1344
1280 class CannotSendRequest(ImproperConnectionState): 1345 class CannotSendRequest(ImproperConnectionState):
1281 pass 1346 pass
1282 1347
1283 class CannotSendHeader(ImproperConnectionState): 1348 class CannotSendHeader(ImproperConnectionState):
1284 pass 1349 pass
1285 1350
1286 class ResponseNotReady(ImproperConnectionState): 1351 class ResponseNotReady(ImproperConnectionState):
1287 pass 1352 pass
1288 1353
1289 class BadStatusLine(HTTPException): 1354 class BadStatusLine(HTTPException):
1290 def __init__(self, line): 1355 def __init__(self, line):
1291 if not line: 1356 if not line:
1292 line = repr(line) 1357 line = repr(line)
1293 self.args = line, 1358 self.args = line,
1294 self.line = line 1359 self.line = line
1295 1360
1296 class LineTooLong(HTTPException): 1361 class LineTooLong(HTTPException):
1297 def __init__(self, line_type): 1362 def __init__(self, line_type):
1298 HTTPException.__init__(self, "got more than %d bytes when reading %s" 1363 HTTPException.__init__(self, "got more than %d bytes when reading %s"
1299 % (_MAXLINE, line_type)) 1364 % (_MAXLINE, line_type))
1300 1365
1301 # for backwards compatibility 1366 # for backwards compatibility
1302 error = HTTPException 1367 error = HTTPException
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+