Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(85723)

Delta Between Two Patch Sets: Lib/test/pickletester.py

Issue 16510: Using appropriate checks in tests
Left Patch Set: Created 5 years, 9 months ago
Right Patch Set: Created 5 years, 6 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « no previous file | Lib/test/test_abc.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 import copyreg
1 import io 2 import io
2 import unittest
3 import pickle 3 import pickle
4 import pickletools 4 import pickletools
5 import random
6 import struct
5 import sys 7 import sys
6 import copyreg 8 import unittest
7 import weakref 9 import weakref
8 from http.cookies import SimpleCookie 10 from http.cookies import SimpleCookie
9 11
10 from test.support import ( 12 from test.support import (
11 TestFailed, TESTFN, run_with_locale, no_tracing, 13 TestFailed, TESTFN, run_with_locale, no_tracing,
12 _2G, _4G, bigmemtest, 14 _2G, _4G, bigmemtest,
13 ) 15 )
14 16
15 from pickle import bytes_types 17 from pickle import bytes_types
16 18
17 # Tests that try a number of pickle protocols should have a 19 # Tests that try a number of pickle protocols should have a
18 # for proto in protocols: 20 # for proto in protocols:
19 # kind of outer loop. 21 # kind of outer loop.
20 protocols = range(pickle.HIGHEST_PROTOCOL + 1) 22 protocols = range(pickle.HIGHEST_PROTOCOL + 1)
21
22 ascii_char_size = 1
23 23
24 24
25 # Return True if opcode code appears in the pickle, else False. 25 # Return True if opcode code appears in the pickle, else False.
26 def opcode_in_pickle(code, pickle): 26 def opcode_in_pickle(code, pickle):
27 for op, dummy, dummy in pickletools.genops(pickle): 27 for op, dummy, dummy in pickletools.genops(pickle):
28 if op.code == code.decode("latin-1"): 28 if op.code == code.decode("latin-1"):
29 return True 29 return True
30 return False 30 return False
31 31
32 # Return the number of times opcode code appears in pickle. 32 # Return the number of times opcode code appears in pickle.
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after
88 return self.__dict__ == other.__dict__ 88 return self.__dict__ == other.__dict__
89 89
90 class D(C): 90 class D(C):
91 def __init__(self, arg): 91 def __init__(self, arg):
92 pass 92 pass
93 93
94 class E(C): 94 class E(C):
95 def __getinitargs__(self): 95 def __getinitargs__(self):
96 return () 96 return ()
97 97
98 class H(object):
99 pass
100
98 import __main__ 101 import __main__
99 __main__.C = C 102 __main__.C = C
100 C.__module__ = "__main__" 103 C.__module__ = "__main__"
101 __main__.D = D 104 __main__.D = D
102 D.__module__ = "__main__" 105 D.__module__ = "__main__"
103 __main__.E = E 106 __main__.E = E
104 E.__module__ = "__main__" 107 E.__module__ = "__main__"
108 __main__.H = H
109 H.__module__ = "__main__"
105 110
106 class myint(int): 111 class myint(int):
107 def __init__(self, x): 112 def __init__(self, x):
108 self.str = str(x) 113 self.str = str(x)
109 114
110 class initarg(C): 115 class initarg(C):
111 116
112 def __init__(self, a, b): 117 def __init__(self, a, b):
113 self.a = a 118 self.a = a
114 self.b = b 119 self.b = b
(...skipping 285 matching lines...) Expand 10 before | Expand all | Expand 10 after
400 DATA5 = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key' 405 DATA5 = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
401 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U' 406 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
402 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07' 407 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
403 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U' 408 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
404 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b' 409 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
405 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.') 410 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
406 411
407 # set([3]) pickled from 2.x with protocol 2 412 # set([3]) pickled from 2.x with protocol 2
408 DATA6 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.' 413 DATA6 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
409 414
415 python2_exceptions_without_args = (
416 ArithmeticError,
417 AssertionError,
418 AttributeError,
419 BaseException,
420 BufferError,
421 BytesWarning,
422 DeprecationWarning,
423 EOFError,
424 EnvironmentError,
425 Exception,
426 FloatingPointError,
427 FutureWarning,
428 GeneratorExit,
429 IOError,
430 ImportError,
431 ImportWarning,
432 IndentationError,
433 IndexError,
434 KeyError,
435 KeyboardInterrupt,
436 LookupError,
437 MemoryError,
438 NameError,
439 NotImplementedError,
440 OSError,
441 OverflowError,
442 PendingDeprecationWarning,
443 ReferenceError,
444 RuntimeError,
445 RuntimeWarning,
446 # StandardError is gone in Python 3, we map it to Exception
447 StopIteration,
448 SyntaxError,
449 SyntaxWarning,
450 SystemError,
451 SystemExit,
452 TabError,
453 TypeError,
454 UnboundLocalError,
455 UnicodeError,
456 UnicodeWarning,
457 UserWarning,
458 ValueError,
459 Warning,
460 ZeroDivisionError,
461 )
462
463 exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
464
465 # Exception objects without arguments pickled from 2.x with protocol 2
466 DATA7 = {
467 exception :
468 exception_pickle.replace(b'?', exception.__name__.encode("ascii"))
469 for exception in python2_exceptions_without_args
470 }
471
472 # StandardError is mapped to Exception, test that separately
473 DATA8 = exception_pickle.replace(b'?', b'StandardError')
474
475 # UnicodeEncodeError object pickled from 2.x with protocol 2
476 DATA9 = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
477 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
478 b'U\x03badq\x03tq\x04Rq\x05.')
479
410 480
411 def create_data(): 481 def create_data():
412 c = C() 482 c = C()
413 c.foo = 1 483 c.foo = 1
414 c.bar = 2 484 c.bar = 2
415 x = [0, 1, 2.0, 3.0+0j] 485 x = [0, 1, 2.0, 3.0+0j]
416 # Append some integer test cases at cPickle.c's internal size 486 # Append some integer test cases at cPickle.c's internal size
417 # cutoffs. 487 # cutoffs.
418 uint1max = 0xff 488 uint1max = 0xff
419 uint2max = 0xffff 489 uint2max = 0xffff
420 int4max = 0x7fffffff 490 int4max = 0x7fffffff
421 x.extend([1, -1, 491 x.extend([1, -1,
422 uint1max, -uint1max, -uint1max-1, 492 uint1max, -uint1max, -uint1max-1,
423 uint2max, -uint2max, -uint2max-1, 493 uint2max, -uint2max, -uint2max-1,
424 int4max, -int4max, -int4max-1]) 494 int4max, -int4max, -int4max-1])
425 y = ('abc', 'abc', c, c) 495 y = ('abc', 'abc', c, c)
426 x.append(y) 496 x.append(y)
427 x.append(y) 497 x.append(y)
428 x.append(5) 498 x.append(5)
429 return x 499 return x
430 500
501
431 class AbstractPickleTests(unittest.TestCase): 502 class AbstractPickleTests(unittest.TestCase):
432 # Subclass must define self.dumps, self.loads. 503 # Subclass must define self.dumps, self.loads.
433 504
505 optimized = False
506
434 _testdata = create_data() 507 _testdata = create_data()
435 508
436 def setUp(self): 509 def setUp(self):
437 pass 510 pass
511
512 def assert_is_copy(self, obj, objcopy, msg=None):
513 """Utility method to verify if two objects are copies of each others.
514 """
515 if msg is None:
516 msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
517 self.assertEqual(obj, objcopy, msg=msg)
518 self.assertIs(type(obj), type(objcopy), msg=msg)
519 if hasattr(obj, '__dict__'):
520 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
521 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
522 if hasattr(obj, '__slots__'):
523 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
524 for slot in obj.__slots__:
525 self.assertEqual(
526 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
527 self.assertEqual(getattr(obj, slot, None),
528 getattr(objcopy, slot, None), msg=msg)
438 529
439 def test_misc(self): 530 def test_misc(self):
440 # test various datatypes not tested by testdata 531 # test various datatypes not tested by testdata
441 for proto in protocols: 532 for proto in protocols:
442 x = myint(4) 533 x = myint(4)
443 s = self.dumps(x, proto) 534 s = self.dumps(x, proto)
444 y = self.loads(s) 535 y = self.loads(s)
445 self.assertEqual(x, y) 536 self.assert_is_copy(x, y)
446 537
447 x = (1, ()) 538 x = (1, ())
448 s = self.dumps(x, proto) 539 s = self.dumps(x, proto)
449 y = self.loads(s) 540 y = self.loads(s)
450 self.assertEqual(x, y) 541 self.assert_is_copy(x, y)
451 542
452 x = initarg(1, x) 543 x = initarg(1, x)
453 s = self.dumps(x, proto) 544 s = self.dumps(x, proto)
454 y = self.loads(s) 545 y = self.loads(s)
455 self.assertEqual(x, y) 546 self.assert_is_copy(x, y)
456 547
457 # XXX test __reduce__ protocol? 548 # XXX test __reduce__ protocol?
458 549
459 def test_roundtrip_equality(self): 550 def test_roundtrip_equality(self):
460 expected = self._testdata 551 expected = self._testdata
461 for proto in protocols: 552 for proto in protocols:
462 s = self.dumps(expected, proto) 553 s = self.dumps(expected, proto)
463 got = self.loads(s) 554 got = self.loads(s)
464 self.assertEqual(expected, got) 555 self.assert_is_copy(expected, got)
465 556
466 def test_load_from_data0(self): 557 def test_load_from_data0(self):
467 self.assertEqual(self._testdata, self.loads(DATA0)) 558 self.assert_is_copy(self._testdata, self.loads(DATA0))
468 559
469 def test_load_from_data1(self): 560 def test_load_from_data1(self):
470 self.assertEqual(self._testdata, self.loads(DATA1)) 561 self.assert_is_copy(self._testdata, self.loads(DATA1))
471 562
472 def test_load_from_data2(self): 563 def test_load_from_data2(self):
473 self.assertEqual(self._testdata, self.loads(DATA2)) 564 self.assert_is_copy(self._testdata, self.loads(DATA2))
474 565
475 def test_load_classic_instance(self): 566 def test_load_classic_instance(self):
476 # See issue5180. Test loading 2.x pickles that 567 # See issue5180. Test loading 2.x pickles that
477 # contain an instance of old style class. 568 # contain an instance of old style class.
478 for X, args in [(C, ()), (D, ('x',)), (E, ())]: 569 for X, args in [(C, ()), (D, ('x',)), (E, ())]:
479 xname = X.__name__.encode('ascii') 570 xname = X.__name__.encode('ascii')
480 # Protocol 0 (text mode pickle): 571 # Protocol 0 (text mode pickle):
481 """ 572 """
482 0: ( MARK 573 0: ( MARK
483 1: i INST '__main__ X' (MARK at 0) 574 1: i INST '__main__ X' (MARK at 0)
484 15: p PUT 0 575 15: p PUT 0
485 18: ( MARK 576 18: ( MARK
486 19: d DICT (MARK at 18) 577 19: d DICT (MARK at 18)
487 20: p PUT 1 578 20: p PUT 1
488 23: b BUILD 579 23: b BUILD
489 24: . STOP 580 24: . STOP
490 """ 581 """
491 pickle0 = (b"(i__main__\n" 582 pickle0 = (b"(i__main__\n"
492 b"X\n" 583 b"X\n"
493 b"p0\n" 584 b"p0\n"
494 b"(dp1\nb.").replace(b'X', xname) 585 b"(dp1\nb.").replace(b'X', xname)
495 self.assertEqual(X(*args), self.loads(pickle0)) 586 self.assert_is_copy(X(*args), self.loads(pickle0))
496 587
497 # Protocol 1 (binary mode pickle) 588 # Protocol 1 (binary mode pickle)
498 """ 589 """
499 0: ( MARK 590 0: ( MARK
500 1: c GLOBAL '__main__ X' 591 1: c GLOBAL '__main__ X'
501 15: q BINPUT 0 592 15: q BINPUT 0
502 17: o OBJ (MARK at 0) 593 17: o OBJ (MARK at 0)
503 18: q BINPUT 1 594 18: q BINPUT 1
504 20: } EMPTY_DICT 595 20: } EMPTY_DICT
505 21: q BINPUT 2 596 21: q BINPUT 2
506 23: b BUILD 597 23: b BUILD
507 24: . STOP 598 24: . STOP
508 """ 599 """
509 pickle1 = (b'(c__main__\n' 600 pickle1 = (b'(c__main__\n'
510 b'X\n' 601 b'X\n'
511 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 602 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
512 self.assertEqual(X(*args), self.loads(pickle1)) 603 self.assert_is_copy(X(*args), self.loads(pickle1))
513 604
514 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1) 605 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
515 """ 606 """
516 0: \x80 PROTO 2 607 0: \x80 PROTO 2
517 2: ( MARK 608 2: ( MARK
518 3: c GLOBAL '__main__ X' 609 3: c GLOBAL '__main__ X'
519 17: q BINPUT 0 610 17: q BINPUT 0
520 19: o OBJ (MARK at 2) 611 19: o OBJ (MARK at 2)
521 20: q BINPUT 1 612 20: q BINPUT 1
522 22: } EMPTY_DICT 613 22: } EMPTY_DICT
523 23: q BINPUT 2 614 23: q BINPUT 2
524 25: b BUILD 615 25: b BUILD
525 26: . STOP 616 26: . STOP
526 """ 617 """
527 pickle2 = (b'\x80\x02(c__main__\n' 618 pickle2 = (b'\x80\x02(c__main__\n'
528 b'X\n' 619 b'X\n'
529 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 620 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
530 self.assertEqual(X(*args), self.loads(pickle2)) 621 self.assert_is_copy(X(*args), self.loads(pickle2))
531 622
532 # There are gratuitous differences between pickles produced by 623 # There are gratuitous differences between pickles produced by
533 # pickle and cPickle, largely because cPickle starts PUT indices at 624 # pickle and cPickle, largely because cPickle starts PUT indices at
534 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() -- 625 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() --
535 # there's a comment with an exclamation point there whose meaning 626 # there's a comment with an exclamation point there whose meaning
536 # is a mystery. cPickle also suppresses PUT for objects with a refcount 627 # is a mystery. cPickle also suppresses PUT for objects with a refcount
537 # of 1. 628 # of 1.
538 def dont_test_disassembly(self): 629 def dont_test_disassembly(self):
539 from io import StringIO 630 from io import StringIO
540 from pickletools import dis 631 from pickletools import dis
541 632
542 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS): 633 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
543 s = self.dumps(self._testdata, proto) 634 s = self.dumps(self._testdata, proto)
544 filelike = StringIO() 635 filelike = StringIO()
545 dis(s, out=filelike) 636 dis(s, out=filelike)
546 got = filelike.getvalue() 637 got = filelike.getvalue()
547 self.assertEqual(expected, got) 638 self.assertEqual(expected, got)
548 639
549 def test_recursive_list(self): 640 def test_recursive_list(self):
550 l = [] 641 l = []
551 l.append(l) 642 l.append(l)
552 for proto in protocols: 643 for proto in protocols:
553 s = self.dumps(l, proto) 644 s = self.dumps(l, proto)
554 x = self.loads(s) 645 x = self.loads(s)
646 self.assertIsInstance(x, list)
555 self.assertEqual(len(x), 1) 647 self.assertEqual(len(x), 1)
556 self.assertIs(x, x[0]) 648 self.assertIs(x, x[0])
557 649
558 def test_recursive_tuple(self): 650 def test_recursive_tuple(self):
559 t = ([],) 651 t = ([],)
560 t[0].append(t) 652 t[0].append(t)
561 for proto in protocols: 653 for proto in protocols:
562 s = self.dumps(t, proto) 654 s = self.dumps(t, proto)
563 x = self.loads(s) 655 x = self.loads(s)
656 self.assertIsInstance(x, tuple)
564 self.assertEqual(len(x), 1) 657 self.assertEqual(len(x), 1)
565 self.assertEqual(len(x[0]), 1) 658 self.assertEqual(len(x[0]), 1)
566 self.assertIs(x, x[0][0]) 659 self.assertIs(x, x[0][0])
567 660
568 def test_recursive_dict(self): 661 def test_recursive_dict(self):
569 d = {} 662 d = {}
570 d[1] = d 663 d[1] = d
571 for proto in protocols: 664 for proto in protocols:
572 s = self.dumps(d, proto) 665 s = self.dumps(d, proto)
573 x = self.loads(s) 666 x = self.loads(s)
667 self.assertIsInstance(x, dict)
574 self.assertEqual(list(x.keys()), [1]) 668 self.assertEqual(list(x.keys()), [1])
575 self.assertIs(x[1], x) 669 self.assertIs(x[1], x)
670
671 def test_recursive_set(self):
672 h = H()
673 y = set({h})
674 h.attr = y
675 for proto in protocols:
676 s = self.dumps(y, proto)
677 x = self.loads(s)
678 self.assertIsInstance(x, set)
679 self.assertIs(list(x)[0].attr, x)
680 self.assertEqual(len(x), 1)
681
682 def test_recursive_frozenset(self):
683 h = H()
684 y = frozenset({h})
685 h.attr = y
686 for proto in protocols:
687 s = self.dumps(y, proto)
688 x = self.loads(s)
689 self.assertIsInstance(x, frozenset)
690 self.assertIs(list(x)[0].attr, x)
691 self.assertEqual(len(x), 1)
576 692
577 def test_recursive_inst(self): 693 def test_recursive_inst(self):
578 i = C() 694 i = C()
579 i.attr = i 695 i.attr = i
580 for proto in protocols: 696 for proto in protocols:
581 s = self.dumps(i, proto) 697 s = self.dumps(i, proto)
582 x = self.loads(s) 698 x = self.loads(s)
699 self.assertIsInstance(x, C)
583 self.assertEqual(dir(x), dir(i)) 700 self.assertEqual(dir(x), dir(i))
584 self.assertIs(x.attr, x) 701 self.assertIs(x.attr, x)
585 702
586 def test_recursive_multi(self): 703 def test_recursive_multi(self):
587 l = [] 704 l = []
588 d = {1:l} 705 d = {1:l}
589 i = C() 706 i = C()
590 i.attr = d 707 i.attr = d
591 l.append(i) 708 l.append(i)
592 for proto in protocols: 709 for proto in protocols:
593 s = self.dumps(l, proto) 710 s = self.dumps(l, proto)
594 x = self.loads(s) 711 x = self.loads(s)
712 self.assertIsInstance(x, list)
595 self.assertEqual(len(x), 1) 713 self.assertEqual(len(x), 1)
596 self.assertEqual(dir(x[0]), dir(i)) 714 self.assertEqual(dir(x[0]), dir(i))
597 self.assertEqual(list(x[0].attr.keys()), [1]) 715 self.assertEqual(list(x[0].attr.keys()), [1])
598 self.assertIs(x[0].attr[1], x) 716 self.assertIs(x[0].attr[1], x)
599 717
600 def test_get(self): 718 def test_get(self):
601 self.assertRaises(KeyError, self.loads, b'g0\np0') 719 self.assertRaises(KeyError, self.loads, b'g0\np0')
602 self.assertEqual(self.loads(b'((Kdtp0\nh\x00l.))'), [(100,), (100,)]) 720 self.assert_is_copy([(100,), (100,)],
721 self.loads(b'((Kdtp0\nh\x00l.))'))
603 722
604 def test_unicode(self): 723 def test_unicode(self):
605 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>', 724 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
606 '<\\>', '<\\\U00012345>', 725 '<\\>', '<\\\U00012345>',
607 # surrogates 726 # surrogates
608 '<\udc80>'] 727 '<\udc80>']
609 for proto in protocols: 728 for proto in protocols:
610 for u in endcases: 729 for u in endcases:
611 p = self.dumps(u, proto) 730 p = self.dumps(u, proto)
612 u2 = self.loads(p) 731 u2 = self.loads(p)
613 self.assertEqual(u2, u) 732 self.assert_is_copy(u, u2)
614 733
615 def test_unicode_high_plane(self): 734 def test_unicode_high_plane(self):
616 t = '\U00012345' 735 t = '\U00012345'
617 for proto in protocols: 736 for proto in protocols:
618 p = self.dumps(t, proto) 737 p = self.dumps(t, proto)
619 t2 = self.loads(p) 738 t2 = self.loads(p)
620 self.assertEqual(t2, t) 739 self.assert_is_copy(t, t2)
621 740
622 def test_bytes(self): 741 def test_bytes(self):
623 for proto in protocols: 742 for proto in protocols:
624 for s in b'', b'xyz', b'xyz'*100: 743 for s in b'', b'xyz', b'xyz'*100:
625 p = self.dumps(s, proto) 744 p = self.dumps(s, proto)
626 self.assertEqual(self.loads(p), s) 745 self.assert_is_copy(s, self.loads(p))
627 for s in [bytes([i]) for i in range(256)]: 746 for s in [bytes([i]) for i in range(256)]:
628 p = self.dumps(s, proto) 747 p = self.dumps(s, proto)
629 self.assertEqual(self.loads(p), s) 748 self.assert_is_copy(s, self.loads(p))
630 for s in [bytes([i, i]) for i in range(256)]: 749 for s in [bytes([i, i]) for i in range(256)]:
631 p = self.dumps(s, proto) 750 p = self.dumps(s, proto)
632 self.assertEqual(self.loads(p), s) 751 self.assert_is_copy(s, self.loads(p))
633 752
634 def test_ints(self): 753 def test_ints(self):
635 import sys 754 import sys
636 for proto in protocols: 755 for proto in protocols:
637 n = sys.maxsize 756 n = sys.maxsize
638 while n: 757 while n:
639 for expected in (-n, n): 758 for expected in (-n, n):
640 s = self.dumps(expected, proto) 759 s = self.dumps(expected, proto)
641 n2 = self.loads(s) 760 n2 = self.loads(s)
642 self.assertEqual(expected, n2) 761 self.assert_is_copy(expected, n2)
643 n = n >> 1 762 n = n >> 1
644 763
645 def test_maxint64(self): 764 def test_maxint64(self):
646 maxint64 = (1 << 63) - 1 765 maxint64 = (1 << 63) - 1
647 data = b'I' + str(maxint64).encode("ascii") + b'\n.' 766 data = b'I' + str(maxint64).encode("ascii") + b'\n.'
648 got = self.loads(data) 767 got = self.loads(data)
649 self.assertEqual(got, maxint64) 768 self.assert_is_copy(maxint64, got)
650 769
651 # Try too with a bogus literal. 770 # Try too with a bogus literal.
652 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' 771 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
653 self.assertRaises(ValueError, self.loads, data) 772 self.assertRaises(ValueError, self.loads, data)
654 773
655 def test_long(self): 774 def test_long(self):
656 for proto in protocols: 775 for proto in protocols:
657 # 256 bytes is where LONG4 begins. 776 # 256 bytes is where LONG4 begins.
658 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257: 777 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257:
659 nbase = 1 << nbits 778 nbase = 1 << nbits
660 for npos in nbase-1, nbase, nbase+1: 779 for npos in nbase-1, nbase, nbase+1:
661 for n in npos, -npos: 780 for n in npos, -npos:
662 pickle = self.dumps(n, proto) 781 pickle = self.dumps(n, proto)
663 got = self.loads(pickle) 782 got = self.loads(pickle)
664 self.assertEqual(n, got) 783 self.assert_is_copy(n, got)
665 # Try a monster. This is quadratic-time in protos 0 & 1, so don't 784 # Try a monster. This is quadratic-time in protos 0 & 1, so don't
666 # bother with those. 785 # bother with those.
667 nbase = int("deadbeeffeedface", 16) 786 nbase = int("deadbeeffeedface", 16)
668 nbase += nbase << 1000000 787 nbase += nbase << 1000000
669 for n in nbase, -nbase: 788 for n in nbase, -nbase:
670 p = self.dumps(n, 2) 789 p = self.dumps(n, 2)
671 got = self.loads(p) 790 got = self.loads(p)
791 # assert_is_copy is very expensive here as it precomputes
792 # a failure message by computing the repr() of n and got,
793 # we just do the check ourselves.
794 self.assertIs(type(got), int)
672 self.assertEqual(n, got) 795 self.assertEqual(n, got)
673 796
674 def test_float(self): 797 def test_float(self):
675 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5, 798 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
676 3.14, 263.44582062374053, 6.022e23, 1e30] 799 3.14, 263.44582062374053, 6.022e23, 1e30]
677 test_values = test_values + [-x for x in test_values] 800 test_values = test_values + [-x for x in test_values]
678 for proto in protocols: 801 for proto in protocols:
679 for value in test_values: 802 for value in test_values:
680 pickle = self.dumps(value, proto) 803 pickle = self.dumps(value, proto)
681 got = self.loads(pickle) 804 got = self.loads(pickle)
682 self.assertEqual(value, got) 805 self.assert_is_copy(value, got)
683 806
684 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') 807 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
685 def test_float_format(self): 808 def test_float_format(self):
686 # make sure that floats are formatted locale independent with proto 0 809 # make sure that floats are formatted locale independent with proto 0
687 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') 810 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
688 811
689 def test_reduce(self): 812 def test_reduce(self):
690 pass 813 for proto in protocols:
814 inst = AAA()
815 dumped = self.dumps(inst, proto)
816 loaded = self.loads(dumped)
817 self.assertEqual(loaded, REDUCE_A)
691 818
692 def test_getinitargs(self): 819 def test_getinitargs(self):
693 pass 820 for proto in protocols:
821 inst = initarg(1, 2)
822 dumped = self.dumps(inst, proto)
823 loaded = self.loads(dumped)
824 self.assert_is_copy(inst, loaded)
694 825
695 def test_pop_empty_stack(self): 826 def test_pop_empty_stack(self):
696 # Test issue7455 827 # Test issue7455
697 s = b'0' 828 s = b'0'
698 self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s) 829 self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s)
699 830
700 def test_metaclass(self): 831 def test_metaclass(self):
701 a = use_metaclass() 832 a = use_metaclass()
702 for proto in protocols: 833 for proto in protocols:
703 s = self.dumps(a, proto) 834 s = self.dumps(a, proto)
704 b = self.loads(s) 835 b = self.loads(s)
705 self.assertEqual(a.__class__, b.__class__) 836 self.assertEqual(a.__class__, b.__class__)
706 837
707 def test_dynamic_class(self): 838 def test_dynamic_class(self):
708 a = create_dynamic_class("my_dynamic_class", (object,)) 839 a = create_dynamic_class("my_dynamic_class", (object,))
709 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) 840 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
710 for proto in protocols: 841 for proto in protocols:
711 s = self.dumps(a, proto) 842 s = self.dumps(a, proto)
712 b = self.loads(s) 843 b = self.loads(s)
713 self.assertEqual(a, b) 844 self.assertEqual(a, b)
845 self.assertIs(type(a), type(b))
714 846
715 def test_structseq(self): 847 def test_structseq(self):
716 import time 848 import time
717 import os 849 import os
718 850
719 t = time.localtime() 851 t = time.localtime()
720 for proto in protocols: 852 for proto in protocols:
721 s = self.dumps(t, proto) 853 s = self.dumps(t, proto)
722 u = self.loads(s) 854 u = self.loads(s)
723 self.assertEqual(t, u) 855 self.assert_is_copy(t, u)
724 if hasattr(os, "stat"): 856 if hasattr(os, "stat"):
725 t = os.stat(os.curdir) 857 t = os.stat(os.curdir)
726 s = self.dumps(t, proto) 858 s = self.dumps(t, proto)
727 u = self.loads(s) 859 u = self.loads(s)
728 self.assertEqual(t, u) 860 self.assert_is_copy(t, u)
729 if hasattr(os, "statvfs"): 861 if hasattr(os, "statvfs"):
730 t = os.statvfs(os.curdir) 862 t = os.statvfs(os.curdir)
731 s = self.dumps(t, proto) 863 s = self.dumps(t, proto)
732 u = self.loads(s) 864 u = self.loads(s)
733 self.assertEqual(t, u) 865 self.assert_is_copy(t, u)
734 866
735 def test_ellipsis(self): 867 def test_ellipsis(self):
736 for proto in protocols: 868 for proto in protocols:
737 s = self.dumps(..., proto) 869 s = self.dumps(..., proto)
738 u = self.loads(s) 870 u = self.loads(s)
739 self.assertEqual(..., u) 871 self.assertIs(..., u)
740 872
741 def test_notimplemented(self): 873 def test_notimplemented(self):
742 for proto in protocols: 874 for proto in protocols:
743 s = self.dumps(NotImplemented, proto) 875 s = self.dumps(NotImplemented, proto)
744 u = self.loads(s) 876 u = self.loads(s)
745 self.assertEqual(NotImplemented, u) 877 self.assertIs(NotImplemented, u)
878
879 def test_singleton_types(self):
880 # Issue #6477: Test that types of built-in singletons can be pickled.
881 singletons = [None, ..., NotImplemented]
882 for singleton in singletons:
883 for proto in protocols:
884 s = self.dumps(type(singleton), proto)
885 u = self.loads(s)
886 self.assertIs(type(singleton), u)
746 887
747 # Tests for protocol 2 888 # Tests for protocol 2
748 889
749 def test_proto(self): 890 def test_proto(self):
891 for proto in protocols:
892 pickled = self.dumps(None, proto)
893 if proto >= 2:
894 proto_header = pickle.PROTO + bytes([proto])
895 self.assertTrue(pickled.startswith(proto_header))
896 else:
897 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
898
899 oob = protocols[-1] + 1 # a future protocol
750 build_none = pickle.NONE + pickle.STOP 900 build_none = pickle.NONE + pickle.STOP
751 for proto in protocols:
752 expected = build_none
753 if proto >= 2:
754 expected = pickle.PROTO + bytes([proto]) + expected
755 p = self.dumps(None, proto)
756 self.assertEqual(p, expected)
757
758 oob = protocols[-1] + 1 # a future protocol
759 badpickle = pickle.PROTO + bytes([oob]) + build_none 901 badpickle = pickle.PROTO + bytes([oob]) + build_none
760 try: 902 try:
761 self.loads(badpickle) 903 self.loads(badpickle)
762 except ValueError as detail: 904 except ValueError as err:
763 self.assertTrue(str(detail).startswith( 905 self.assertIn("unsupported pickle protocol", str(err))
764 "unsupported pickle protocol"))
765 else: 906 else:
766 self.fail("expected bad protocol number to raise ValueError") 907 self.fail("expected bad protocol number to raise ValueError")
767 908
768 def test_long1(self): 909 def test_long1(self):
769 x = 12345678910111213141516178920 910 x = 12345678910111213141516178920
770 for proto in protocols: 911 for proto in protocols:
771 s = self.dumps(x, proto) 912 s = self.dumps(x, proto)
772 y = self.loads(s) 913 y = self.loads(s)
773 self.assertEqual(x, y) 914 self.assert_is_copy(x, y)
774 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2) 915 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
775 916
776 def test_long4(self): 917 def test_long4(self):
777 x = 12345678910111213141516178920 << (256*8) 918 x = 12345678910111213141516178920 << (256*8)
778 for proto in protocols: 919 for proto in protocols:
779 s = self.dumps(x, proto) 920 s = self.dumps(x, proto)
780 y = self.loads(s) 921 y = self.loads(s)
781 self.assertEqual(x, y) 922 self.assert_is_copy(x, y)
782 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2) 923 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
783 924
784 def test_short_tuples(self): 925 def test_short_tuples(self):
785 # Map (proto, len(tuple)) to expected opcode. 926 # Map (proto, len(tuple)) to expected opcode.
786 expected_opcode = {(0, 0): pickle.TUPLE, 927 expected_opcode = {(0, 0): pickle.TUPLE,
787 (0, 1): pickle.TUPLE, 928 (0, 1): pickle.TUPLE,
788 (0, 2): pickle.TUPLE, 929 (0, 2): pickle.TUPLE,
789 (0, 3): pickle.TUPLE, 930 (0, 3): pickle.TUPLE,
790 (0, 4): pickle.TUPLE, 931 (0, 4): pickle.TUPLE,
791 932
(...skipping 17 matching lines...) Expand all
809 } 950 }
810 a = () 951 a = ()
811 b = (1,) 952 b = (1,)
812 c = (1, 2) 953 c = (1, 2)
813 d = (1, 2, 3) 954 d = (1, 2, 3)
814 e = (1, 2, 3, 4) 955 e = (1, 2, 3, 4)
815 for proto in protocols: 956 for proto in protocols:
816 for x in a, b, c, d, e: 957 for x in a, b, c, d, e:
817 s = self.dumps(x, proto) 958 s = self.dumps(x, proto)
818 y = self.loads(s) 959 y = self.loads(s)
819 self.assertEqual(x, y, (proto, x, s, y)) 960 self.assert_is_copy(x, y)
820 expected = expected_opcode[proto, len(x)] 961 expected = expected_opcode[min(proto, 3), len(x)]
821 self.assertEqual(opcode_in_pickle(expected, s), True) 962 self.assertTrue(opcode_in_pickle(expected, s))
822 963
823 def test_singletons(self): 964 def test_singletons(self):
824 # Map (proto, singleton) to expected opcode. 965 # Map (proto, singleton) to expected opcode.
825 expected_opcode = {(0, None): pickle.NONE, 966 expected_opcode = {(0, None): pickle.NONE,
826 (1, None): pickle.NONE, 967 (1, None): pickle.NONE,
827 (2, None): pickle.NONE, 968 (2, None): pickle.NONE,
828 (3, None): pickle.NONE, 969 (3, None): pickle.NONE,
829 970
830 (0, True): pickle.INT, 971 (0, True): pickle.INT,
831 (1, True): pickle.INT, 972 (1, True): pickle.INT,
832 (2, True): pickle.NEWTRUE, 973 (2, True): pickle.NEWTRUE,
833 (3, True): pickle.NEWTRUE, 974 (3, True): pickle.NEWTRUE,
834 975
835 (0, False): pickle.INT, 976 (0, False): pickle.INT,
836 (1, False): pickle.INT, 977 (1, False): pickle.INT,
837 (2, False): pickle.NEWFALSE, 978 (2, False): pickle.NEWFALSE,
838 (3, False): pickle.NEWFALSE, 979 (3, False): pickle.NEWFALSE,
839 } 980 }
840 for proto in protocols: 981 for proto in protocols:
841 for x in None, False, True: 982 for x in None, False, True:
842 s = self.dumps(x, proto) 983 s = self.dumps(x, proto)
843 y = self.loads(s) 984 y = self.loads(s)
844 self.assertIs(x, y, (proto, x, s, y)) 985 self.assertIs(x, y, (proto, x, s, y))
845 expected = expected_opcode[proto, x] 986 expected = expected_opcode[min(proto, 3), x]
846 self.assertEqual(opcode_in_pickle(expected, s), True) 987 self.assertEqual(opcode_in_pickle(expected, s), True)
847 988
848 def test_newobj_tuple(self): 989 def test_newobj_tuple(self):
849 x = MyTuple([1, 2, 3]) 990 x = MyTuple([1, 2, 3])
850 x.foo = 42 991 x.foo = 42
851 x.bar = "hello" 992 x.bar = "hello"
852 for proto in protocols: 993 for proto in protocols:
853 s = self.dumps(x, proto) 994 s = self.dumps(x, proto)
854 y = self.loads(s) 995 y = self.loads(s)
855 self.assertEqual(tuple(x), tuple(y)) 996 self.assert_is_copy(x, y)
856 self.assertEqual(x.__dict__, y.__dict__)
857 997
858 def test_newobj_list(self): 998 def test_newobj_list(self):
859 x = MyList([1, 2, 3]) 999 x = MyList([1, 2, 3])
860 x.foo = 42 1000 x.foo = 42
861 x.bar = "hello" 1001 x.bar = "hello"
862 for proto in protocols: 1002 for proto in protocols:
863 s = self.dumps(x, proto) 1003 s = self.dumps(x, proto)
864 y = self.loads(s) 1004 y = self.loads(s)
865 self.assertEqual(list(x), list(y)) 1005 self.assert_is_copy(x, y)
866 self.assertEqual(x.__dict__, y.__dict__)
867 1006
868 def test_newobj_generic(self): 1007 def test_newobj_generic(self):
869 for proto in protocols: 1008 for proto in protocols:
870 for C in myclasses: 1009 for C in myclasses:
871 B = C.__base__ 1010 B = C.__base__
872 x = C(C.sample) 1011 x = C(C.sample)
873 x.foo = 42 1012 x.foo = 42
874 s = self.dumps(x, proto) 1013 s = self.dumps(x, proto)
875 y = self.loads(s) 1014 y = self.loads(s)
876 detail = (proto, C, B, x, y, type(y)) 1015 detail = (proto, C, B, x, y, type(y))
1016 self.assert_is_copy(x, y) # XXX revisit
877 self.assertEqual(B(x), B(y), detail) 1017 self.assertEqual(B(x), B(y), detail)
878 self.assertEqual(x.__dict__, y.__dict__, detail) 1018 self.assertEqual(x.__dict__, y.__dict__, detail)
879 1019
880 def test_newobj_proxies(self): 1020 def test_newobj_proxies(self):
881 # NEWOBJ should use the __class__ rather than the raw type 1021 # NEWOBJ should use the __class__ rather than the raw type
882 classes = myclasses[:] 1022 classes = myclasses[:]
883 # Cannot create weakproxies to these classes 1023 # Cannot create weakproxies to these classes
884 for c in (MyInt, MyTuple): 1024 for c in (MyInt, MyTuple):
885 classes.remove(c) 1025 classes.remove(c)
886 for proto in protocols: 1026 for proto in protocols:
(...skipping 18 matching lines...) Expand all
905 try: 1045 try:
906 copyreg.add_extension(__name__, "MyList", extcode) 1046 copyreg.add_extension(__name__, "MyList", extcode)
907 x = MyList([1, 2, 3]) 1047 x = MyList([1, 2, 3])
908 x.foo = 42 1048 x.foo = 42
909 x.bar = "hello" 1049 x.bar = "hello"
910 1050
911 # Dump using protocol 1 for comparison. 1051 # Dump using protocol 1 for comparison.
912 s1 = self.dumps(x, 1) 1052 s1 = self.dumps(x, 1)
913 self.assertIn(__name__.encode("utf-8"), s1) 1053 self.assertIn(__name__.encode("utf-8"), s1)
914 self.assertIn(b"MyList", s1) 1054 self.assertIn(b"MyList", s1)
915 self.assertEqual(opcode_in_pickle(opcode, s1), False) 1055 self.assertFalse(opcode_in_pickle(opcode, s1))
916 1056
917 y = self.loads(s1) 1057 y = self.loads(s1)
918 self.assertEqual(list(x), list(y)) 1058 self.assert_is_copy(x, y)
919 self.assertEqual(x.__dict__, y.__dict__)
920 1059
921 # Dump using protocol 2 for test. 1060 # Dump using protocol 2 for test.
922 s2 = self.dumps(x, 2) 1061 s2 = self.dumps(x, 2)
923 self.assertNotIn(__name__.encode("utf-8"), s2) 1062 self.assertNotIn(__name__.encode("utf-8"), s2)
924 self.assertNotIn(b"MyList", s2) 1063 self.assertNotIn(b"MyList", s2)
925 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) 1064 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
926 1065
927 y = self.loads(s2) 1066 y = self.loads(s2)
928 self.assertEqual(list(x), list(y)) 1067 self.assert_is_copy(x, y)
929 self.assertEqual(x.__dict__, y.__dict__)
930
931 finally: 1068 finally:
932 e.restore() 1069 e.restore()
933 1070
934 def test_global_ext1(self): 1071 def test_global_ext1(self):
935 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code 1072 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code
936 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code 1073 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code
937 1074
938 def test_global_ext2(self): 1075 def test_global_ext2(self):
939 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code 1076 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code
940 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code 1077 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code
941 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness 1078 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness
942 1079
943 def test_global_ext4(self): 1080 def test_global_ext4(self):
944 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code 1081 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code
945 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code 1082 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code
946 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness 1083 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness
947 1084
948 def test_list_chunking(self): 1085 def test_list_chunking(self):
949 n = 10 # too small to chunk 1086 n = 10 # too small to chunk
950 x = list(range(n)) 1087 x = list(range(n))
951 for proto in protocols: 1088 for proto in protocols:
952 s = self.dumps(x, proto) 1089 s = self.dumps(x, proto)
953 y = self.loads(s) 1090 y = self.loads(s)
954 self.assertEqual(x, y) 1091 self.assert_is_copy(x, y)
955 num_appends = count_opcode(pickle.APPENDS, s) 1092 num_appends = count_opcode(pickle.APPENDS, s)
956 self.assertEqual(num_appends, proto > 0) 1093 self.assertEqual(num_appends, proto > 0)
957 1094
958 n = 2500 # expect at least two chunks when proto > 0 1095 n = 2500 # expect at least two chunks when proto > 0
959 x = list(range(n)) 1096 x = list(range(n))
960 for proto in protocols: 1097 for proto in protocols:
961 s = self.dumps(x, proto) 1098 s = self.dumps(x, proto)
962 y = self.loads(s) 1099 y = self.loads(s)
963 self.assertEqual(x, y) 1100 self.assert_is_copy(x, y)
964 num_appends = count_opcode(pickle.APPENDS, s) 1101 num_appends = count_opcode(pickle.APPENDS, s)
965 if proto == 0: 1102 if proto == 0:
966 self.assertEqual(num_appends, 0) 1103 self.assertEqual(num_appends, 0)
967 else: 1104 else:
968 self.assertGreaterEqual(num_appends, 2) 1105 self.assertGreaterEqual(num_appends, 2)
969 1106
970 def test_dict_chunking(self): 1107 def test_dict_chunking(self):
971 n = 10 # too small to chunk 1108 n = 10 # too small to chunk
972 x = dict.fromkeys(range(n)) 1109 x = dict.fromkeys(range(n))
973 for proto in protocols: 1110 for proto in protocols:
974 s = self.dumps(x, proto) 1111 s = self.dumps(x, proto)
975 self.assertIsInstance(s, bytes_types) 1112 self.assertIsInstance(s, bytes_types)
976 y = self.loads(s) 1113 y = self.loads(s)
977 self.assertEqual(x, y) 1114 self.assert_is_copy(x, y)
978 num_setitems = count_opcode(pickle.SETITEMS, s) 1115 num_setitems = count_opcode(pickle.SETITEMS, s)
979 self.assertEqual(num_setitems, proto > 0) 1116 self.assertEqual(num_setitems, proto > 0)
980 1117
981 n = 2500 # expect at least two chunks when proto > 0 1118 n = 2500 # expect at least two chunks when proto > 0
982 x = dict.fromkeys(range(n)) 1119 x = dict.fromkeys(range(n))
983 for proto in protocols: 1120 for proto in protocols:
984 s = self.dumps(x, proto) 1121 s = self.dumps(x, proto)
985 y = self.loads(s) 1122 y = self.loads(s)
986 self.assertEqual(x, y) 1123 self.assert_is_copy(x, y)
987 num_setitems = count_opcode(pickle.SETITEMS, s) 1124 num_setitems = count_opcode(pickle.SETITEMS, s)
988 if proto == 0: 1125 if proto == 0:
989 self.assertEqual(num_setitems, 0) 1126 self.assertEqual(num_setitems, 0)
990 else: 1127 else:
991 self.assertGreaterEqual(num_setitems, 2) 1128 self.assertGreaterEqual(num_setitems, 2)
992 1129
1130 def test_set_chunking(self):
1131 n = 10 # too small to chunk
1132 x = set(range(n))
1133 for proto in protocols:
1134 s = self.dumps(x, proto)
1135 y = self.loads(s)
1136 self.assert_is_copy(x, y)
1137 num_additems = count_opcode(pickle.ADDITEMS, s)
1138 if proto < 4:
1139 self.assertEqual(num_additems, 0)
1140 else:
1141 self.assertEqual(num_additems, 1)
1142
1143 n = 2500 # expect at least two chunks when proto >= 4
1144 x = set(range(n))
1145 for proto in protocols:
1146 s = self.dumps(x, proto)
1147 y = self.loads(s)
1148 self.assert_is_copy(x, y)
1149 num_additems = count_opcode(pickle.ADDITEMS, s)
1150 if proto < 4:
1151 self.assertEqual(num_additems, 0)
1152 else:
1153 self.assertGreaterEqual(num_additems, 2)
1154
993 def test_simple_newobj(self): 1155 def test_simple_newobj(self):
994 x = object.__new__(SimpleNewObj) # avoid __init__ 1156 x = object.__new__(SimpleNewObj) # avoid __init__
995 x.abc = 666 1157 x.abc = 666
996 for proto in protocols: 1158 for proto in protocols:
997 s = self.dumps(x, proto) 1159 s = self.dumps(x, proto)
998 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), proto >= 2) 1160 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
1161 2 <= proto < 4)
1162 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
1163 proto >= 4)
999 y = self.loads(s) # will raise TypeError if __init__ called 1164 y = self.loads(s) # will raise TypeError if __init__ called
1000 self.assertEqual(y.abc, 666) 1165 self.assert_is_copy(x, y)
1001 self.assertEqual(x.__dict__, y.__dict__)
1002 1166
1003 def test_newobj_list_slots(self): 1167 def test_newobj_list_slots(self):
1004 x = SlotList([1, 2, 3]) 1168 x = SlotList([1, 2, 3])
1005 x.foo = 42 1169 x.foo = 42
1006 x.bar = "hello" 1170 x.bar = "hello"
1007 s = self.dumps(x, 2) 1171 s = self.dumps(x, 2)
1008 y = self.loads(s) 1172 y = self.loads(s)
1009 self.assertEqual(list(x), list(y)) 1173 self.assert_is_copy(x, y)
1010 self.assertEqual(x.__dict__, y.__dict__)
1011 self.assertEqual(x.foo, y.foo)
1012 self.assertEqual(x.bar, y.bar)
1013 1174
1014 def test_reduce_overrides_default_reduce_ex(self): 1175 def test_reduce_overrides_default_reduce_ex(self):
1015 for proto in protocols: 1176 for proto in protocols:
1016 x = REX_one() 1177 x = REX_one()
1017 self.assertEqual(x._reduce_called, 0) 1178 self.assertEqual(x._reduce_called, 0)
1018 s = self.dumps(x, proto) 1179 s = self.dumps(x, proto)
1019 self.assertEqual(x._reduce_called, 1) 1180 self.assertEqual(x._reduce_called, 1)
1020 y = self.loads(s) 1181 y = self.loads(s)
1021 self.assertEqual(y._reduce_called, 0) 1182 self.assertEqual(y._reduce_called, 0)
1022 1183
(...skipping 28 matching lines...) Expand all
1051 for proto in protocols: 1212 for proto in protocols:
1052 x = REX_five() 1213 x = REX_five()
1053 self.assertEqual(x._reduce_called, 0) 1214 self.assertEqual(x._reduce_called, 0)
1054 s = self.dumps(x, proto) 1215 s = self.dumps(x, proto)
1055 self.assertEqual(x._reduce_called, 1) 1216 self.assertEqual(x._reduce_called, 1)
1056 y = self.loads(s) 1217 y = self.loads(s)
1057 self.assertEqual(y._reduce_called, 1) 1218 self.assertEqual(y._reduce_called, 1)
1058 1219
1059 @no_tracing 1220 @no_tracing
1060 def test_bad_getattr(self): 1221 def test_bad_getattr(self):
1222 # Issue #3514: crash when there is an infinite loop in __getattr__
1061 x = BadGetattr() 1223 x = BadGetattr()
1062 for proto in 0, 1: 1224 for proto in protocols:
1063 self.assertRaises(RuntimeError, self.dumps, x, proto) 1225 self.assertRaises(RuntimeError, self.dumps, x, proto)
1064 # protocol 2 don't raise a RuntimeError.
1065 d = self.dumps(x, 2)
1066 1226
1067 def test_reduce_bad_iterator(self): 1227 def test_reduce_bad_iterator(self):
1068 # Issue4176: crash when 4th and 5th items of __reduce__() 1228 # Issue4176: crash when 4th and 5th items of __reduce__()
1069 # are not iterators 1229 # are not iterators
1070 class C(object): 1230 class C(object):
1071 def __reduce__(self): 1231 def __reduce__(self):
1072 # 4th item is not an iterator 1232 # 4th item is not an iterator
1073 return list, (), None, [], None 1233 return list, (), None, [], None
1074 class D(object): 1234 class D(object):
1075 def __reduce__(self): 1235 def __reduce__(self):
(...skipping 12 matching lines...) Expand all
1088 pass 1248 pass
1089 1249
1090 def test_many_puts_and_gets(self): 1250 def test_many_puts_and_gets(self):
1091 # Test that internal data structures correctly deal with lots of 1251 # Test that internal data structures correctly deal with lots of
1092 # puts/gets. 1252 # puts/gets.
1093 keys = ("aaa" + str(i) for i in range(100)) 1253 keys = ("aaa" + str(i) for i in range(100))
1094 large_dict = dict((k, [4, 5, 6]) for k in keys) 1254 large_dict = dict((k, [4, 5, 6]) for k in keys)
1095 obj = [dict(large_dict), dict(large_dict), dict(large_dict)] 1255 obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
1096 1256
1097 for proto in protocols: 1257 for proto in protocols:
1098 dumped = self.dumps(obj, proto) 1258 with self.subTest(proto=proto):
1099 loaded = self.loads(dumped) 1259 dumped = self.dumps(obj, proto)
1100 self.assertEqual(loaded, obj, 1260 loaded = self.loads(dumped)
1101 "Failed protocol %d: %r != %r" 1261 self.assert_is_copy(obj, loaded)
1102 % (proto, obj, loaded))
1103 1262
1104 def test_attribute_name_interning(self): 1263 def test_attribute_name_interning(self):
1105 # Test that attribute names of pickled objects are interned when 1264 # Test that attribute names of pickled objects are interned when
1106 # unpickling. 1265 # unpickling.
1107 for proto in protocols: 1266 for proto in protocols:
1108 x = C() 1267 x = C()
1109 x.foo = 42 1268 x.foo = 42
1110 x.bar = "hello" 1269 x.bar = "hello"
1111 s = self.dumps(x, proto) 1270 s = self.dumps(x, proto)
1112 y = self.loads(s) 1271 y = self.loads(s)
1113 x_keys = sorted(x.__dict__) 1272 x_keys = sorted(x.__dict__)
1114 y_keys = sorted(y.__dict__) 1273 y_keys = sorted(y.__dict__)
1115 for x_key, y_key in zip(x_keys, y_keys): 1274 for x_key, y_key in zip(x_keys, y_keys):
1116 self.assertIs(x_key, y_key) 1275 self.assertIs(x_key, y_key)
1117 1276
1118 def test_unpickle_from_2x(self): 1277 def test_unpickle_from_2x(self):
1119 # Unpickle non-trivial data from Python 2.x. 1278 # Unpickle non-trivial data from Python 2.x.
1120 loaded = self.loads(DATA3) 1279 loaded = self.loads(DATA3)
1121 self.assertEqual(loaded, set([1, 2])) 1280 self.assertEqual(loaded, set([1, 2]))
1122 loaded = self.loads(DATA4) 1281 loaded = self.loads(DATA4)
1123 self.assertEqual(type(loaded), type(range(0))) 1282 self.assertEqual(type(loaded), type(range(0)))
1124 self.assertEqual(list(loaded), list(range(5))) 1283 self.assertEqual(list(loaded), list(range(5)))
1125 loaded = self.loads(DATA5) 1284 loaded = self.loads(DATA5)
1126 self.assertEqual(type(loaded), SimpleCookie) 1285 self.assertEqual(type(loaded), SimpleCookie)
1127 self.assertEqual(list(loaded.keys()), ["key"]) 1286 self.assertEqual(list(loaded.keys()), ["key"])
1128 self.assertEqual(loaded["key"].value, "Set-Cookie: key=value") 1287 self.assertEqual(loaded["key"].value, "Set-Cookie: key=value")
1129 1288
1289 for (exc, data) in DATA7.items():
1290 loaded = self.loads(data)
1291 self.assertIs(type(loaded), exc)
1292
1293 loaded = self.loads(DATA8)
1294 self.assertIs(type(loaded), Exception)
1295
1296 loaded = self.loads(DATA9)
1297 self.assertIs(type(loaded), UnicodeEncodeError)
1298 self.assertEqual(loaded.object, "foo")
1299 self.assertEqual(loaded.encoding, "ascii")
1300 self.assertEqual(loaded.start, 0)
1301 self.assertEqual(loaded.end, 1)
1302 self.assertEqual(loaded.reason, "bad")
1303
1130 def test_pickle_to_2x(self): 1304 def test_pickle_to_2x(self):
1131 # Pickle non-trivial data with protocol 2, expecting that it yields 1305 # Pickle non-trivial data with protocol 2, expecting that it yields
1132 # the same result as Python 2.x did. 1306 # the same result as Python 2.x did.
1133 # NOTE: this test is a bit too strong since we can produce different 1307 # NOTE: this test is a bit too strong since we can produce different
1134 # bytecode that 2.x will still understand. 1308 # bytecode that 2.x will still understand.
1135 dumped = self.dumps(range(5), 2) 1309 dumped = self.dumps(range(5), 2)
1136 self.assertEqual(dumped, DATA4) 1310 self.assertEqual(dumped, DATA4)
1137 dumped = self.dumps(set([3]), 2) 1311 dumped = self.dumps(set([3]), 2)
1138 self.assertEqual(dumped, DATA6) 1312 self.assertEqual(dumped, DATA6)
1139 1313
1314 def test_load_python2_str_as_bytes(self):
1315 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
1316 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
1317 encoding="bytes"), b'a\x00\xa0')
1318 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
1319 self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
1320 encoding="bytes"), b'a\x00\xa0')
1321 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
1322 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
1323 encoding="bytes"), b'a\x00\xa0')
1324
1325 def test_load_python2_unicode_as_str(self):
1326 # From Python 2: pickle.dumps(u'π', protocol=0)
1327 self.assertEqual(self.loads(b'V\\u03c0\n.',
1328 encoding='bytes'), 'π')
1329 # From Python 2: pickle.dumps(u'π', protocol=1)
1330 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
1331 encoding="bytes"), 'π')
1332 # From Python 2: pickle.dumps(u'π', protocol=2)
1333 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
1334 encoding="bytes"), 'π')
1335
1336 def test_load_long_python2_str_as_bytes(self):
1337 # From Python 2: pickle.dumps('x' * 300, protocol=1)
1338 self.assertEqual(self.loads(pickle.BINSTRING +
1339 struct.pack("<I", 300) +
1340 b'x' * 300 + pickle.STOP,
1341 encoding='bytes'), b'x' * 300)
1342
1140 def test_large_pickles(self): 1343 def test_large_pickles(self):
1141 # Test the correctness of internal buffering routines when handling 1344 # Test the correctness of internal buffering routines when handling
1142 # large data. 1345 # large data.
1143 for proto in protocols: 1346 for proto in protocols:
1144 data = (1, min, b'xy' * (30 * 1024), len) 1347 data = (1, min, b'xy' * (30 * 1024), len)
1145 dumped = self.dumps(data, proto) 1348 dumped = self.dumps(data, proto)
1146 loaded = self.loads(dumped) 1349 loaded = self.loads(dumped)
1147 self.assertEqual(len(loaded), len(data)) 1350 self.assertEqual(len(loaded), len(data))
1148 self.assertEqual(loaded, data) 1351 self.assertEqual(loaded, data)
1149 1352
1150 def test_empty_bytestring(self): 1353 def test_empty_bytestring(self):
1151 # issue 11286 1354 # issue 11286
1152 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') 1355 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
1153 self.assertEqual(empty, '') 1356 self.assertEqual(empty, '')
1154 1357
1155 def test_int_pickling_efficiency(self): 1358 def test_int_pickling_efficiency(self):
1156 # Test compacity of int representation (see issue #12744) 1359 # Test compacity of int representation (see issue #12744)
1157 for proto in protocols: 1360 for proto in protocols:
1158 sizes = [len(self.dumps(2**n, proto)) for n in range(70)] 1361 with self.subTest(proto=proto):
1159 # the size function is monotonic 1362 pickles = [self.dumps(2**n, proto) for n in range(70)]
1160 self.assertEqual(sorted(sizes), sizes) 1363 sizes = list(map(len, pickles))
1161 if proto >= 2: 1364 # the size function is monotonic
1162 self.assertLessEqual(sizes[-1], 14) 1365 self.assertEqual(sorted(sizes), sizes)
1366 if proto >= 2:
1367 for p in pickles:
1368 self.assertFalse(opcode_in_pickle(pickle.LONG, p))
1163 1369
1164 def check_negative_32b_binXXX(self, dumped): 1370 def check_negative_32b_binXXX(self, dumped):
1165 if sys.maxsize > 2**32: 1371 if sys.maxsize > 2**32:
1166 self.skipTest("test is only meaningful on 32-bit builds") 1372 self.skipTest("test is only meaningful on 32-bit builds")
1167 # XXX Pure Python pickle reads lengths as signed and passes 1373 # XXX Pure Python pickle reads lengths as signed and passes
1168 # them directly to read() (hence the EOFError) 1374 # them directly to read() (hence the EOFError)
1169 with self.assertRaises((pickle.UnpicklingError, EOFError, 1375 with self.assertRaises((pickle.UnpicklingError, EOFError,
1170 ValueError, OverflowError)): 1376 ValueError, OverflowError)):
1171 self.loads(dumped) 1377 self.loads(dumped)
1172 1378
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after
1235 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) 1441 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto)
1236 1442
1237 def test_setitems_on_non_dicts(self): 1443 def test_setitems_on_non_dicts(self):
1238 obj = REX_seven({1: -1, 2: -2, 3: -3}) 1444 obj = REX_seven({1: -1, 2: -2, 3: -3})
1239 for proto in protocols: 1445 for proto in protocols:
1240 if proto == 0: 1446 if proto == 0:
1241 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto) 1447 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto)
1242 else: 1448 else:
1243 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto) 1449 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
1244 1450
1451 # Exercise framing (proto >= 4) for significant workloads
1452
1453 FRAME_SIZE_TARGET = 64 * 1024
1454
1455 def check_frame_opcodes(self, pickled):
1456 """
1457 Check the arguments of FRAME opcodes in a protocol 4+ pickle.
1458 """
1459 frame_opcode_size = 9
1460 last_arg = last_pos = None
1461 for op, arg, pos in pickletools.genops(pickled):
1462 if op.name != 'FRAME':
1463 continue
1464 if last_pos is not None:
1465 # The previous frame's size should be equal to the number
1466 # of bytes up to the current frame.
1467 frame_size = pos - last_pos - frame_opcode_size
1468 self.assertEqual(frame_size, last_arg)
1469 last_arg, last_pos = arg, pos
1470 # The last frame's size should be equal to the number of bytes up
1471 # to the pickle's end.
1472 frame_size = len(pickled) - last_pos - frame_opcode_size
1473 self.assertEqual(frame_size, last_arg)
1474
1475 def test_framing_many_objects(self):
1476 obj = list(range(10**5))
1477 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1478 with self.subTest(proto=proto):
1479 pickled = self.dumps(obj, proto)
1480 unpickled = self.loads(pickled)
1481 self.assertEqual(obj, unpickled)
1482 bytes_per_frame = (len(pickled) /
1483 count_opcode(pickle.FRAME, pickled))
1484 self.assertGreater(bytes_per_frame,
1485 self.FRAME_SIZE_TARGET / 2)
1486 self.assertLessEqual(bytes_per_frame,
1487 self.FRAME_SIZE_TARGET * 1)
1488 self.check_frame_opcodes(pickled)
1489
1490 def test_framing_large_objects(self):
1491 N = 1024 * 1024
1492 obj = [b'x' * N, b'y' * N, b'z' * N]
1493 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1494 with self.subTest(proto=proto):
1495 pickled = self.dumps(obj, proto)
1496 unpickled = self.loads(pickled)
1497 self.assertEqual(obj, unpickled)
1498 n_frames = count_opcode(pickle.FRAME, pickled)
1499 self.assertGreaterEqual(n_frames, len(obj))
1500 self.check_frame_opcodes(pickled)
1501
1502 def test_optional_frames(self):
1503 if pickle.HIGHEST_PROTOCOL < 4:
1504 return
1505
1506 def remove_frames(pickled, keep_frame=None):
1507 """Remove frame opcodes from the given pickle."""
1508 frame_starts = []
1509 # 1 byte for the opcode and 8 for the argument
1510 frame_opcode_size = 9
1511 for opcode, _, pos in pickletools.genops(pickled):
1512 if opcode.name == 'FRAME':
1513 frame_starts.append(pos)
1514
1515 newpickle = bytearray()
1516 last_frame_end = 0
1517 for i, pos in enumerate(frame_starts):
1518 if keep_frame and keep_frame(i):
1519 continue
1520 newpickle += pickled[last_frame_end:pos]
1521 last_frame_end = pos + frame_opcode_size
1522 newpickle += pickled[last_frame_end:]
1523 return newpickle
1524
1525 frame_size = self.FRAME_SIZE_TARGET
1526 num_frames = 20
1527 obj = [bytes([i]) * frame_size for i in range(num_frames)]
1528
1529 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1530 pickled = self.dumps(obj, proto)
1531
1532 frameless_pickle = remove_frames(pickled)
1533 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
1534 self.assertEqual(obj, self.loads(frameless_pickle))
1535
1536 some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
1537 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
1538 count_opcode(pickle.FRAME, pickled))
1539 self.assertEqual(obj, self.loads(some_frames_pickle))
1540
1541 def test_nested_names(self):
1542 global Nested
1543 class Nested:
1544 class A:
1545 class B:
1546 class C:
1547 pass
1548
1549 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1550 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
1551 with self.subTest(proto=proto, obj=obj):
1552 unpickled = self.loads(self.dumps(obj, proto))
1553 self.assertIs(obj, unpickled)
1554
1555 def test_py_methods(self):
1556 global PyMethodsTest
1557 class PyMethodsTest:
1558 @staticmethod
1559 def cheese():
1560 return "cheese"
1561 @classmethod
1562 def wine(cls):
1563 assert cls is PyMethodsTest
1564 return "wine"
1565 def biscuits(self):
1566 assert isinstance(self, PyMethodsTest)
1567 return "biscuits"
1568 class Nested:
1569 "Nested class"
1570 @staticmethod
1571 def ketchup():
1572 return "ketchup"
1573 @classmethod
1574 def maple(cls):
1575 assert cls is PyMethodsTest.Nested
1576 return "maple"
1577 def pie(self):
1578 assert isinstance(self, PyMethodsTest.Nested)
1579 return "pie"
1580
1581 py_methods = (
1582 PyMethodsTest.cheese,
1583 PyMethodsTest.wine,
1584 PyMethodsTest().biscuits,
1585 PyMethodsTest.Nested.ketchup,
1586 PyMethodsTest.Nested.maple,
1587 PyMethodsTest.Nested().pie
1588 )
1589 py_unbound_methods = (
1590 (PyMethodsTest.biscuits, PyMethodsTest),
1591 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
1592 )
1593 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1594 for method in py_methods:
1595 with self.subTest(proto=proto, method=method):
1596 unpickled = self.loads(self.dumps(method, proto))
1597 self.assertEqual(method(), unpickled())
1598 for method, cls in py_unbound_methods:
1599 obj = cls()
1600 with self.subTest(proto=proto, method=method):
1601 unpickled = self.loads(self.dumps(method, proto))
1602 self.assertEqual(method(obj), unpickled(obj))
1603
1604 def test_c_methods(self):
1605 global Subclass
1606 class Subclass(tuple):
1607 class Nested(str):
1608 pass
1609
1610 c_methods = (
1611 # bound built-in method
1612 ("abcd".index, ("c",)),
1613 # unbound built-in method
1614 (str.index, ("abcd", "c")),
1615 # bound "slot" method
1616 ([1, 2, 3].__len__, ()),
1617 # unbound "slot" method
1618 (list.__len__, ([1, 2, 3],)),
1619 # bound "coexist" method
1620 ({1, 2}.__contains__, (2,)),
1621 # unbound "coexist" method
1622 (set.__contains__, ({1, 2}, 2)),
1623 # built-in class method
1624 (dict.fromkeys, (("a", 1), ("b", 2))),
1625 # built-in static method
1626 (bytearray.maketrans, (b"abc", b"xyz")),
1627 # subclass methods
1628 (Subclass([1,2,2]).count, (2,)),
1629 (Subclass.count, (Subclass([1,2,2]), 2)),
1630 (Subclass.Nested("sweet").count, ("e",)),
1631 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
1632 )
1633 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1634 for method, args in c_methods:
1635 with self.subTest(proto=proto, method=method):
1636 unpickled = self.loads(self.dumps(method, proto))
1637 self.assertEqual(method(*args), unpickled(*args))
1638
1245 1639
1246 class BigmemPickleTests(unittest.TestCase): 1640 class BigmemPickleTests(unittest.TestCase):
1247 1641
1248 # Binary protocols can serialize longs of up to 2GB-1 1642 # Binary protocols can serialize longs of up to 2GB-1
1249 1643
1250 @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) 1644 @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
1251 def test_huge_long_32b(self, size): 1645 def test_huge_long_32b(self, size):
1252 data = 1 << (8 * size) 1646 data = 1 << (8 * size)
1253 try: 1647 try:
1254 for proto in protocols: 1648 for proto in protocols:
1255 if proto < 2: 1649 if proto < 2:
1256 continue 1650 continue
1257 with self.assertRaises((ValueError, OverflowError)): 1651 with self.subTest(proto=proto):
1258 self.dumps(data, protocol=proto) 1652 with self.assertRaises((ValueError, OverflowError)):
1653 self.dumps(data, protocol=proto)
1259 finally: 1654 finally:
1260 data = None 1655 data = None
1261 1656
1262 # Protocol 3 can serialize up to 4GB-1 as a bytes object 1657 # Protocol 3 can serialize up to 4GB-1 as a bytes object
1263 # (older protocols don't have a dedicated opcode for bytes and are 1658 # (older protocols don't have a dedicated opcode for bytes and are
1264 # too inefficient) 1659 # too inefficient)
1265 1660
1266 @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False) 1661 @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
1267 def test_huge_bytes_32b(self, size): 1662 def test_huge_bytes_32b(self, size):
1268 data = b"abcd" * (size // 4) 1663 data = b"abcd" * (size // 4)
1269 try: 1664 try:
1270 for proto in protocols: 1665 for proto in protocols:
1271 if proto < 3: 1666 if proto < 3:
1272 continue 1667 continue
1273 try: 1668 with self.subTest(proto=proto):
1274 pickled = self.dumps(data, protocol=proto) 1669 try:
1275 self.assertIn(b"abcd", pickled[:15]) 1670 pickled = self.dumps(data, protocol=proto)
1276 self.assertIn(b"abcd", pickled[-15:]) 1671 header = (pickle.BINBYTES +
1277 finally: 1672 struct.pack("<I", len(data)))
1278 pickled = None 1673 data_start = pickled.index(data)
1674 self.assertEqual(
1675 header,
1676 pickled[data_start-len(header):data_start])
1677 finally:
1678 pickled = None
1279 finally: 1679 finally:
1280 data = None 1680 data = None
1281 1681
1282 @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False) 1682 @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
1283 def test_huge_bytes_64b(self, size): 1683 def test_huge_bytes_64b(self, size):
1284 data = b"a" * size 1684 data = b"acbd" * (size // 4)
1285 try: 1685 try:
1286 for proto in protocols: 1686 for proto in protocols:
1287 if proto < 3: 1687 if proto < 3:
1288 continue 1688 continue
1289 with self.assertRaises((ValueError, OverflowError)): 1689 with self.subTest(proto=proto):
1290 self.dumps(data, protocol=proto) 1690 if proto == 3:
1691 # Protocol 3 does not support large bytes objects.
1692 # Verify that we do not crash when processing one.
1693 with self.assertRaises((ValueError, OverflowError)):
1694 self.dumps(data, protocol=proto)
1695 continue
1696 try:
1697 pickled = self.dumps(data, protocol=proto)
1698 header = (pickle.BINBYTES8 +
1699 struct.pack("<Q", len(data)))
1700 data_start = pickled.index(data)
1701 self.assertEqual(
1702 header,
1703 pickled[data_start-len(header):data_start])
1704 finally:
1705 pickled = None
1291 finally: 1706 finally:
1292 data = None 1707 data = None
1293 1708
1294 # All protocols use 1-byte per printable ASCII character; we add another 1709 # All protocols use 1-byte per printable ASCII character; we add another
1295 # byte because the encoded form has to be copied into the internal buffer. 1710 # byte because the encoded form has to be copied into the internal buffer.
1296 1711
1297 @bigmemtest(size=_2G, memuse=2 + ascii_char_size, dry_run=False) 1712 @bigmemtest(size=_2G, memuse=8, dry_run=False)
1298 def test_huge_str_32b(self, size): 1713 def test_huge_str_32b(self, size):
1299 data = "abcd" * (size // 4) 1714 data = "abcd" * (size // 4)
1300 try: 1715 try:
1301 for proto in protocols: 1716 for proto in protocols:
1302 try: 1717 if proto == 0:
1303 pickled = self.dumps(data, protocol=proto) 1718 continue
1304 self.assertIn(b"abcd", pickled[:15]) 1719 with self.subTest(proto=proto):
1305 self.assertIn(b"abcd", pickled[-15:]) 1720 try:
1306 finally: 1721 pickled = self.dumps(data, protocol=proto)
1307 pickled = None 1722 header = (pickle.BINUNICODE +
1723 struct.pack("<I", len(data)))
1724 data_start = pickled.index(b'abcd')
1725 self.assertEqual(
1726 header,
1727 pickled[data_start-len(header):data_start])
1728 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
1729 pickled.index(b"abcd")), len(data))
1730 finally:
1731 pickled = None
1308 finally: 1732 finally:
1309 data = None 1733 data = None
1310 1734
1311 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 1735 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
1312 # 2**32 - 1 bytes of utf-8 encoded unicode. 1736 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
1313 1737 # unicode strings however.
1314 @bigmemtest(size=_4G, memuse=1 + ascii_char_size, dry_run=False) 1738
1739 @bigmemtest(size=_4G, memuse=8, dry_run=False)
1315 def test_huge_str_64b(self, size): 1740 def test_huge_str_64b(self, size):
1316 data = "a" * size 1741 data = "abcd" * (size // 4)
1317 try: 1742 try:
1318 for proto in protocols: 1743 for proto in protocols:
1319 if proto == 0: 1744 if proto == 0:
1320 continue 1745 continue
1321 with self.assertRaises((ValueError, OverflowError)): 1746 with self.subTest(proto=proto):
1322 self.dumps(data, protocol=proto) 1747 if proto < 4:
1748 with self.assertRaises((ValueError, OverflowError)):
1749 self.dumps(data, protocol=proto)
1750 continue
1751 try:
1752 pickled = self.dumps(data, protocol=proto)
1753 header = (pickle.BINUNICODE8 +
1754 struct.pack("<Q", len(data)))
1755 data_start = pickled.index(b'abcd')
1756 self.assertEqual(
1757 header,
1758 pickled[data_start-len(header):data_start])
1759 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
1760 pickled.index(b"abcd")), len(data))
1761 finally:
1762 pickled = None
1323 finally: 1763 finally:
1324 data = None 1764 data = None
1325 1765
1326 1766
1327 # Test classes for reduce_ex 1767 # Test classes for reduce_ex
1328 1768
1329 class REX_one(object): 1769 class REX_one(object):
1330 """No __reduce_ex__ here, but inheriting it from object""" 1770 """No __reduce_ex__ here, but inheriting it from object"""
1331 _reduce_called = 0 1771 _reduce_called = 0
1332 def __reduce__(self): 1772 def __reduce__(self):
(...skipping 23 matching lines...) Expand all
1356 return object.__reduce_ex__(self, proto) 1796 return object.__reduce_ex__(self, proto)
1357 1797
1358 class REX_five(object): 1798 class REX_five(object):
1359 """This one used to fail with infinite recursion""" 1799 """This one used to fail with infinite recursion"""
1360 _reduce_called = 0 1800 _reduce_called = 0
1361 def __reduce__(self): 1801 def __reduce__(self):
1362 self._reduce_called = 1 1802 self._reduce_called = 1
1363 return object.__reduce__(self) 1803 return object.__reduce__(self)
1364 1804
1365 class REX_six(object): 1805 class REX_six(object):
1366 """This class is used to check the 4th argument (list iterator) of the reduc e 1806 """This class is used to check the 4th argument (list iterator) of
1367 protocol. 1807 the reduce protocol.
1368 """ 1808 """
1369 def __init__(self, items=None): 1809 def __init__(self, items=None):
1370 self.items = items if items is not None else [] 1810 self.items = items if items is not None else []
1371 def __eq__(self, other): 1811 def __eq__(self, other):
1372 return type(self) is type(other) and self.items == self.items 1812 return type(self) is type(other) and self.items == self.items
1373 def append(self, item): 1813 def append(self, item):
1374 self.items.append(item) 1814 self.items.append(item)
1375 def __reduce__(self): 1815 def __reduce__(self):
1376 return type(self), (), None, iter(self.items), None 1816 return type(self), (), None, iter(self.items), None
1377 1817
1378 class REX_seven(object): 1818 class REX_seven(object):
1379 """This class is used to check the 5th argument (dict iterator) of the reduc e 1819 """This class is used to check the 5th argument (dict iterator) of
1380 protocol. 1820 the reduce protocol.
1381 """ 1821 """
1382 def __init__(self, table=None): 1822 def __init__(self, table=None):
1383 self.table = table if table is not None else {} 1823 self.table = table if table is not None else {}
1384 def __eq__(self, other): 1824 def __eq__(self, other):
1385 return type(self) is type(other) and self.table == self.table 1825 return type(self) is type(other) and self.table == self.table
1386 def __setitem__(self, key, value): 1826 def __setitem__(self, key, value):
1387 self.table[key] = value 1827 self.table[key] = value
1388 def __reduce__(self): 1828 def __reduce__(self):
1389 return type(self), (), None, None, iter(self.table.items()) 1829 return type(self), (), None, None, iter(self.table.items())
1390 1830
(...skipping 17 matching lines...) Expand all
1408 1848
1409 class MyTuple(tuple): 1849 class MyTuple(tuple):
1410 sample = (1, 2, 3) 1850 sample = (1, 2, 3)
1411 1851
1412 class MyList(list): 1852 class MyList(list):
1413 sample = [1, 2, 3] 1853 sample = [1, 2, 3]
1414 1854
1415 class MyDict(dict): 1855 class MyDict(dict):
1416 sample = {"a": 1, "b": 2} 1856 sample = {"a": 1, "b": 2}
1417 1857
1858 class MySet(set):
1859 sample = {"a", "b"}
1860
1861 class MyFrozenSet(frozenset):
1862 sample = frozenset({"a", "b"})
1863
1418 myclasses = [MyInt, MyFloat, 1864 myclasses = [MyInt, MyFloat,
1419 MyComplex, 1865 MyComplex,
1420 MyStr, MyUnicode, 1866 MyStr, MyUnicode,
1421 MyTuple, MyList, MyDict] 1867 MyTuple, MyList, MyDict, MySet, MyFrozenSet]
1422 1868
1423 1869
1424 class SlotList(MyList): 1870 class SlotList(MyList):
1425 __slots__ = ["foo"] 1871 __slots__ = ["foo"]
1426 1872
1427 class SimpleNewObj(object): 1873 class SimpleNewObj(object):
1428 def __init__(self, a, b, c): 1874 def __init__(self, a, b, c):
1429 # raise an error, to make sure this isn't called 1875 # raise an error, to make sure this isn't called
1430 raise TypeError("SimpleNewObj.__init__() didn't expect to get called") 1876 raise TypeError("SimpleNewObj.__init__() didn't expect to get called")
1877 def __eq__(self, other):
1878 return self.__dict__ == other.__dict__
1431 1879
1432 class BadGetattr: 1880 class BadGetattr:
1433 def __getattr__(self, key): 1881 def __getattr__(self, key):
1434 self.foo 1882 self.foo
1435 1883
1436 1884
1437 class AbstractPickleModuleTests(unittest.TestCase): 1885 class AbstractPickleModuleTests(unittest.TestCase):
1438 1886
1439 def test_dump_closed_file(self): 1887 def test_dump_closed_file(self):
1440 import os 1888 import os
(...skipping 16 matching lines...) Expand all
1457 def test_load_from_and_dump_to_file(self): 1905 def test_load_from_and_dump_to_file(self):
1458 stream = io.BytesIO() 1906 stream = io.BytesIO()
1459 data = [123, {}, 124] 1907 data = [123, {}, 124]
1460 pickle.dump(data, stream) 1908 pickle.dump(data, stream)
1461 stream.seek(0) 1909 stream.seek(0)
1462 unpickled = pickle.load(stream) 1910 unpickled = pickle.load(stream)
1463 self.assertEqual(unpickled, data) 1911 self.assertEqual(unpickled, data)
1464 1912
1465 def test_highest_protocol(self): 1913 def test_highest_protocol(self):
1466 # Of course this needs to be changed when HIGHEST_PROTOCOL changes. 1914 # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
1467 self.assertEqual(pickle.HIGHEST_PROTOCOL, 3) 1915 self.assertEqual(pickle.HIGHEST_PROTOCOL, 4)
1468 1916
1469 def test_callapi(self): 1917 def test_callapi(self):
1470 f = io.BytesIO() 1918 f = io.BytesIO()
1471 # With and without keyword arguments 1919 # With and without keyword arguments
1472 pickle.dump(123, f, -1) 1920 pickle.dump(123, f, -1)
1473 pickle.dump(123, file=f, protocol=-1) 1921 pickle.dump(123, file=f, protocol=-1)
1474 pickle.dumps(123, -1) 1922 pickle.dumps(123, -1)
1475 pickle.dumps(123, protocol=-1) 1923 pickle.dumps(123, protocol=-1)
1476 pickle.Pickler(f, -1) 1924 pickle.Pickler(f, -1)
1477 pickle.Pickler(f, protocol=-1) 1925 pickle.Pickler(f, protocol=-1)
(...skipping 19 matching lines...) Expand all
1497 class AbstractPersistentPicklerTests(unittest.TestCase): 1945 class AbstractPersistentPicklerTests(unittest.TestCase):
1498 1946
1499 # This class defines persistent_id() and persistent_load() 1947 # This class defines persistent_id() and persistent_load()
1500 # functions that should be used by the pickler. All even integers 1948 # functions that should be used by the pickler. All even integers
1501 # are pickled using persistent ids. 1949 # are pickled using persistent ids.
1502 1950
1503 def persistent_id(self, object): 1951 def persistent_id(self, object):
1504 if isinstance(object, int) and object % 2 == 0: 1952 if isinstance(object, int) and object % 2 == 0:
1505 self.id_count += 1 1953 self.id_count += 1
1506 return str(object) 1954 return str(object)
1955 elif object == "test_false_value":
1956 self.false_count += 1
1957 return ""
1507 else: 1958 else:
1508 return None 1959 return None
1509 1960
1510 def persistent_load(self, oid): 1961 def persistent_load(self, oid):
1511 self.load_count += 1 1962 if not oid:
1512 object = int(oid) 1963 self.load_false_count += 1
1513 assert object % 2 == 0 1964 return "test_false_value"
1514 return object 1965 else:
1966 self.load_count += 1
1967 object = int(oid)
1968 assert object % 2 == 0
1969 return object
1515 1970
1516 def test_persistence(self): 1971 def test_persistence(self):
1517 self.id_count = 0 1972 L = list(range(10)) + ["test_false_value"]
1518 self.load_count = 0 1973 for proto in protocols:
1519 L = list(range(10)) 1974 self.id_count = 0
1520 self.assertEqual(self.loads(self.dumps(L)), L) 1975 self.false_count = 0
1521 self.assertEqual(self.id_count, 5) 1976 self.load_false_count = 0
1522 self.assertEqual(self.load_count, 5) 1977 self.load_count = 0
1523 1978 self.assertEqual(self.loads(self.dumps(L, proto)), L)
1524 def test_bin_persistence(self): 1979 self.assertEqual(self.id_count, 5)
1525 self.id_count = 0 1980 self.assertEqual(self.false_count, 1)
1526 self.load_count = 0 1981 self.assertEqual(self.load_count, 5)
1527 L = list(range(10)) 1982 self.assertEqual(self.load_false_count, 1)
1528 self.assertEqual(self.loads(self.dumps(L, 1)), L)
1529 self.assertEqual(self.id_count, 5)
1530 self.assertEqual(self.load_count, 5)
1531 1983
1532 1984
1533 class AbstractPicklerUnpicklerObjectTests(unittest.TestCase): 1985 class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
1534 1986
1535 pickler_class = None 1987 pickler_class = None
1536 unpickler_class = None 1988 unpickler_class = None
1537 1989
1538 def setUp(self): 1990 def setUp(self):
1539 assert self.pickler_class 1991 assert self.pickler_class
1540 assert self.unpickler_class 1992 assert self.unpickler_class
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after
1638 self.assertEqual(unpickler.load(), data1) 2090 self.assertEqual(unpickler.load(), data1)
1639 2091
1640 f.seek(0) 2092 f.seek(0)
1641 f.truncate() 2093 f.truncate()
1642 f.write(pickled2) 2094 f.write(pickled2)
1643 f.seek(0) 2095 f.seek(0)
1644 self.assertEqual(unpickler.load(), data2) 2096 self.assertEqual(unpickler.load(), data2)
1645 2097
1646 def _check_multiple_unpicklings(self, ioclass): 2098 def _check_multiple_unpicklings(self, ioclass):
1647 for proto in protocols: 2099 for proto in protocols:
1648 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len] 2100 with self.subTest(proto=proto):
1649 f = ioclass() 2101 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
1650 pickler = self.pickler_class(f, protocol=proto) 2102 f = ioclass()
1651 pickler.dump(data1) 2103 pickler = self.pickler_class(f, protocol=proto)
1652 pickled = f.getvalue() 2104 pickler.dump(data1)
1653 2105 pickled = f.getvalue()
1654 N = 5 2106
1655 f = ioclass(pickled * N) 2107 N = 5
1656 unpickler = self.unpickler_class(f) 2108 f = ioclass(pickled * N)
1657 for i in range(N): 2109 unpickler = self.unpickler_class(f)
1658 if f.seekable(): 2110 for i in range(N):
1659 pos = f.tell() 2111 if f.seekable():
1660 self.assertEqual(unpickler.load(), data1) 2112 pos = f.tell()
1661 if f.seekable(): 2113 self.assertEqual(unpickler.load(), data1)
1662 self.assertEqual(f.tell(), pos + len(pickled)) 2114 if f.seekable():
1663 self.assertRaises(EOFError, unpickler.load) 2115 self.assertEqual(f.tell(), pos + len(pickled))
2116 self.assertRaises(EOFError, unpickler.load)
1664 2117
1665 def test_multiple_unpicklings_seekable(self): 2118 def test_multiple_unpicklings_seekable(self):
1666 self._check_multiple_unpicklings(io.BytesIO) 2119 self._check_multiple_unpicklings(io.BytesIO)
1667 2120
1668 def test_multiple_unpicklings_unseekable(self): 2121 def test_multiple_unpicklings_unseekable(self):
1669 self._check_multiple_unpicklings(UnseekableIO) 2122 self._check_multiple_unpicklings(UnseekableIO)
1670 2123
1671 def test_unpickling_buffering_readline(self): 2124 def test_unpickling_buffering_readline(self):
1672 # Issue #12687: the unpickler's buffering logic could fail with 2125 # Issue #12687: the unpickler's buffering logic could fail with
1673 # text mode opcodes. 2126 # text mode opcodes.
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after
1791 for j in range(0, len(p), 20): 2244 for j in range(0, len(p), 20):
1792 b = bytes(p[j:j+20]) 2245 b = bytes(p[j:j+20])
1793 print(" {0!r}".format(b)) 2246 print(" {0!r}".format(b))
1794 print(")") 2247 print(")")
1795 print() 2248 print()
1796 print("# Disassembly of DATA{0}".format(i)) 2249 print("# Disassembly of DATA{0}".format(i))
1797 print("DATA{0}_DIS = \"\"\"\\".format(i)) 2250 print("DATA{0}_DIS = \"\"\"\\".format(i))
1798 dis(p) 2251 dis(p)
1799 print("\"\"\"") 2252 print("\"\"\"")
1800 print() 2253 print()
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+