Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(27047)

Delta Between Two Patch Sets: Lib/test/support.py

Issue 13390: Hunt memory allocations in addition to reference leaks
Left Patch Set: Created 7 years, 10 months ago
Right Patch Set: Created 6 years, 9 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Lib/test/regrtest.py ('k') | Lib/test/test_sys.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 """Supporting definitions for the Python regression tests.""" 1 """Supporting definitions for the Python regression tests."""
2 2
3 if __name__ != 'test.support': 3 if __name__ != 'test.support':
4 raise ImportError('support must be imported from the test package') 4 raise ImportError('support must be imported from the test package')
5 5
6 import contextlib 6 import contextlib
7 import errno 7 import errno
8 import functools 8 import functools
9 import gc 9 import gc
10 import socket 10 import socket
11 import sys 11 import sys
12 import os 12 import os
13 import platform 13 import platform
14 import shutil 14 import shutil
15 import warnings 15 import warnings
16 import unittest 16 import unittest
17 import importlib 17 import importlib
18 import collections.abc 18 import collections.abc
19 import re 19 import re
20 import subprocess 20 import subprocess
21 import imp 21 import imp
22 import time 22 import time
23 import sysconfig 23 import sysconfig
24 import fnmatch 24 import fnmatch
25 import logging.handlers 25 import logging.handlers
26 import struct 26 import struct
27 import tempfile
28 import _testcapi
27 29
28 try: 30 try:
29 import _thread, threading 31 import _thread, threading
30 except ImportError: 32 except ImportError:
31 _thread = None 33 _thread = None
32 threading = None 34 threading = None
33 try: 35 try:
34 import multiprocessing.process 36 import multiprocessing.process
35 except ImportError: 37 except ImportError:
36 multiprocessing = None 38 multiprocessing = None
37 39
38 try: 40 try:
39 import faulthandler
40 except ImportError:
41 faulthandler = None
42
43 try:
44 import zlib 41 import zlib
45 except ImportError: 42 except ImportError:
46 zlib = None 43 zlib = None
47 44
45 try:
46 import bz2
47 except ImportError:
48 bz2 = None
49
50 try:
51 import lzma
52 except ImportError:
53 lzma = None
54
48 __all__ = [ 55 __all__ = [
49 "Error", "TestFailed", "ResourceDenied", "import_module", 56 "Error", "TestFailed", "ResourceDenied", "import_module", "verbose",
50 "verbose", "use_resources", "max_memuse", "record_original_stdout", 57 "use_resources", "max_memuse", "record_original_stdout",
51 "get_original_stdout", "unload", "unlink", "rmtree", "forget", 58 "get_original_stdout", "unload", "unlink", "rmtree", "forget",
52 "is_resource_enabled", "requires", "requires_freebsd_version", 59 "is_resource_enabled", "requires", "requires_freebsd_version",
53 "requires_linux_version", "requires_mac_ver", "find_unused_port", "bind_port ", 60 "requires_linux_version", "requires_mac_ver", "find_unused_port",
54 "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD", "temp_cwd", 61 "bind_port", "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD",
55 "findfile", "create_empty_file", "sortdict", "check_syntax_error", "open_url resource", 62 "temp_cwd", "findfile", "create_empty_file", "sortdict",
56 "check_warnings", "CleanImport", "EnvironmentVarGuard", "TransientResource", 63 "check_syntax_error", "open_urlresource", "check_warnings", "CleanImport",
57 "captured_stdout", "captured_stdin", "captured_stderr", "time_out", 64 "EnvironmentVarGuard", "TransientResource", "captured_stdout",
58 "socket_peer_reset", "ioerror_peer_reset", "run_with_locale", 'temp_umask', 65 "captured_stdin", "captured_stderr", "time_out", "socket_peer_reset",
66 "ioerror_peer_reset", "run_with_locale", 'temp_umask',
59 "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest", 67 "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest",
60 "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup", 68 "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
61 "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail", 69 "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail",
62 "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754", 70 "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754",
63 "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", 71 "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink",
64 "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE", "failfast", 72 "skip_unless_xattr", "import_fresh_module", "requires_zlib",
65 "anticipate_failure" 73 "PIPE_MAX_SIZE", "failfast", "anticipate_failure", "run_with_tz",
74 "requires_bz2", "requires_lzma"
66 ] 75 ]
67 76
68 class Error(Exception): 77 class Error(Exception):
69 """Base class for regression test exceptions.""" 78 """Base class for regression test exceptions."""
70 79
71 class TestFailed(Error): 80 class TestFailed(Error):
72 """Test failed.""" 81 """Test failed."""
73 82
74 class ResourceDenied(unittest.SkipTest): 83 class ResourceDenied(unittest.SkipTest):
75 """Test skipped because it requested a disallowed resource. 84 """Test skipped because it requested a disallowed resource.
76 85
77 This is raised when a test calls requires() for a resource that 86 This is raised when a test calls requires() for a resource that
78 has not be enabled. It is used to distinguish between expected 87 has not be enabled. It is used to distinguish between expected
79 and unexpected skips. 88 and unexpected skips.
80 """ 89 """
81 90
82 @contextlib.contextmanager 91 @contextlib.contextmanager
83 def _ignore_deprecated_imports(ignore=True): 92 def _ignore_deprecated_imports(ignore=True):
84 """Context manager to suppress package and module deprecation 93 """Context manager to suppress package and module deprecation
85 warnings when importing them. 94 warnings when importing them.
86 95
87 If ignore is False, this context manager has no effect.""" 96 If ignore is False, this context manager has no effect.
97 """
88 if ignore: 98 if ignore:
89 with warnings.catch_warnings(): 99 with warnings.catch_warnings():
90 warnings.filterwarnings("ignore", ".+ (module|package)", 100 warnings.filterwarnings("ignore", ".+ (module|package)",
91 DeprecationWarning) 101 DeprecationWarning)
92 yield 102 yield
93 else: 103 else:
94 yield 104 yield
95 105
96 106
97 def import_module(name, deprecated=False): 107 def import_module(name, deprecated=False, *, required_on=()):
98 """Import and return the module to be tested, raising SkipTest if 108 """Import and return the module to be tested, raising SkipTest if
99 it is not available. 109 it is not available.
100 110
101 If deprecated is True, any module or package deprecation messages 111 If deprecated is True, any module or package deprecation messages
102 will be suppressed.""" 112 will be suppressed. If a module is required on a platform but optional for
113 others, set required_on to an iterable of platform prefixes which will be
114 compared against sys.platform.
115 """
103 with _ignore_deprecated_imports(deprecated): 116 with _ignore_deprecated_imports(deprecated):
104 try: 117 try:
105 return importlib.import_module(name) 118 return importlib.import_module(name)
106 except ImportError as msg: 119 except ImportError as msg:
120 if sys.platform.startswith(tuple(required_on)):
121 raise
107 raise unittest.SkipTest(str(msg)) 122 raise unittest.SkipTest(str(msg))
108 123
109 124
110 def _save_and_remove_module(name, orig_modules): 125 def _save_and_remove_module(name, orig_modules):
111 """Helper function to save and remove a module from sys.modules 126 """Helper function to save and remove a module from sys.modules
112 127
113 Raise ImportError if the module can't be imported.""" 128 Raise ImportError if the module can't be imported.
129 """
114 # try to import the module and raise an error if it can't be imported 130 # try to import the module and raise an error if it can't be imported
115 if name not in sys.modules: 131 if name not in sys.modules:
116 __import__(name) 132 __import__(name)
117 del sys.modules[name] 133 del sys.modules[name]
118 for modname in list(sys.modules): 134 for modname in list(sys.modules):
119 if modname == name or modname.startswith(name + '.'): 135 if modname == name or modname.startswith(name + '.'):
120 orig_modules[modname] = sys.modules[modname] 136 orig_modules[modname] = sys.modules[modname]
121 del sys.modules[modname] 137 del sys.modules[modname]
122 138
123 def _save_and_block_module(name, orig_modules): 139 def _save_and_block_module(name, orig_modules):
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after
209 225
210 def get_original_stdout(): 226 def get_original_stdout():
211 return _original_stdout or sys.stdout 227 return _original_stdout or sys.stdout
212 228
213 def unload(name): 229 def unload(name):
214 try: 230 try:
215 del sys.modules[name] 231 del sys.modules[name]
216 except KeyError: 232 except KeyError:
217 pass 233 pass
218 234
235 if sys.platform.startswith("win"):
236 def _waitfor(func, pathname, waitall=False):
237 # Peform the operation
238 func(pathname)
239 # Now setup the wait loop
240 if waitall:
241 dirname = pathname
242 else:
243 dirname, name = os.path.split(pathname)
244 dirname = dirname or '.'
245 # Check for `pathname` to be removed from the filesystem.
246 # The exponential backoff of the timeout amounts to a total
247 # of ~1 second after which the deletion is probably an error
248 # anyway.
249 # Testing on a i7@4.3GHz shows that usually only 1 iteration is
250 # required when contention occurs.
251 timeout = 0.001
252 while timeout < 1.0:
253 # Note we are only testing for the existance of the file(s) in
254 # the contents of the directory regardless of any security or
255 # access rights. If we have made it this far, we have sufficient
256 # permissions to do that much using Python's equivalent of the
257 # Windows API FindFirstFile.
258 # Other Windows APIs can fail or give incorrect results when
259 # dealing with files that are pending deletion.
260 L = os.listdir(dirname)
261 if not (L if waitall else name in L):
262 return
263 # Increase the timeout and try again
264 time.sleep(timeout)
265 timeout *= 2
266 warnings.warn('tests may fail, delete still pending for ' + pathname,
267 RuntimeWarning, stacklevel=4)
268
269 def _unlink(filename):
270 _waitfor(os.unlink, filename)
271
272 def _rmdir(dirname):
273 _waitfor(os.rmdir, dirname)
274
275 def _rmtree(path):
276 def _rmtree_inner(path):
277 for name in os.listdir(path):
278 fullname = os.path.join(path, name)
279 if os.path.isdir(fullname):
280 _waitfor(_rmtree_inner, fullname, waitall=True)
281 os.rmdir(fullname)
282 else:
283 os.unlink(fullname)
284 _waitfor(_rmtree_inner, path, waitall=True)
285 _waitfor(os.rmdir, path)
286 else:
287 _unlink = os.unlink
288 _rmdir = os.rmdir
289 _rmtree = shutil.rmtree
290
219 def unlink(filename): 291 def unlink(filename):
220 try: 292 try:
221 os.unlink(filename) 293 _unlink(filename)
222 except OSError as error: 294 except OSError as error:
223 # The filename need not exist. 295 # The filename need not exist.
224 if error.errno not in (errno.ENOENT, errno.ENOTDIR): 296 if error.errno not in (errno.ENOENT, errno.ENOTDIR):
225 raise 297 raise
226 298
299 def rmdir(dirname):
300 try:
301 _rmdir(dirname)
302 except OSError as error:
303 # The directory need not exist.
304 if error.errno != errno.ENOENT:
305 raise
306
227 def rmtree(path): 307 def rmtree(path):
228 try: 308 try:
229 shutil.rmtree(path) 309 _rmtree(path)
230 except OSError as error: 310 except OSError as error:
231 if error.errno != errno.ENOENT: 311 if error.errno != errno.ENOENT:
232 raise 312 raise
233 313
234 def make_legacy_pyc(source): 314 def make_legacy_pyc(source):
235 """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location. 315 """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
236 316
237 The choice of .pyc or .pyo extension is done based on the __debug__ flag 317 The choice of .pyc or .pyo extension is done based on the __debug__ flag
238 value. 318 value.
239 319
(...skipping 238 matching lines...) Expand 10 before | Expand all | Expand 10 after
478 if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): 558 if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
479 sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) 559 sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
480 560
481 sock.bind((host, 0)) 561 sock.bind((host, 0))
482 port = sock.getsockname()[1] 562 port = sock.getsockname()[1]
483 return port 563 return port
484 564
485 def _is_ipv6_enabled(): 565 def _is_ipv6_enabled():
486 """Check whether IPv6 is enabled on this host.""" 566 """Check whether IPv6 is enabled on this host."""
487 if socket.has_ipv6: 567 if socket.has_ipv6:
568 sock = None
488 try: 569 try:
489 sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) 570 sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
490 sock.bind(('::1', 0)) 571 sock.bind(('::1', 0))
572 return True
491 except (socket.error, socket.gaierror): 573 except (socket.error, socket.gaierror):
492 pass 574 pass
493 else: 575 finally:
494 sock.close() 576 if sock:
495 return True 577 sock.close()
496 return False 578 return False
497 579
498 IPV6_ENABLED = _is_ipv6_enabled() 580 IPV6_ENABLED = _is_ipv6_enabled()
499 581
500 582
501 # A constant likely larger than the underlying OS pipe buffer size. 583 # A constant likely larger than the underlying OS pipe buffer size.
502 # Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe 584 # Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe
503 # buffer size: take 1M to be sure. 585 # buffer size: take 1M to be sure.
504 PIPE_MAX_SIZE = 1024 * 1024 586 PIPE_MAX_SIZE = 1024 * 1024
505 587
506 588
507 # decorator for skipping tests on non-IEEE 754 platforms 589 # decorator for skipping tests on non-IEEE 754 platforms
508 requires_IEEE_754 = unittest.skipUnless( 590 requires_IEEE_754 = unittest.skipUnless(
509 float.__getformat__("double").startswith("IEEE"), 591 float.__getformat__("double").startswith("IEEE"),
510 "test requires IEEE 754 doubles") 592 "test requires IEEE 754 doubles")
511 593
512 requires_zlib = unittest.skipUnless(zlib, 'requires zlib') 594 requires_zlib = unittest.skipUnless(zlib, 'requires zlib')
595
596 requires_bz2 = unittest.skipUnless(bz2, 'requires bz2')
597
598 requires_lzma = unittest.skipUnless(lzma, 'requires lzma')
513 599
514 is_jython = sys.platform.startswith('java') 600 is_jython = sys.platform.startswith('java')
515 601
516 # Filename used for testing 602 # Filename used for testing
517 if os.name == 'java': 603 if os.name == 'java':
518 # Jython disallows @ in module names 604 # Jython disallows @ in module names
519 TESTFN = '$test' 605 TESTFN = '$test'
520 else: 606 else:
521 TESTFN = '@test' 607 TESTFN = '@test'
522 608
523 # Disambiguate TESTFN for parallel testing, while letting it remain a valid 609 # Disambiguate TESTFN for parallel testing, while letting it remain a valid
524 # module name. 610 # module name.
525 TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid()) 611 TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
526 612
613 # FS_NONASCII: non-ASCII character encodable by os.fsencode(),
614 # or None if there is no such character.
615 FS_NONASCII = None
616 for character in (
617 # First try printable and common characters to have a readable filename.
618 # For each character, the encoding list are just example of encodings able
619 # to encode the character (the list is not exhaustive).
620
621 # U+00E6 (Latin Small Letter Ae): cp1252, iso-8859-1
622 '\u00E6',
623 # U+0130 (Latin Capital Letter I With Dot Above): cp1254, iso8859_3
624 '\u0130',
625 # U+0141 (Latin Capital Letter L With Stroke): cp1250, cp1257
626 '\u0141',
627 # U+03C6 (Greek Small Letter Phi): cp1253
628 '\u03C6',
629 # U+041A (Cyrillic Capital Letter Ka): cp1251
630 '\u041A',
631 # U+05D0 (Hebrew Letter Alef): Encodable to cp424
632 '\u05D0',
633 # U+060C (Arabic Comma): cp864, cp1006, iso8859_6, mac_arabic
634 '\u060C',
635 # U+062A (Arabic Letter Teh): cp720
636 '\u062A',
637 # U+0E01 (Thai Character Ko Kai): cp874
638 '\u0E01',
639
640 # Then try more "special" characters. "special" because they may be
641 # interpreted or displayed differently depending on the exact locale
642 # encoding and the font.
643
644 # U+00A0 (No-Break Space)
645 '\u00A0',
646 # U+20AC (Euro Sign)
647 '\u20AC',
648 ):
649 try:
650 os.fsdecode(os.fsencode(character))
651 except UnicodeError:
652 pass
653 else:
654 FS_NONASCII = character
655 break
527 656
528 # TESTFN_UNICODE is a non-ascii filename 657 # TESTFN_UNICODE is a non-ascii filename
529 TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f" 658 TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
530 if sys.platform == 'darwin': 659 if sys.platform == 'darwin':
531 # In Mac OS X's VFS API file names are, by definition, canonically 660 # In Mac OS X's VFS API file names are, by definition, canonically
532 # decomposed Unicode, encoded using UTF-8. See QA1173: 661 # decomposed Unicode, encoded using UTF-8. See QA1173:
533 # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html 662 # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
534 import unicodedata 663 import unicodedata
535 TESTFN_UNICODE = unicodedata.normalize('NFD', TESTFN_UNICODE) 664 TESTFN_UNICODE = unicodedata.normalize('NFD', TESTFN_UNICODE)
536 TESTFN_ENCODING = sys.getfilesystemencoding() 665 TESTFN_ENCODING = sys.getfilesystemencoding()
(...skipping 24 matching lines...) Expand all
561 b'\xff'.decode(TESTFN_ENCODING) 690 b'\xff'.decode(TESTFN_ENCODING)
562 except UnicodeDecodeError: 691 except UnicodeDecodeError:
563 # 0xff will be encoded using the surrogate character u+DCFF 692 # 0xff will be encoded using the surrogate character u+DCFF
564 TESTFN_UNENCODABLE = TESTFN \ 693 TESTFN_UNENCODABLE = TESTFN \
565 + b'-\xff'.decode(TESTFN_ENCODING, 'surrogateescape') 694 + b'-\xff'.decode(TESTFN_ENCODING, 'surrogateescape')
566 else: 695 else:
567 # File system encoding (eg. ISO-8859-* encodings) can encode 696 # File system encoding (eg. ISO-8859-* encodings) can encode
568 # the byte 0xff. Skip some unicode filename tests. 697 # the byte 0xff. Skip some unicode filename tests.
569 pass 698 pass
570 699
700 # TESTFN_UNDECODABLE is a filename (bytes type) that should *not* be able to be
701 # decoded from the filesystem encoding (in strict mode). It can be None if we
702 # cannot generate such filename (ex: the latin1 encoding can decode any byte
703 # sequence). On UNIX, TESTFN_UNDECODABLE can be decoded by os.fsdecode() thanks
704 # to the surrogateescape error handler (PEP 383), but not from the filesystem
705 # encoding in strict mode.
706 TESTFN_UNDECODABLE = None
707 for name in (
708 # b'\xff' is not decodable by os.fsdecode() with code page 932. Windows
709 # accepts it to create a file or a directory, or don't accept to enter to
710 # such directory (when the bytes name is used). So test b'\xe7' first: it is
711 # not decodable from cp932.
712 b'\xe7w\xf0',
713 # undecodable from ASCII, UTF-8
714 b'\xff',
715 # undecodable from iso8859-3, iso8859-6, iso8859-7, cp424, iso8859-8, cp856
716 # and cp857
717 b'\xae\xd5'
718 # undecodable from UTF-8 (UNIX and Mac OS X)
719 b'\xed\xb2\x80', b'\xed\xb4\x80',
720 # undecodable from shift_jis, cp869, cp874, cp932, cp1250, cp1251, cp1252,
721 # cp1253, cp1254, cp1255, cp1257, cp1258
722 b'\x81\x98',
723 ):
724 try:
725 name.decode(TESTFN_ENCODING)
726 except UnicodeDecodeError:
727 TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
728 break
729
730 if FS_NONASCII:
731 TESTFN_NONASCII = TESTFN + '-' + FS_NONASCII
732 else:
733 TESTFN_NONASCII = None
734
571 # Save the initial cwd 735 # Save the initial cwd
572 SAVEDCWD = os.getcwd() 736 SAVEDCWD = os.getcwd()
573 737
574 @contextlib.contextmanager 738 @contextlib.contextmanager
575 def temp_cwd(name='tempcwd', quiet=False, path=None): 739 def temp_cwd(name='tempcwd', quiet=False, path=None):
576 """ 740 """
577 Context manager that temporarily changes the CWD. 741 Context manager that temporarily changes the CWD.
578 742
579 An existing path may be provided as *path*, in which case this 743 An existing path may be provided as *path*, in which case this
580 function makes no changes to the file system. 744 function makes no changes to the file system.
(...skipping 13 matching lines...) Expand all
594 except OSError: 758 except OSError:
595 if not quiet: 759 if not quiet:
596 raise 760 raise
597 warnings.warn('tests may fail, unable to create temp CWD ' + name, 761 warnings.warn('tests may fail, unable to create temp CWD ' + name,
598 RuntimeWarning, stacklevel=3) 762 RuntimeWarning, stacklevel=3)
599 try: 763 try:
600 os.chdir(path) 764 os.chdir(path)
601 except OSError: 765 except OSError:
602 if not quiet: 766 if not quiet:
603 raise 767 raise
604 warnings.warn('tests may fail, unable to change the CWD to ' + name, 768 warnings.warn('tests may fail, unable to change the CWD to ' + path,
605 RuntimeWarning, stacklevel=3) 769 RuntimeWarning, stacklevel=3)
606 try: 770 try:
607 yield os.getcwd() 771 yield os.getcwd()
608 finally: 772 finally:
609 os.chdir(saved_dir) 773 os.chdir(saved_dir)
610 if is_temporary: 774 if is_temporary:
611 rmtree(name) 775 rmtree(name)
612 776
613 777
614 if hasattr(os, "umask"): 778 if hasattr(os, "umask"):
(...skipping 329 matching lines...) Expand 10 before | Expand all | Expand 10 after
944 with the Internet connection manifest themselves as exceptions.""" 1108 with the Internet connection manifest themselves as exceptions."""
945 default_errnos = [ 1109 default_errnos = [
946 ('ECONNREFUSED', 111), 1110 ('ECONNREFUSED', 111),
947 ('ECONNRESET', 104), 1111 ('ECONNRESET', 104),
948 ('EHOSTUNREACH', 113), 1112 ('EHOSTUNREACH', 113),
949 ('ENETUNREACH', 101), 1113 ('ENETUNREACH', 101),
950 ('ETIMEDOUT', 110), 1114 ('ETIMEDOUT', 110),
951 ] 1115 ]
952 default_gai_errnos = [ 1116 default_gai_errnos = [
953 ('EAI_AGAIN', -3), 1117 ('EAI_AGAIN', -3),
1118 ('EAI_FAIL', -4),
954 ('EAI_NONAME', -2), 1119 ('EAI_NONAME', -2),
955 ('EAI_NODATA', -5), 1120 ('EAI_NODATA', -5),
956 # Encountered when trying to resolve IPv6-only hostnames 1121 # Encountered when trying to resolve IPv6-only hostnames
957 ('WSANO_DATA', 11004), 1122 ('WSANO_DATA', 11004),
958 ] 1123 ]
959 1124
960 denied = ResourceDenied("Resource %r is not available" % resource_name) 1125 denied = ResourceDenied("Resource %r is not available" % resource_name)
961 captured_errnos = errnos 1126 captured_errnos = errnos
962 gai_errnos = [] 1127 gai_errnos = []
963 if not captured_errnos: 1128 if not captured_errnos:
(...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after
1060 def python_is_optimized(): 1225 def python_is_optimized():
1061 """Find if Python was built with optimizations.""" 1226 """Find if Python was built with optimizations."""
1062 cflags = sysconfig.get_config_var('PY_CFLAGS') or '' 1227 cflags = sysconfig.get_config_var('PY_CFLAGS') or ''
1063 final_opt = "" 1228 final_opt = ""
1064 for opt in cflags.split(): 1229 for opt in cflags.split():
1065 if opt.startswith('-O'): 1230 if opt.startswith('-O'):
1066 final_opt = opt 1231 final_opt = opt
1067 return final_opt != '' and final_opt != '-O0' 1232 return final_opt != '' and final_opt != '-O0'
1068 1233
1069 1234
1235 _header = 'nP'
1236 _align = '0n'
1237 if hasattr(sys, "gettotalrefcount"):
1238 _header = '2P' + _header
1239 _align = '0P'
1240 _vheader = _header + 'n'
1241
1242 def calcobjsize(fmt):
1243 return struct.calcsize(_header + fmt + _align)
1244
1245 def calcvobjsize(fmt):
1246 return struct.calcsize(_vheader + fmt + _align)
1247
1248
1249 _TPFLAGS_HAVE_GC = 1<<14
1250 _TPFLAGS_HEAPTYPE = 1<<9
1251
1252 def check_sizeof(test, o, size):
1253 result = sys.getsizeof(o)
1254 # add GC header size
1255 if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\
1256 ((type(o) != type) and (type(o).__flags__ & _TPFLAGS_HAVE_GC))):
1257 size += _testcapi.SIZEOF_PYGC_HEAD
1258 msg = 'wrong size for %s: got %d, expected %d' \
1259 % (type(o), result, size)
1260 test.assertEqual(result, size, msg)
1261
1070 #======================================================================= 1262 #=======================================================================
1071 # Decorator for running a function in a different locale, correctly resetting 1263 # Decorator for running a function in a different locale, correctly resetting
1072 # it afterwards. 1264 # it afterwards.
1073 1265
1074 def run_with_locale(catstr, *locales): 1266 def run_with_locale(catstr, *locales):
1075 def decorator(func): 1267 def decorator(func):
1076 def inner(*args, **kwds): 1268 def inner(*args, **kwds):
1077 try: 1269 try:
1078 import locale 1270 import locale
1079 category = getattr(locale, catstr) 1271 category = getattr(locale, catstr)
(...skipping 17 matching lines...) Expand all
1097 return func(*args, **kwds) 1289 return func(*args, **kwds)
1098 finally: 1290 finally:
1099 if locale and orig_locale: 1291 if locale and orig_locale:
1100 locale.setlocale(category, orig_locale) 1292 locale.setlocale(category, orig_locale)
1101 inner.__name__ = func.__name__ 1293 inner.__name__ = func.__name__
1102 inner.__doc__ = func.__doc__ 1294 inner.__doc__ = func.__doc__
1103 return inner 1295 return inner
1104 return decorator 1296 return decorator
1105 1297
1106 #======================================================================= 1298 #=======================================================================
1299 # Decorator for running a function in a specific timezone, correctly
1300 # resetting it afterwards.
1301
1302 def run_with_tz(tz):
1303 def decorator(func):
1304 def inner(*args, **kwds):
1305 try:
1306 tzset = time.tzset
1307 except AttributeError:
1308 raise unittest.SkipTest("tzset required")
1309 if 'TZ' in os.environ:
1310 orig_tz = os.environ['TZ']
1311 else:
1312 orig_tz = None
1313 os.environ['TZ'] = tz
1314 tzset()
1315
1316 # now run the function, resetting the tz on exceptions
1317 try:
1318 return func(*args, **kwds)
1319 finally:
1320 if orig_tz is None:
1321 del os.environ['TZ']
1322 else:
1323 os.environ['TZ'] = orig_tz
1324 time.tzset()
1325
1326 inner.__name__ = func.__name__
1327 inner.__doc__ = func.__doc__
1328 return inner
1329 return decorator
1330
1331 #=======================================================================
1107 # Big-memory-test support. Separate from 'resources' because memory use 1332 # Big-memory-test support. Separate from 'resources' because memory use
1108 # should be configurable. 1333 # should be configurable.
1109 1334
1110 # Some handy shorthands. Note that these are used for byte-limits as well 1335 # Some handy shorthands. Note that these are used for byte-limits as well
1111 # as size-limits, in the various bigmem tests 1336 # as size-limits, in the various bigmem tests
1112 _1M = 1024*1024 1337 _1M = 1024*1024
1113 _1G = 1024 * _1M 1338 _1G = 1024 * _1M
1114 _2G = 2 * _1G 1339 _2G = 2 * _1G
1115 _4G = 4 * _1G 1340 _4G = 4 * _1G
1116 1341
(...skipping 21 matching lines...) Expand all
1138 max_memuse = memlimit 1363 max_memuse = memlimit
1139 1364
1140 class _MemoryWatchdog: 1365 class _MemoryWatchdog:
1141 """An object which periodically watches the process' memory consumption 1366 """An object which periodically watches the process' memory consumption
1142 and prints it out. 1367 and prints it out.
1143 """ 1368 """
1144 1369
1145 def __init__(self): 1370 def __init__(self):
1146 self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) 1371 self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
1147 self.started = False 1372 self.started = False
1148 self.thread = None 1373
1374 def start(self):
1149 try: 1375 try:
1150 self.page_size = os.sysconf('SC_PAGESIZE') 1376 f = open(self.procfile, 'r')
1151 except (ValueError, AttributeError):
1152 try:
1153 self.page_size = os.sysconf('SC_PAGE_SIZE')
1154 except (ValueError, AttributeError):
1155 self.page_size = 4096
1156
1157 def consumer(self, fd):
1158 HEADER = "l"
1159 header_size = struct.calcsize(HEADER)
1160 try:
1161 while True:
1162 header = os.read(fd, header_size)
1163 if len(header) < header_size:
1164 # Pipe closed on other end
1165 break
1166 data_len, = struct.unpack(HEADER, header)
1167 data = os.read(fd, data_len)
1168 statm = data.decode('ascii')
1169 data = int(statm.split()[5])
1170 print(" ... process data size: {data:.1f}G"
1171 .format(data=data * self.page_size / (1024 ** 3)))
1172 finally:
1173 os.close(fd)
1174
1175 def start(self):
1176 if not faulthandler or not hasattr(faulthandler, '_file_watchdog'):
1177 return
1178 try:
1179 rfd = os.open(self.procfile, os.O_RDONLY)
1180 except OSError as e: 1377 except OSError as e:
1181 warnings.warn('/proc not available for stats: {}'.format(e), 1378 warnings.warn('/proc not available for stats: {}'.format(e),
1182 RuntimeWarning) 1379 RuntimeWarning)
1183 sys.stderr.flush() 1380 sys.stderr.flush()
1184 return 1381 return
1185 pipe_fd, wfd = os.pipe() 1382
1186 # _file_watchdog() doesn't take the GIL in its child thread, and 1383 watchdog_script = findfile("memory_watchdog.py")
1187 # therefore collects statistics timely 1384 self.mem_watchdog = subprocess.Popen([sys.executable, watchdog_script],
1188 faulthandler._file_watchdog(rfd, wfd, 1.0) 1385 stdin=f, stderr=subprocess.DEVNULL)
1386 f.close()
1189 self.started = True 1387 self.started = True
1190 self.thread = threading.Thread(target=self.consumer, args=(pipe_fd,))
1191 self.thread.daemon = True
1192 self.thread.start()
1193 1388
1194 def stop(self): 1389 def stop(self):
1195 if not self.started: 1390 if self.started:
1196 return 1391 self.mem_watchdog.terminate()
1197 faulthandler._cancel_file_watchdog() 1392 self.mem_watchdog.wait()
1198 self.thread.join()
1199 1393
1200 1394
1201 def bigmemtest(size, memuse, dry_run=True): 1395 def bigmemtest(size, memuse, dry_run=True):
1202 """Decorator for bigmem tests. 1396 """Decorator for bigmem tests.
1203 1397
1204 'minsize' is the minimum useful size for the test (in arbitrary, 1398 'minsize' is the minimum useful size for the test (in arbitrary,
1205 test-interpreted units.) 'memuse' is the number of 'bytes per size' for 1399 test-interpreted units.) 'memuse' is the number of 'bytes per size' for
1206 the test, or a good estimate of it. 1400 the test, or a good estimate of it.
1207 1401
1208 if 'dry_run' is False, it means the test doesn't support dummy runs 1402 if 'dry_run' is False, it means the test doesn't support dummy runs
1209 when -M is not specified. 1403 when -M is not specified.
1210 """ 1404 """
1211 def decorator(f): 1405 def decorator(f):
1212 def wrapper(self): 1406 def wrapper(self):
1213 size = wrapper.size 1407 size = wrapper.size
1214 memuse = wrapper.memuse 1408 memuse = wrapper.memuse
1215 if not real_max_memuse: 1409 if not real_max_memuse:
1216 maxsize = 5147 1410 maxsize = 5147
1217 else: 1411 else:
1218 maxsize = size 1412 maxsize = size
1219 1413
1220 if ((real_max_memuse or not dry_run) 1414 if ((real_max_memuse or not dry_run)
1221 and real_max_memuse < maxsize * memuse): 1415 and real_max_memuse < maxsize * memuse):
1222 raise unittest.SkipTest( 1416 raise unittest.SkipTest(
1223 "not enough memory: %.1fG minimum needed" 1417 "not enough memory: %.1fG minimum needed"
1224 % (size * memuse / (1024 ** 3))) 1418 % (size * memuse / (1024 ** 3)))
1225 1419
1226 if real_max_memuse and verbose and faulthandler and threading: 1420 if real_max_memuse and verbose:
1227 print() 1421 print()
1228 print(" ... expected peak memory use: {peak:.1f}G" 1422 print(" ... expected peak memory use: {peak:.1f}G"
1229 .format(peak=size * memuse / (1024 ** 3))) 1423 .format(peak=size * memuse / (1024 ** 3)))
1230 watchdog = _MemoryWatchdog() 1424 watchdog = _MemoryWatchdog()
1231 watchdog.start() 1425 watchdog.start()
1232 else: 1426 else:
1233 watchdog = None 1427 watchdog = None
1234 1428
1235 try: 1429 try:
1236 return f(self, maxsize) 1430 return f(self, maxsize)
(...skipping 161 matching lines...) Expand 10 before | Expand all | Expand 10 after
1398 if fnmatch.fnmatchcase(name, match_tests): 1592 if fnmatch.fnmatchcase(name, match_tests):
1399 return True 1593 return True
1400 return False 1594 return False
1401 _filter_suite(suite, case_pred) 1595 _filter_suite(suite, case_pred)
1402 _run_suite(suite) 1596 _run_suite(suite)
1403 1597
1404 1598
1405 #======================================================================= 1599 #=======================================================================
1406 # doctest driver. 1600 # doctest driver.
1407 1601
1408 def run_doctest(module, verbosity=None): 1602 def run_doctest(module, verbosity=None, optionflags=0):
1409 """Run doctest on the given module. Return (#failures, #tests). 1603 """Run doctest on the given module. Return (#failures, #tests).
1410 1604
1411 If optional argument verbosity is not specified (or is None), pass 1605 If optional argument verbosity is not specified (or is None), pass
1412 support's belief about verbosity on to doctest. Else doctest's 1606 support's belief about verbosity on to doctest. Else doctest's
1413 usual behavior is used (it searches sys.argv for -v). 1607 usual behavior is used (it searches sys.argv for -v).
1414 """ 1608 """
1415 1609
1416 import doctest 1610 import doctest
1417 1611
1418 if verbosity is None: 1612 if verbosity is None:
1419 verbosity = verbose 1613 verbosity = verbose
1420 else: 1614 else:
1421 verbosity = None 1615 verbosity = None
1422 1616
1423 f, t = doctest.testmod(module, verbose=verbosity) 1617 f, t = doctest.testmod(module, verbose=verbosity, optionflags=optionflags)
1424 if f: 1618 if f:
1425 raise TestFailed("%d of %d doctests failed" % (f, t)) 1619 raise TestFailed("%d of %d doctests failed" % (f, t))
1426 if verbose: 1620 if verbose:
1427 print('doctest (%s) ... %d tests with zero failures' % 1621 print('doctest (%s) ... %d tests with zero failures' %
1428 (module.__name__, t)) 1622 (module.__name__, t))
1429 return f, t 1623 return f, t
1430 1624
1431 1625
1432 #======================================================================= 1626 #=======================================================================
1433 # Support for saving and restoring the imported modules. 1627 # Support for saving and restoring the imported modules.
(...skipping 137 matching lines...) Expand 10 before | Expand all | Expand 10 after
1571 finally: 1765 finally:
1572 del obj[item] 1766 del obj[item]
1573 1767
1574 def strip_python_stderr(stderr): 1768 def strip_python_stderr(stderr):
1575 """Strip the stderr of a Python process from potential debug output 1769 """Strip the stderr of a Python process from potential debug output
1576 emitted by the interpreter. 1770 emitted by the interpreter.
1577 1771
1578 This will typically be run on the result of the communicate() method 1772 This will typically be run on the result of the communicate() method
1579 of a subprocess.Popen object. 1773 of a subprocess.Popen object.
1580 """ 1774 """
1581 stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?$", b"", stderr).strip() 1775 stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip()
1582 return stderr 1776 return stderr
1583 1777
1584 def args_from_interpreter_flags(): 1778 def args_from_interpreter_flags():
1585 """Return a list of command-line arguments reproducing the current 1779 """Return a list of command-line arguments reproducing the current
1586 settings in sys.flags and sys.warnoptions.""" 1780 settings in sys.flags and sys.warnoptions."""
1587 flag_opt_map = { 1781 return subprocess._args_from_interpreter_flags()
1588 'bytes_warning': 'b',
1589 'dont_write_bytecode': 'B',
1590 'ignore_environment': 'E',
1591 'no_user_site': 's',
1592 'no_site': 'S',
1593 'optimize': 'O',
1594 'verbose': 'v',
1595 }
1596 args = []
1597 for flag, opt in flag_opt_map.items():
1598 v = getattr(sys.flags, flag)
1599 if v > 0:
1600 args.append('-' + opt * v)
1601 for opt in sys.warnoptions:
1602 args.append('-W' + opt)
1603 return args
1604 1782
1605 #============================================================ 1783 #============================================================
1606 # Support for assertions about logging. 1784 # Support for assertions about logging.
1607 #============================================================ 1785 #============================================================
1608 1786
1609 class TestHandler(logging.handlers.BufferingHandler): 1787 class TestHandler(logging.handlers.BufferingHandler):
1610 def __init__(self, matcher): 1788 def __init__(self, matcher):
1611 # BufferingHandler takes a "capacity" argument 1789 # BufferingHandler takes a "capacity" argument
1612 # so as to know when to flush. As we're overriding 1790 # so as to know when to flush. As we're overriding
1613 # shouldFlush anyway, we can set a capacity of zero. 1791 # shouldFlush anyway, we can set a capacity of zero.
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
1683 os.remove(symlink_path) 1861 os.remove(symlink_path)
1684 _can_symlink = can 1862 _can_symlink = can
1685 return can 1863 return can
1686 1864
1687 def skip_unless_symlink(test): 1865 def skip_unless_symlink(test):
1688 """Skip decorator for tests that require functional symlink""" 1866 """Skip decorator for tests that require functional symlink"""
1689 ok = can_symlink() 1867 ok = can_symlink()
1690 msg = "Requires functional symlink implementation" 1868 msg = "Requires functional symlink implementation"
1691 return test if ok else unittest.skip(msg)(test) 1869 return test if ok else unittest.skip(msg)(test)
1692 1870
1871 _can_xattr = None
1872 def can_xattr():
1873 global _can_xattr
1874 if _can_xattr is not None:
1875 return _can_xattr
1876 if not hasattr(os, "setxattr"):
1877 can = False
1878 else:
1879 tmp_fp, tmp_name = tempfile.mkstemp()
1880 try:
1881 with open(TESTFN, "wb") as fp:
1882 try:
1883 # TESTFN & tempfile may use different file systems with
1884 # different capabilities
1885 os.setxattr(tmp_fp, b"user.test", b"")
1886 os.setxattr(fp.fileno(), b"user.test", b"")
1887 # Kernels < 2.6.39 don't respect setxattr flags.
1888 kernel_version = platform.release()
1889 m = re.match("2.6.(\d{1,2})", kernel_version)
1890 can = m is None or int(m.group(1)) >= 39
1891 except OSError:
1892 can = False
1893 finally:
1894 unlink(TESTFN)
1895 unlink(tmp_name)
1896 _can_xattr = can
1897 return can
1898
1899 def skip_unless_xattr(test):
1900 """Skip decorator for tests that require functional extended attributes"""
1901 ok = can_xattr()
1902 msg = "no non-broken extended attribute support"
1903 return test if ok else unittest.skip(msg)(test)
1904
1693 def patch(test_instance, object_to_patch, attr_name, new_value): 1905 def patch(test_instance, object_to_patch, attr_name, new_value):
1694 """Override 'object_to_patch'.'attr_name' with 'new_value'. 1906 """Override 'object_to_patch'.'attr_name' with 'new_value'.
1695 1907
1696 Also, add a cleanup procedure to 'test_instance' to restore 1908 Also, add a cleanup procedure to 'test_instance' to restore
1697 'object_to_patch' value for 'attr_name'. 1909 'object_to_patch' value for 'attr_name'.
1698 The 'attr_name' should be a valid attribute for 'object_to_patch'. 1910 The 'attr_name' should be a valid attribute for 'object_to_patch'.
1699 1911
1700 """ 1912 """
1701 # check that 'attr_name' is a real attribute for 'object_to_patch' 1913 # check that 'attr_name' is a real attribute for 'object_to_patch'
1702 # will raise AttributeError if it does not exist 1914 # will raise AttributeError if it does not exist
(...skipping 12 matching lines...) Expand all
1715 def cleanup(): 1927 def cleanup():
1716 if attr_is_local: 1928 if attr_is_local:
1717 setattr(object_to_patch, attr_name, old_value) 1929 setattr(object_to_patch, attr_name, old_value)
1718 else: 1930 else:
1719 delattr(object_to_patch, attr_name) 1931 delattr(object_to_patch, attr_name)
1720 1932
1721 test_instance.addCleanup(cleanup) 1933 test_instance.addCleanup(cleanup)
1722 1934
1723 # actually override the attribute 1935 # actually override the attribute
1724 setattr(object_to_patch, attr_name, new_value) 1936 setattr(object_to_patch, attr_name, new_value)
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+