Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(945)

Delta Between Two Patch Sets: Lib/test/support.py

Issue 7652: Merge C version of decimal into py3k.
Left Patch Set: Created 8 years ago
Right Patch Set: Created 7 years, 3 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
« no previous file with change/comment | « Lib/decimal.py ('k') | Lib/test/test_decimal.py » ('j') | no next file with change/comment »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
LEFTRIGHT
1 """Supporting definitions for the Python regression tests.""" 1 """Supporting definitions for the Python regression tests."""
2 2
3 if __name__ != 'test.support': 3 if __name__ != 'test.support':
4 raise ImportError('support must be imported from the test package') 4 raise ImportError('support must be imported from the test package')
5 5
6 import contextlib 6 import contextlib
7 import errno 7 import errno
8 import functools 8 import functools
9 import gc 9 import gc
10 import socket 10 import socket
11 import sys 11 import sys
12 import os 12 import os
13 import platform 13 import platform
14 import shutil 14 import shutil
15 import warnings 15 import warnings
16 import unittest 16 import unittest
17 import importlib 17 import importlib
18 import collections.abc 18 import collections.abc
19 import re 19 import re
20 import subprocess 20 import subprocess
21 import imp 21 import imp
22 import time 22 import time
23 import sysconfig 23 import sysconfig
24 import fnmatch
24 import logging.handlers 25 import logging.handlers
26 import struct
25 27
26 try: 28 try:
27 import _thread 29 import _thread, threading
28 except ImportError: 30 except ImportError:
29 _thread = None 31 _thread = None
32 threading = None
33 try:
34 import multiprocessing.process
35 except ImportError:
36 multiprocessing = None
37
38 try:
39 import faulthandler
40 except ImportError:
41 faulthandler = None
30 42
31 try: 43 try:
32 import zlib 44 import zlib
33 except ImportError: 45 except ImportError:
34 zlib = None 46 zlib = None
35 47
48 try:
49 import fcntl
50 except ImportError:
51 fcntl = None
52
36 __all__ = [ 53 __all__ = [
37 "Error", "TestFailed", "ResourceDenied", "import_module", 54 "Error", "TestFailed", "ResourceDenied", "import_module",
38 "verbose", "use_resources", "max_memuse", "record_original_stdout", 55 "verbose", "use_resources", "max_memuse", "record_original_stdout",
39 "get_original_stdout", "unload", "unlink", "rmtree", "forget", 56 "get_original_stdout", "unload", "unlink", "rmtree", "forget",
40 "is_resource_enabled", "requires", "requires_linux_version", 57 "is_resource_enabled", "requires", "requires_freebsd_version",
41 "requires_mac_ver", "find_unused_port", "bind_port", 58 "requires_linux_version", "requires_mac_ver", "find_unused_port", "bind_port ",
42 "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD", "temp_cwd", 59 "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD", "temp_cwd",
43 "findfile", "sortdict", "check_syntax_error", "open_urlresource", 60 "findfile", "create_empty_file", "sortdict", "check_syntax_error", "open_url resource",
44 "check_warnings", "CleanImport", "EnvironmentVarGuard", "TransientResource", 61 "check_warnings", "CleanImport", "EnvironmentVarGuard", "TransientResource",
45 "captured_stdout", "captured_stdin", "captured_stderr", "time_out", 62 "captured_stdout", "captured_stdin", "captured_stderr", "time_out",
46 "socket_peer_reset", "ioerror_peer_reset", "run_with_locale", 'temp_umask', 63 "socket_peer_reset", "ioerror_peer_reset", "run_with_locale", 'temp_umask',
47 "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest", 64 "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest",
48 "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup", 65 "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
49 "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail", 66 "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail",
50 "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754", 67 "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754",
51 "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", 68 "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink",
52 "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE" 69 "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE", "failfast",
70 "anticipate_failure"
53 ] 71 ]
54 72
55 class Error(Exception): 73 class Error(Exception):
56 """Base class for regression test exceptions.""" 74 """Base class for regression test exceptions."""
57 75
58 class TestFailed(Error): 76 class TestFailed(Error):
59 """Test failed.""" 77 """Test failed."""
60 78
61 class ResourceDenied(unittest.SkipTest): 79 class ResourceDenied(unittest.SkipTest):
62 """Test skipped because it requested a disallowed resource. 80 """Test skipped because it requested a disallowed resource.
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
111 """Helper function to save and block a module in sys.modules 129 """Helper function to save and block a module in sys.modules
112 130
113 Return True if the module was in sys.modules, False otherwise.""" 131 Return True if the module was in sys.modules, False otherwise."""
114 saved = True 132 saved = True
115 try: 133 try:
116 orig_modules[name] = sys.modules[name] 134 orig_modules[name] = sys.modules[name]
117 except KeyError: 135 except KeyError:
118 saved = False 136 saved = False
119 sys.modules[name] = None 137 sys.modules[name] = None
120 return saved 138 return saved
139
140
141 def anticipate_failure(condition):
142 """Decorator to mark a test that is known to be broken in some cases
143
144 Any use of this decorator should have a comment identifying the
145 associated tracker issue.
146 """
147 if condition:
148 return unittest.expectedFailure
149 return lambda f: f
121 150
122 151
123 def import_fresh_module(name, fresh=(), blocked=(), deprecated=False): 152 def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
124 """Imports and returns a module, deliberately bypassing the sys.modules cach e 153 """Imports and returns a module, deliberately bypassing the sys.modules cach e
125 and importing a fresh copy of the module. Once the import is complete, 154 and importing a fresh copy of the module. Once the import is complete,
126 the sys.modules cache is restored to its original state. 155 the sys.modules cache is restored to its original state.
127 156
128 Modules named in fresh are also imported anew if needed by the import. 157 Modules named in fresh are also imported anew if needed by the import.
129 If one of these modules can't be imported, None is returned. 158 If one of these modules can't be imported, None is returned.
130 159
(...skipping 25 matching lines...) Expand all
156 for name_to_remove in names_to_remove: 185 for name_to_remove in names_to_remove:
157 del sys.modules[name_to_remove] 186 del sys.modules[name_to_remove]
158 return fresh_module 187 return fresh_module
159 188
160 189
161 def get_attribute(obj, name): 190 def get_attribute(obj, name):
162 """Get an attribute, raising SkipTest if AttributeError is raised.""" 191 """Get an attribute, raising SkipTest if AttributeError is raised."""
163 try: 192 try:
164 attribute = getattr(obj, name) 193 attribute = getattr(obj, name)
165 except AttributeError: 194 except AttributeError:
166 raise unittest.SkipTest("module %s has no attribute %s" % ( 195 raise unittest.SkipTest("object %r has no attribute %r" % (obj, name))
167 obj.__name__, name))
168 else: 196 else:
169 return attribute 197 return attribute
170 198
171 verbose = 1 # Flag set to 0 by regrtest.py 199 verbose = 1 # Flag set to 0 by regrtest.py
172 use_resources = None # Flag set to [] by regrtest.py 200 use_resources = None # Flag set to [] by regrtest.py
173 max_memuse = 0 # Disable bigmem tests (they will still be run with 201 max_memuse = 0 # Disable bigmem tests (they will still be run with
174 # small sizes, to make sure they work.) 202 # small sizes, to make sure they work.)
175 real_max_memuse = 0 203 real_max_memuse = 0
204 failfast = False
205 match_tests = None
176 206
177 # _original_stdout is meant to hold stdout at the time regrtest began. 207 # _original_stdout is meant to hold stdout at the time regrtest began.
178 # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. 208 # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
179 # The point is to have some flavor of stdout the user can actually see. 209 # The point is to have some flavor of stdout the user can actually see.
180 _original_stdout = None 210 _original_stdout = None
181 def record_original_stdout(stdout): 211 def record_original_stdout(stdout):
182 global _original_stdout 212 global _original_stdout
183 _original_stdout = stdout 213 _original_stdout = stdout
184 214
185 def get_original_stdout(): 215 def get_original_stdout():
(...skipping 10 matching lines...) Expand all
196 os.unlink(filename) 226 os.unlink(filename)
197 except OSError as error: 227 except OSError as error:
198 # The filename need not exist. 228 # The filename need not exist.
199 if error.errno not in (errno.ENOENT, errno.ENOTDIR): 229 if error.errno not in (errno.ENOENT, errno.ENOTDIR):
200 raise 230 raise
201 231
202 def rmtree(path): 232 def rmtree(path):
203 try: 233 try:
204 shutil.rmtree(path) 234 shutil.rmtree(path)
205 except OSError as error: 235 except OSError as error:
206 # Unix returns ENOENT, Windows returns ESRCH. 236 if error.errno != errno.ENOENT:
207 if error.errno not in (errno.ENOENT, errno.ESRCH):
208 raise 237 raise
209 238
210 def make_legacy_pyc(source): 239 def make_legacy_pyc(source):
211 """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location. 240 """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
212 241
213 The choice of .pyc or .pyo extension is done based on the __debug__ flag 242 The choice of .pyc or .pyo extension is done based on the __debug__ flag
214 value. 243 value.
215 244
216 :param source: The file system path to the source file. The source file 245 :param source: The file system path to the source file. The source file
217 does not need to exist, however the PEP 3147 pyc file must exist. 246 does not need to exist, however the PEP 3147 pyc file must exist.
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
282 executing. 311 executing.
283 """ 312 """
284 if resource == 'gui' and not _is_gui_available(): 313 if resource == 'gui' and not _is_gui_available():
285 raise unittest.SkipTest("Cannot use the 'gui' resource") 314 raise unittest.SkipTest("Cannot use the 'gui' resource")
286 # see if the caller's module is __main__ - if so, treat as if 315 # see if the caller's module is __main__ - if so, treat as if
287 # the resource was set 316 # the resource was set
288 if sys._getframe(1).f_globals.get("__name__") == "__main__": 317 if sys._getframe(1).f_globals.get("__name__") == "__main__":
289 return 318 return
290 if not is_resource_enabled(resource): 319 if not is_resource_enabled(resource):
291 if msg is None: 320 if msg is None:
292 msg = "Use of the `%s' resource not enabled" % resource 321 msg = "Use of the %r resource not enabled" % resource
293 raise ResourceDenied(msg) 322 raise ResourceDenied(msg)
294 323
295 def requires_linux_version(*min_version): 324 def _requires_unix_version(sysname, min_version):
296 """Decorator raising SkipTest if the OS is Linux and the kernel version is 325 """Decorator raising SkipTest if the OS is `sysname` and the version is less
297 less than min_version. 326 than `min_version`.
298 327
299 For example, @requires_linux_version(2, 6, 35) raises SkipTest if the Linux 328 For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if
300 kernel version is less than 2.6.35. 329 the FreeBSD version is less than 7.2.
301 """ 330 """
302 def decorator(func): 331 def decorator(func):
303 @functools.wraps(func) 332 @functools.wraps(func)
304 def wrapper(*args, **kw): 333 def wrapper(*args, **kw):
305 if sys.platform.startswith('linux'): 334 if platform.system() == sysname:
306 version_txt = platform.release().split('-', 1)[0] 335 version_txt = platform.release().split('-', 1)[0]
307 try: 336 try:
308 version = tuple(map(int, version_txt.split('.'))) 337 version = tuple(map(int, version_txt.split('.')))
309 except ValueError: 338 except ValueError:
310 pass 339 pass
311 else: 340 else:
312 if version < min_version: 341 if version < min_version:
313 min_version_txt = '.'.join(map(str, min_version)) 342 min_version_txt = '.'.join(map(str, min_version))
314 raise unittest.SkipTest( 343 raise unittest.SkipTest(
315 "Linux kernel %s or higher required, not %s" 344 "%s version %s or higher required, not %s"
316 % (min_version_txt, version_txt)) 345 % (sysname, min_version_txt, version_txt))
317 return func(*args, **kw)
318 wrapper.min_version = min_version
319 return wrapper 346 return wrapper
320 return decorator 347 return decorator
348
349 def requires_freebsd_version(*min_version):
350 """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version i s
351 less than `min_version`.
352
353 For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD
354 version is less than 7.2.
355 """
356 return _requires_unix_version('FreeBSD', min_version)
357
358 def requires_linux_version(*min_version):
359 """Decorator raising SkipTest if the OS is Linux and the Linux version is
360 less than `min_version`.
361
362 For example, @requires_linux_version(2, 6, 32) raises SkipTest if the Linux
363 version is less than 2.6.32.
364 """
365 return _requires_unix_version('Linux', min_version)
321 366
322 def requires_mac_ver(*min_version): 367 def requires_mac_ver(*min_version):
323 """Decorator raising SkipTest if the OS is Mac OS X and the OS X 368 """Decorator raising SkipTest if the OS is Mac OS X and the OS X
324 version if less than min_version. 369 version if less than min_version.
325 370
326 For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version 371 For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version
327 is lesser than 10.5. 372 is lesser than 10.5.
328 """ 373 """
329 def decorator(func): 374 def decorator(func):
330 @functools.wraps(func) 375 @functools.wraps(func)
(...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after
564 warnings.warn('tests may fail, unable to change the CWD to ' + name, 609 warnings.warn('tests may fail, unable to change the CWD to ' + name,
565 RuntimeWarning, stacklevel=3) 610 RuntimeWarning, stacklevel=3)
566 try: 611 try:
567 yield os.getcwd() 612 yield os.getcwd()
568 finally: 613 finally:
569 os.chdir(saved_dir) 614 os.chdir(saved_dir)
570 if is_temporary: 615 if is_temporary:
571 rmtree(name) 616 rmtree(name)
572 617
573 618
574 @contextlib.contextmanager 619 if hasattr(os, "umask"):
575 def temp_umask(umask): 620 @contextlib.contextmanager
576 """Context manager that temporarily sets the process umask.""" 621 def temp_umask(umask):
577 oldmask = os.umask(umask) 622 """Context manager that temporarily sets the process umask."""
578 try: 623 oldmask = os.umask(umask)
579 yield 624 try:
580 finally: 625 yield
581 os.umask(oldmask) 626 finally:
627 os.umask(oldmask)
582 628
583 629
584 def findfile(file, here=__file__, subdir=None): 630 def findfile(file, here=__file__, subdir=None):
585 """Try to find a file on sys.path and the working directory. If it is not 631 """Try to find a file on sys.path and the working directory. If it is not
586 found the argument passed to the function is returned (this does not 632 found the argument passed to the function is returned (this does not
587 necessarily signal failure; could still be the legitimate path).""" 633 necessarily signal failure; could still be the legitimate path)."""
588 if os.path.isabs(file): 634 if os.path.isabs(file):
589 return file 635 return file
590 if subdir is not None: 636 if subdir is not None:
591 file = os.path.join(subdir, file) 637 file = os.path.join(subdir, file)
592 path = sys.path 638 path = sys.path
593 path = [os.path.dirname(here)] + path 639 path = [os.path.dirname(here)] + path
594 for dn in path: 640 for dn in path:
595 fn = os.path.join(dn, file) 641 fn = os.path.join(dn, file)
596 if os.path.exists(fn): return fn 642 if os.path.exists(fn): return fn
597 return file 643 return file
598 644
645 def create_empty_file(filename):
646 """Create an empty file. If the file already exists, truncate it."""
647 fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
648 os.close(fd)
649
599 def sortdict(dict): 650 def sortdict(dict):
600 "Like repr(dict), but in sorted order." 651 "Like repr(dict), but in sorted order."
601 items = sorted(dict.items()) 652 items = sorted(dict.items())
602 reprpairs = ["%r: %r" % pair for pair in items] 653 reprpairs = ["%r: %r" % pair for pair in items]
603 withcommas = ", ".join(reprpairs) 654 withcommas = ", ".join(reprpairs)
604 return "{%s}" % withcommas 655 return "{%s}" % withcommas
605 656
606 def make_bad_fd(): 657 def make_bad_fd():
607 """ 658 """
608 Create an invalid file descriptor by opening and closing a file and return 659 Create an invalid file descriptor by opening and closing a file and return
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after
653 s = f.read() 704 s = f.read()
654 while s: 705 while s:
655 out.write(s) 706 out.write(s)
656 s = f.read() 707 s = f.read()
657 finally: 708 finally:
658 f.close() 709 f.close()
659 710
660 f = check_valid_file(fn) 711 f = check_valid_file(fn)
661 if f is not None: 712 if f is not None:
662 return f 713 return f
663 raise TestFailed('invalid resource "%s"' % fn) 714 raise TestFailed('invalid resource %r' % fn)
664 715
665 716
666 class WarningsRecorder(object): 717 class WarningsRecorder(object):
667 """Convenience wrapper for the warnings list returned on 718 """Convenience wrapper for the warnings list returned on
668 entry to the warnings.catch_warnings() context manager. 719 entry to the warnings.catch_warnings() context manager.
669 """ 720 """
670 def __init__(self, warnings_list): 721 def __init__(self, warnings_list):
671 self._warnings = warnings_list 722 self._warnings = warnings_list
672 self._last = 0 723 self._last = 0
673 724
(...skipping 223 matching lines...) Expand 10 before | Expand all | Expand 10 after
897 """Return a context manager that raises ResourceDenied when various issues 948 """Return a context manager that raises ResourceDenied when various issues
898 with the Internet connection manifest themselves as exceptions.""" 949 with the Internet connection manifest themselves as exceptions."""
899 default_errnos = [ 950 default_errnos = [
900 ('ECONNREFUSED', 111), 951 ('ECONNREFUSED', 111),
901 ('ECONNRESET', 104), 952 ('ECONNRESET', 104),
902 ('EHOSTUNREACH', 113), 953 ('EHOSTUNREACH', 113),
903 ('ENETUNREACH', 101), 954 ('ENETUNREACH', 101),
904 ('ETIMEDOUT', 110), 955 ('ETIMEDOUT', 110),
905 ] 956 ]
906 default_gai_errnos = [ 957 default_gai_errnos = [
958 ('EAI_AGAIN', -3),
959 ('EAI_FAIL', -4),
907 ('EAI_NONAME', -2), 960 ('EAI_NONAME', -2),
908 ('EAI_NODATA', -5), 961 ('EAI_NODATA', -5),
909 # Encountered when trying to resolve IPv6-only hostnames 962 # Encountered when trying to resolve IPv6-only hostnames
910 ('WSANO_DATA', 11004), 963 ('WSANO_DATA', 11004),
911 ] 964 ]
912 965
913 denied = ResourceDenied("Resource '%s' is not available" % resource_name) 966 denied = ResourceDenied("Resource %r is not available" % resource_name)
914 captured_errnos = errnos 967 captured_errnos = errnos
915 gai_errnos = [] 968 gai_errnos = []
916 if not captured_errnos: 969 if not captured_errnos:
917 captured_errnos = [getattr(errno, name, num) 970 captured_errnos = [getattr(errno, name, num)
918 for (name, num) in default_errnos] 971 for (name, num) in default_errnos]
919 gai_errnos = [getattr(socket, name, num) 972 gai_errnos = [getattr(socket, name, num)
920 for (name, num) in default_gai_errnos] 973 for (name, num) in default_gai_errnos]
921 974
922 def filter_error(err): 975 def filter_error(err):
923 n = getattr(err, 'errno', None) 976 n = getattr(err, 'errno', None)
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after
992 methods may be called later than expected and weakrefs may remain alive for 1045 methods may be called later than expected and weakrefs may remain alive for
993 longer than expected. This function tries its best to force all garbage 1046 longer than expected. This function tries its best to force all garbage
994 objects to disappear. 1047 objects to disappear.
995 """ 1048 """
996 gc.collect() 1049 gc.collect()
997 if is_jython: 1050 if is_jython:
998 time.sleep(0.1) 1051 time.sleep(0.1)
999 gc.collect() 1052 gc.collect()
1000 gc.collect() 1053 gc.collect()
1001 1054
1055 @contextlib.contextmanager
1056 def disable_gc():
1057 have_gc = gc.isenabled()
1058 gc.disable()
1059 try:
1060 yield
1061 finally:
1062 if have_gc:
1063 gc.enable()
1064
1002 1065
1003 def python_is_optimized(): 1066 def python_is_optimized():
1004 """Find if Python was built with optimizations.""" 1067 """Find if Python was built with optimizations."""
1005 cflags = sysconfig.get_config_var('PY_CFLAGS') or '' 1068 cflags = sysconfig.get_config_var('PY_CFLAGS') or ''
1006 final_opt = "" 1069 final_opt = ""
1007 for opt in cflags.split(): 1070 for opt in cflags.split():
1008 if opt.startswith('-O'): 1071 if opt.startswith('-O'):
1009 final_opt = opt 1072 final_opt = opt
1010 return final_opt and final_opt != '-O0' 1073 return final_opt != '' and final_opt != '-O0'
1011 1074
1012 1075
1013 #======================================================================= 1076 #=======================================================================
1014 # Decorator for running a function in a different locale, correctly resetting 1077 # Decorator for running a function in a different locale, correctly resetting
1015 # it afterwards. 1078 # it afterwards.
1016 1079
1017 def run_with_locale(catstr, *locales): 1080 def run_with_locale(catstr, *locales):
1018 def decorator(func): 1081 def decorator(func):
1019 def inner(*args, **kwds): 1082 def inner(*args, **kwds):
1020 try: 1083 try:
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
1073 if m is None: 1136 if m is None:
1074 raise ValueError('Invalid memory limit %r' % (limit,)) 1137 raise ValueError('Invalid memory limit %r' % (limit,))
1075 memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) 1138 memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
1076 real_max_memuse = memlimit 1139 real_max_memuse = memlimit
1077 if memlimit > MAX_Py_ssize_t: 1140 if memlimit > MAX_Py_ssize_t:
1078 memlimit = MAX_Py_ssize_t 1141 memlimit = MAX_Py_ssize_t
1079 if memlimit < _2G - 1: 1142 if memlimit < _2G - 1:
1080 raise ValueError('Memory limit %r too low to be useful' % (limit,)) 1143 raise ValueError('Memory limit %r too low to be useful' % (limit,))
1081 max_memuse = memlimit 1144 max_memuse = memlimit
1082 1145
1083 def bigmemtest(minsize, memuse): 1146 class _MemoryWatchdog:
1147 """An object which periodically watches the process' memory consumption
1148 and prints it out.
1149 """
1150
1151 def __init__(self):
1152 self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
1153 self.started = False
1154 self.thread = None
1155 try:
1156 self.page_size = os.sysconf('SC_PAGESIZE')
1157 except (ValueError, AttributeError):
1158 try:
1159 self.page_size = os.sysconf('SC_PAGE_SIZE')
1160 except (ValueError, AttributeError):
1161 self.page_size = 4096
1162
1163 def consumer(self, fd):
1164 HEADER = "l"
1165 header_size = struct.calcsize(HEADER)
1166 try:
1167 while True:
1168 header = os.read(fd, header_size)
1169 if len(header) < header_size:
1170 # Pipe closed on other end
1171 break
1172 data_len, = struct.unpack(HEADER, header)
1173 data = os.read(fd, data_len)
1174 statm = data.decode('ascii')
1175 data = int(statm.split()[5])
1176 print(" ... process data size: {data:.1f}G"
1177 .format(data=data * self.page_size / (1024 ** 3)))
1178 finally:
1179 os.close(fd)
1180
1181 def start(self):
1182 if not faulthandler or not hasattr(faulthandler, '_file_watchdog'):
1183 return
1184 try:
1185 rfd = os.open(self.procfile, os.O_RDONLY)
1186 except OSError as e:
1187 warnings.warn('/proc not available for stats: {}'.format(e),
1188 RuntimeWarning)
1189 sys.stderr.flush()
1190 return
1191 pipe_fd, wfd = os.pipe()
1192 # set the write end of the pipe non-blocking to avoid blocking the
1193 # watchdog thread when the consumer doesn't drain the pipe fast enough
1194 if fcntl:
1195 flags = fcntl.fcntl(wfd, fcntl.F_GETFL)
1196 fcntl.fcntl(wfd, fcntl.F_SETFL, flags|os.O_NONBLOCK)
1197 # _file_watchdog() doesn't take the GIL in its child thread, and
1198 # therefore collects statistics timely
1199 faulthandler._file_watchdog(rfd, wfd, 1.0)
1200 self.started = True
1201 self.thread = threading.Thread(target=self.consumer, args=(pipe_fd,))
1202 self.thread.daemon = True
1203 self.thread.start()
1204
1205 def stop(self):
1206 if not self.started:
1207 return
1208 faulthandler._cancel_file_watchdog()
1209 self.thread.join()
1210
1211
1212 def bigmemtest(size, memuse, dry_run=True):
1084 """Decorator for bigmem tests. 1213 """Decorator for bigmem tests.
1085 1214
1086 'minsize' is the minimum useful size for the test (in arbitrary, 1215 'minsize' is the minimum useful size for the test (in arbitrary,
1087 test-interpreted units.) 'memuse' is the number of 'bytes per size' for 1216 test-interpreted units.) 'memuse' is the number of 'bytes per size' for
1088 the test, or a good estimate of it. 1217 the test, or a good estimate of it.
1089 1218
1090 The decorator tries to guess a good value for 'size' and passes it to 1219 if 'dry_run' is False, it means the test doesn't support dummy runs
1091 the decorated test function. If minsize * memuse is more than the 1220 when -M is not specified.
1092 allowed memory use (as defined by max_memuse), the test is skipped.
1093 Otherwise, minsize is adjusted upward to use up to max_memuse.
1094 """
1095 def decorator(f):
1096 def wrapper(self):
1097 # Retrieve values in case someone decided to adjust them
1098 minsize = wrapper.minsize
1099 memuse = wrapper.memuse
1100 if not max_memuse:
1101 # If max_memuse is 0 (the default),
1102 # we still want to run the tests with size set to a few kb,
1103 # to make sure they work. We still want to avoid using
1104 # too much memory, though, but we do that noisily.
1105 maxsize = 5147
1106 self.assertFalse(maxsize * memuse > 20 * _1M)
1107 else:
1108 maxsize = int(max_memuse / memuse)
1109 if maxsize < minsize:
1110 raise unittest.SkipTest(
1111 "not enough memory: %.1fG minimum needed"
1112 % (minsize * memuse / (1024 ** 3)))
1113 return f(self, maxsize)
1114 wrapper.minsize = minsize
1115 wrapper.memuse = memuse
1116 return wrapper
1117 return decorator
1118
1119 def precisionbigmemtest(size, memuse):
1120 """Decorator for bigmem tests that need exact sizes.
1121
1122 Like bigmemtest, but without the size scaling upward to fill available
1123 memory.
1124 """ 1221 """
1125 def decorator(f): 1222 def decorator(f):
1126 def wrapper(self): 1223 def wrapper(self):
1127 size = wrapper.size 1224 size = wrapper.size
1128 memuse = wrapper.memuse 1225 memuse = wrapper.memuse
1129 if not real_max_memuse: 1226 if not real_max_memuse:
1130 maxsize = 5147 1227 maxsize = 5147
1131 else: 1228 else:
1132 maxsize = size 1229 maxsize = size
1133 1230
1134 if real_max_memuse and real_max_memuse < maxsize * memuse: 1231 if ((real_max_memuse or not dry_run)
1135 raise unittest.SkipTest( 1232 and real_max_memuse < maxsize * memuse):
1136 "not enough memory: %.1fG minimum needed" 1233 raise unittest.SkipTest(
1137 % (size * memuse / (1024 ** 3))) 1234 "not enough memory: %.1fG minimum needed"
1138 1235 % (size * memuse / (1024 ** 3)))
1139 return f(self, maxsize) 1236
1237 if real_max_memuse and verbose and faulthandler and threading:
1238 print()
1239 print(" ... expected peak memory use: {peak:.1f}G"
1240 .format(peak=size * memuse / (1024 ** 3)))
1241 watchdog = _MemoryWatchdog()
1242 watchdog.start()
1243 else:
1244 watchdog = None
1245
1246 try:
1247 return f(self, maxsize)
1248 finally:
1249 if watchdog:
1250 watchdog.stop()
1251
1140 wrapper.size = size 1252 wrapper.size = size
1141 wrapper.memuse = memuse 1253 wrapper.memuse = memuse
1142 return wrapper 1254 return wrapper
1143 return decorator 1255 return decorator
1144 1256
1145 def bigaddrspacetest(f): 1257 def bigaddrspacetest(f):
1146 """Decorator for tests that fill the address space.""" 1258 """Decorator for tests that fill the address space."""
1147 def wrapper(self): 1259 def wrapper(self):
1148 if max_memuse < MAX_Py_ssize_t: 1260 if max_memuse < MAX_Py_ssize_t:
1149 if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31: 1261 if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31:
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after
1237 """Decorator for tests which involve reference counting. 1349 """Decorator for tests which involve reference counting.
1238 1350
1239 To start, the decorator does not run the test if is not run by CPython. 1351 To start, the decorator does not run the test if is not run by CPython.
1240 After that, any trace function is unset during the test to prevent 1352 After that, any trace function is unset during the test to prevent
1241 unexpected refcounts caused by the trace function. 1353 unexpected refcounts caused by the trace function.
1242 1354
1243 """ 1355 """
1244 return no_tracing(cpython_only(test)) 1356 return no_tracing(cpython_only(test))
1245 1357
1246 1358
1359 def _filter_suite(suite, pred):
1360 """Recursively filter test cases in a suite based on a predicate."""
1361 newtests = []
1362 for test in suite._tests:
1363 if isinstance(test, unittest.TestSuite):
1364 _filter_suite(test, pred)
1365 newtests.append(test)
1366 else:
1367 if pred(test):
1368 newtests.append(test)
1369 suite._tests = newtests
1370
1247 def _run_suite(suite): 1371 def _run_suite(suite):
1248 """Run tests from a unittest.TestSuite-derived class.""" 1372 """Run tests from a unittest.TestSuite-derived class."""
1249 if verbose: 1373 if verbose:
1250 runner = unittest.TextTestRunner(sys.stdout, verbosity=2) 1374 runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
1375 failfast=failfast)
1251 else: 1376 else:
1252 runner = BasicTestRunner() 1377 runner = BasicTestRunner()
1253 1378
1254 result = runner.run(suite) 1379 result = runner.run(suite)
1255 if not result.wasSuccessful(): 1380 if not result.wasSuccessful():
1256 if len(result.errors) == 1 and not result.failures: 1381 if len(result.errors) == 1 and not result.failures:
1257 err = result.errors[0][1] 1382 err = result.errors[0][1]
1258 elif len(result.failures) == 1 and not result.errors: 1383 elif len(result.failures) == 1 and not result.errors:
1259 err = result.failures[0][1] 1384 err = result.failures[0][1]
1260 else: 1385 else:
1261 err = "multiple errors occurred" 1386 err = "multiple errors occurred"
1262 if not verbose: err += "; run in verbose mode for details" 1387 if not verbose: err += "; run in verbose mode for details"
1263 raise TestFailed(err) 1388 raise TestFailed(err)
1264 1389
1265 1390
1266 def run_unittest(*classes): 1391 def run_unittest(*classes):
1267 """Run tests from unittest.TestCase-derived classes.""" 1392 """Run tests from unittest.TestCase-derived classes."""
1268 valid_types = (unittest.TestSuite, unittest.TestCase) 1393 valid_types = (unittest.TestSuite, unittest.TestCase)
1269 suite = unittest.TestSuite() 1394 suite = unittest.TestSuite()
1270 for cls in classes: 1395 for cls in classes:
1271 if isinstance(cls, str): 1396 if isinstance(cls, str):
1272 if cls in sys.modules: 1397 if cls in sys.modules:
1273 suite.addTest(unittest.findTestCases(sys.modules[cls])) 1398 suite.addTest(unittest.findTestCases(sys.modules[cls]))
1274 else: 1399 else:
1275 raise ValueError("str arguments must be keys in sys.modules") 1400 raise ValueError("str arguments must be keys in sys.modules")
1276 elif isinstance(cls, valid_types): 1401 elif isinstance(cls, valid_types):
1277 suite.addTest(cls) 1402 suite.addTest(cls)
1278 else: 1403 else:
1279 suite.addTest(unittest.makeSuite(cls)) 1404 suite.addTest(unittest.makeSuite(cls))
1405 def case_pred(test):
1406 if match_tests is None:
1407 return True
1408 for name in test.id().split("."):
1409 if fnmatch.fnmatchcase(name, match_tests):
1410 return True
1411 return False
1412 _filter_suite(suite, case_pred)
1280 _run_suite(suite) 1413 _run_suite(suite)
1281 1414
1282 1415
1283 #======================================================================= 1416 #=======================================================================
1284 # doctest driver. 1417 # doctest driver.
1285 1418
1286 def run_doctest(module, verbosity=None, optionflags=0): 1419 def run_doctest(module, verbosity=None, optionflags=0):
1287 """Run doctest on the given module. Return (#failures, #tests). 1420 """Run doctest on the given module. Return (#failures, #tests).
1288 1421
1289 If optional argument verbosity is not specified (or is None), pass 1422 If optional argument verbosity is not specified (or is None), pass
1290 support's belief about verbosity on to doctest. Else doctest's 1423 support's belief about verbosity on to doctest. Else doctest's
1291 usual behavior is used (it searches sys.argv for -v). 1424 usual behavior is used (it searches sys.argv for -v).
1292 """ 1425 """
1293 1426
1294 import doctest 1427 import doctest
1295 1428
1296 if verbosity is None: 1429 if verbosity is None:
1297 verbosity = verbose 1430 verbosity = verbose
1298 else: 1431 else:
1299 verbosity = None 1432 verbosity = None
1300 1433
1301 # Direct doctest output (normally just errors) to real stdout; doctest 1434 f, t = doctest.testmod(module, verbose=verbosity, optionflags=optionflags)
1302 # output shouldn't be compared by regrtest. 1435 if f:
1303 save_stdout = sys.stdout 1436 raise TestFailed("%d of %d doctests failed" % (f, t))
1304 sys.stdout = get_original_stdout()
1305 try:
1306 f, t = doctest.testmod(module, verbose=verbosity,
1307 optionflags=optionflags)
1308 if f:
1309 raise TestFailed("%d of %d doctests failed" % (f, t))
1310 finally:
1311 sys.stdout = save_stdout
1312 if verbose: 1437 if verbose:
1313 print('doctest (%s) ... %d tests with zero failures' % 1438 print('doctest (%s) ... %d tests with zero failures' %
1314 (module.__name__, t)) 1439 (module.__name__, t))
1315 return f, t 1440 return f, t
1316 1441
1317 1442
1318 #======================================================================= 1443 #=======================================================================
1319 # Support for saving and restoring the imported modules. 1444 # Support for saving and restoring the imported modules.
1320 1445
1321 def modules_setup(): 1446 def modules_setup():
(...skipping 21 matching lines...) Expand all
1343 # NOTE: we use thread._count() rather than threading.enumerate() (or the 1468 # NOTE: we use thread._count() rather than threading.enumerate() (or the
1344 # moral equivalent thereof) because a threading.Thread object is still alive 1469 # moral equivalent thereof) because a threading.Thread object is still alive
1345 # until its __bootstrap() method has returned, even after it has been 1470 # until its __bootstrap() method has returned, even after it has been
1346 # unregistered from the threading module. 1471 # unregistered from the threading module.
1347 # thread._count(), on the other hand, only gets decremented *after* the 1472 # thread._count(), on the other hand, only gets decremented *after* the
1348 # __bootstrap() method has returned, which gives us reliable reference counts 1473 # __bootstrap() method has returned, which gives us reliable reference counts
1349 # at the end of a test run. 1474 # at the end of a test run.
1350 1475
1351 def threading_setup(): 1476 def threading_setup():
1352 if _thread: 1477 if _thread:
1353 return _thread._count(), 1478 return _thread._count(), threading._dangling.copy()
1354 else: 1479 else:
1355 return 1, 1480 return 1, ()
1356 1481
1357 def threading_cleanup(nb_threads): 1482 def threading_cleanup(*original_values):
1358 if not _thread: 1483 if not _thread:
1359 return 1484 return
1360 _MAX_COUNT = 10 1485 _MAX_COUNT = 10
1361 for count in range(_MAX_COUNT): 1486 for count in range(_MAX_COUNT):
1362 n = _thread._count() 1487 values = _thread._count(), threading._dangling
1363 if n == nb_threads: 1488 if values == original_values:
1364 break 1489 break
1365 time.sleep(0.1) 1490 time.sleep(0.1)
1491 gc_collect()
1366 # XXX print a warning in case of failure? 1492 # XXX print a warning in case of failure?
1367 1493
1368 def reap_threads(func): 1494 def reap_threads(func):
1369 """Use this function when threads are being used. This will 1495 """Use this function when threads are being used. This will
1370 ensure that the threads are cleaned up even when the test fails. 1496 ensure that the threads are cleaned up even when the test fails.
1371 If threading is unavailable this function does nothing. 1497 If threading is unavailable this function does nothing.
1372 """ 1498 """
1373 if not _thread: 1499 if not _thread:
1374 return func 1500 return func
1375 1501
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
1465 """ 1591 """
1466 stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip() 1592 stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip()
1467 return stderr 1593 return stderr
1468 1594
1469 def args_from_interpreter_flags(): 1595 def args_from_interpreter_flags():
1470 """Return a list of command-line arguments reproducing the current 1596 """Return a list of command-line arguments reproducing the current
1471 settings in sys.flags and sys.warnoptions.""" 1597 settings in sys.flags and sys.warnoptions."""
1472 flag_opt_map = { 1598 flag_opt_map = {
1473 'bytes_warning': 'b', 1599 'bytes_warning': 'b',
1474 'dont_write_bytecode': 'B', 1600 'dont_write_bytecode': 'B',
1601 'hash_randomization': 'R',
1475 'ignore_environment': 'E', 1602 'ignore_environment': 'E',
1476 'no_user_site': 's', 1603 'no_user_site': 's',
1477 'no_site': 'S', 1604 'no_site': 'S',
1478 'optimize': 'O', 1605 'optimize': 'O',
1479 'verbose': 'v', 1606 'verbose': 'v',
1480 } 1607 }
1481 args = [] 1608 args = []
1482 for flag, opt in flag_opt_map.items(): 1609 for flag, opt in flag_opt_map.items():
1483 v = getattr(sys.flags, flag) 1610 v = getattr(sys.flags, flag)
1484 if v > 0: 1611 if v > 0:
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after
1600 def cleanup(): 1727 def cleanup():
1601 if attr_is_local: 1728 if attr_is_local:
1602 setattr(object_to_patch, attr_name, old_value) 1729 setattr(object_to_patch, attr_name, old_value)
1603 else: 1730 else:
1604 delattr(object_to_patch, attr_name) 1731 delattr(object_to_patch, attr_name)
1605 1732
1606 test_instance.addCleanup(cleanup) 1733 test_instance.addCleanup(cleanup)
1607 1734
1608 # actually override the attribute 1735 # actually override the attribute
1609 setattr(object_to_patch, attr_name, new_value) 1736 setattr(object_to_patch, attr_name, new_value)
LEFTRIGHT

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+