diff -r 5ba3b222a866 Doc/library/test.rst --- a/Doc/library/test.rst Sat Jun 06 13:12:40 2015 -0700 +++ b/Doc/library/test.rst Sat Jun 06 14:57:41 2015 -0700 @@ -632,3 +632,201 @@ Class used to record warnings for unit tests. See documentation of :func:`check_warnings` above for more details. + + +Helper utilities for script creation and execution +-------------------------------------------------- + +.. module:: test.support.script_helper + :synopsis: Script creation and execution + +The :mod:`test.support.script_helper` module defines common utilities used +across various tests, such as creating scripts in temporary directories or +running python subprocesses based on given arguments. + +This module defines the following functions: + +.. function:: assert_python_ok(*args, **env_vars) + + Runs the interpreter with *args* and optional environment + variables, asserting that *env_vars* succeeds (``rc == 0``). + + Returns a (return code, stdout, stderr) tuple on success, throws an + appropriate :exc:`AssertionError`` for a non-zero return code containing + stdout and stderr of the failed process. + + If the *__cleanenv* keyword is set, *env_vars* is used as a fresh + environment, otherwise it is added to the environment of the current + process. + + Python is started in isolated mode (command line option ``-I``), + except if the *__isolated* keyword is present and set to False. + +.. function:: assert_python_failure(*args, **env_vars) + + Runs the interpreter with *args* and optional environment + variables, asserting that *env_vars* fails (``rc != 0``). + + Returns a (return code, stdout, stderr) tuple on failure, throws an + appropriate :exc:`AssertionError`` if the return code is zero containing + stdout and stderr of the failed process. + + If the *__cleanenv* keyword is set, *env_vars* is used as a fresh + environment, otherwise it is added to the environment of the current + process. + + Python is started in isolated mode (command line option ``-I``), + except if the *__isolated* keyword is present and set to False. + +.. function:: spawn_python(*args, **kw) + + Runs a Python subprocess with the given arguments and returns the + resulting :class:`subprocess.Popen` instance. + + The ``-E`` option is always passed to the subprocess, both ``stdin`` and + ``stdout`` are configured as binary pipes and ``stderr`` is merged with + ``stdout``.. + + *kw* is passed through to subprocess.Popen as additional keyword + arguments. + +.. function:: kill_python(p) + + Runs the given Popen process until completion and return stdout. + + This will also include stderr output if the process was started with + :func:`spawn_python`. + +.. function:: make_script(script_dir, script_basename, \ + source, omit_suffix=False) + + Creates a new Python script/module in the given directory. + + This function invalidates the import system caches, allowing the created + file to be immediately imported. + + By default, *script_dir* and *script_basename* are combined with + :data:`os.extsep` and the normal ``py`` extension to create the full + path to the file. If *omit_suffix* is true, the base name is used + directly as the name of created file with no extension. + + *source* is a Unicode string which will be written to the file as UTF-8. + +.. function:: make_zip_script(zip_dir, zip_basename, \ + script_name, name_in_zip=None) + + Creates a new zip archive in the given directory, containing the specified + Python script/module + + This function invalidates the import system caches, allowing the created + file to be immediately imported. + + By default, the script is placed at the base of the zip archive using + just the same filename as the script itself. This can be changed by + specifying the *name_in_zip* parameter to choose a particular name. + + There is also a special case for when *script_name* refers to a file + in a ``__pycache__`` directory. In that case, the helper will create a + new compiled bytecode file from the original source file using the + legacy naming scheme and place that in the zip archive instead. + +.. function:: make_pkg(pkg_dir, init_source='') + + Creates a simple self-contained Python package, optionally with a + non-empty init file. + + This function invalidates the import system caches, allowing the created + directory to be immediately imported. + + Equivalent to:: + + os.mkdir(pkg_dir) + make_script(pkg_dir, "__init__", init_source) + + +.. function:: make_zip_pkg(zip_dir, zip_basename, \ + pkg_name, script_basename, source, \ + depth=1, compiled=False) + + Create a self-contained Python package inside a new zip archive in the + given directory. + + This function invalidates the import system caches, allowing the contents + of the created archive to be immediately imported. + + For example, this call:: + + make_zip_pkg('.', 'example', 'mypkg', 'submodule', '', depth=2) + + Would create an archive in the current directory called ``example.zip`` + with the layout:: + + mypkg/ + __init__.py + mypkg/ + __init__.py + submodule.py + + The init modules in the packages inside the archive are always empty, + while *source* is written to the submodule named by *script_basename*. + + *depth* controls how many deeply nested the package is. + + If *compiled* is true, then the the files added to the archive will be + suitable named compiled bytecode files rather than the original source + files. + +.. function:: interpreter_requires_environment() + Returns True if our `sys.executable interpreter` requires environment + variables in order to be able to run at all. + + This is designed to be used with @unittest.skipIf() to annotate tests + that need to use an assert_python*() function to launch an isolated + mode (-I) or no environment mode (-E) sub-interpreter process. + + A normal build & test does not run into this situation but it can happen + when trying to run the standard library test suite from an interpreter that + doesn't have an obvious home with Python's current home finding logic. + + Setting PYTHONHOME is one way to get most of the testsuite to run in that + situation. PYTHONPATH or PYTHONUSERSITE are other common environment + variables that might impact whether or not the interpreter can start. + +.. function:: run_python(*args, **env_vars) + + Starts a python interpreter process with *args* and environment variables + **env_vars. Pipes are created for stdout, stdin and stderr. Returns the + resulting :class:`subprocess.Popen` instance. + + If the *__cleanenv* keyword is set, *env_vars* is used as a fresh + environment, otherwise it is added to the environment of the current + process. + + Python is started in isolated mode (command line option ``-I``), + except if the *__isolated* keyword is present and set to False. + +.. function:: wait_for_process(p) + + Waits for the specified process instance to complete and returns the + runcode, stdout and stderr as a `_PythonRunResult`. + +.. function:: terminate_process(p) + + Terminates the specified process, waits for it to complete, and returns the + runcode, stdout and stderr as a `_PythonRunResult`. + +.. function:: run_python_until_end(*args, **env_vars) + + Starts a python interpreter process with *args* and environment variables + **env_vars. Pipes are created for stdout, stdin and stderr. + + Waits until the process completes and returns the runcode, stdout and + stderr as a `_PythonRunResult`. + + If the *__cleanenv* keyword is set, *env_vars* is used as a fresh + environment, otherwise it is added to the environment of the current + process. + + Python is started in isolated mode (command line option ``-I``), + except if the *__isolated* keyword is present and set to False. + diff -r 5ba3b222a866 Lib/test/support/script_helper.py --- a/Lib/test/support/script_helper.py Sat Jun 06 13:12:40 2015 -0700 +++ b/Lib/test/support/script_helper.py Sat Jun 06 14:57:41 2015 -0700 @@ -6,11 +6,8 @@ import sys import os import os.path -import tempfile import subprocess import py_compile -import contextlib -import shutil import zipfile from importlib.util import source_from_cache @@ -21,21 +18,12 @@ __cached_interp_requires_environment = None def interpreter_requires_environment(): - """ - Returns True if our sys.executable interpreter requires environment + """ Are environment variables required to run interpreter + + Returns True if our `sys.executable` interpreter requires environment variables in order to be able to run at all. - This is designed to be used with @unittest.skipIf() to annotate tests - that need to use an assert_python*() function to launch an isolated - mode (-I) or no environment mode (-E) sub-interpreter process. - - A normal build & test does not run into this situation but it can happen - when trying to run the standard library test suite from an interpreter that - doesn't have an obvious home with Python's current home finding logic. - - Setting PYTHONHOME is one way to get most of the testsuite to run in that - situation. PYTHONPATH or PYTHONUSERSITE are other common environment - variables that might impact whether or not the interpreter can start. + :return: Boolean indicating if environment variables are needed """ global __cached_interp_requires_environment if __cached_interp_requires_environment is None: @@ -80,11 +68,11 @@ return env, command_line_args def run_python(*args, **env_vars): - """ - Start a python interpreter process. + """ Start a python interpreter process. + :param args: Arguments to pass to the process. :param env_vars: Environment variables to apply to the process. - :return: Instance of `_PythonRunResult` containing the result. + :return: An instance of `subprocess.Popen` """ env, command_line_args = _get_python_process_args(*args, **env_vars) return subprocess.Popen(command_line_args, stdin=subprocess.PIPE, @@ -92,8 +80,8 @@ env=env) def wait_for_process(p): - """ - Wait for the specified process to complete and return the result. + """ Wait for the specified process to complete and return the result. + :param p: Instance of process returned by `Popen`. :return: Instance of `_PythonRunResult` containing the result. """ @@ -107,8 +95,7 @@ return _PythonRunResult(p.returncode, out, err) def terminate_process(p): - """ - Terminate the specified process and return the result. + """ Terminate the specified process and return the result. :param p: Instance of process returned by `Popen`. :return: Instance of `_PythonRunResult` containing the result. """ @@ -126,9 +113,11 @@ return _PythonRunResult(p.returncode, out, err) def run_python_until_end(*args, **env_vars): - """ + """ Run interpreter and wait for it to complete. + Run python interpreter process with the specified arguments, wait for it to complete and return the results. + :param args: Arguments to the python process. :param env_vars: Environment variables to apply to the process. :return: `_PythonRunResult` containing the results. @@ -222,6 +211,19 @@ return data def make_script(script_dir, script_basename, source, omit_suffix=False): + """ Create new Python script/module in the given directory. + + This function invalidates the import system caches, allowing the created + file to be immediately imported. + + File contents will be written as utf-8. + + :param script_dir: Directory to put the script into. + :param script_basename: Name of the file, without the extension. + :param source: File contents as unicode string. + :param omit_suffix: If true, the "py" extension will not be appendd. + :return: Path to the script. + """ script_filename = script_basename if not omit_suffix: script_filename += os.extsep + 'py' @@ -234,6 +236,23 @@ return script_name def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): + """ Create new zip archive in given directory. + + This function invalidates the import system caches, allowing the created + file to be immediately imported. + + There is also a special case for when *script_name* refers to a file + in a ``__pycache__`` directory. In that case, the helper will create a + new compiled bytecode file from the original source file using the + legacy naming scheme and place that in the zip archive instead. + + :param zip_dir: Directory to put the zip file in. + :param zip_basename: Name of the file without the extension. + :param script_name: Python script/module to place in the archive. + :param name_in_zip: Name to use for the script/module within the archive. + :return: Tuple containing the path to the zip file and the path to the + script/module within the archive. + """ zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) zip_file = zipfile.ZipFile(zip_name, 'w') @@ -255,11 +274,35 @@ return zip_name, os.path.join(zip_name, name_in_zip) def make_pkg(pkg_dir, init_source=''): + """ Create a simple Python package. + + :param pkg_dir: Directory to create package in. + :param init_source: Package contents. + :return: None + """ os.mkdir(pkg_dir) make_script(pkg_dir, '__init__', init_source) def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, source, depth=1, compiled=False): + """ Create python package inside zip archive. + + Create a self-contained Python package inside a new zip archive in the + given directory. + + This function invalidates the import system caches, allowing the contents + of the created archive to be immediately imported. + + :param zip_dir: Directory to put the zip file in. + :param zip_basename: Name of the file without the extension. + :param pkg_name: Name of the package to be created. + :param script_basename: Name of the submodule file, without the extension. + :param source: File contents as unicode string. + :param depth: How deeply nested the package should be. + :param compiled: If true, files added to the archive will be bytecode. + :return: Tuple containing the path to the zip file and the path to the + script/module within the archive. + """ unlink = [] init_name = make_script(zip_dir, '__init__', '') unlink.append(init_name)