diff --git a/Doc/library/packaging.util.rst b/Doc/library/packaging.util.rst --- a/Doc/library/packaging.util.rst +++ b/Doc/library/packaging.util.rst @@ -118,7 +118,7 @@ This module contains various helpers for .. function:: byte_compile(py_files, optimize=0, force=0, prefix=None, \ - base_dir=None, dry_run=0, direct=None) + base_dir=None, dry_run=0) Byte-compile a collection of Python source files to either :file:`.pyc` or :file:`.pyo` files in a :file:`__pycache__` subdirectory (see :pep:`3147`), @@ -146,10 +146,3 @@ This module contains various helpers for If *dry_run* is true, doesn't actually do anything that would affect the filesystem. - - Byte-compilation is either done directly in this interpreter process with the - standard :mod:`py_compile` module, or indirectly by writing a temporary - script and executing it. Normally, you should let :func:`byte_compile` - figure out to use direct compilation or not (see the source for details). - The *direct* flag is used by the script generated in indirect mode; unless - you know what you're doing, leave it set to ``None``. diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py --- a/Lib/packaging/tests/support.py +++ b/Lib/packaging/tests/support.py @@ -38,7 +38,9 @@ import shutil import logging import weakref import tempfile +import textwrap import sysconfig +from subprocess import Popen, PIPE from packaging.dist import Distribution from packaging.util import resolve_name @@ -56,6 +58,8 @@ __all__ = [ # misc. functions and decorators 'fake_dec', 'create_distribution', 'use_command', 'copy_xxmodule_c', 'fixup_build_ext', + # functions used in byte-compilation tests + 'create_py_module', 'check_pyc_file', 'check_pyo_file', # imported from this module for backport purposes 'unittest', 'requires_zlib', 'skip_2to3_optimize', 'skip_unless_symlink', ] @@ -381,6 +385,88 @@ def fixup_build_ext(cmd): cmd.library_dirs = value.split(os.pathsep) +def create_py_module(filename): + """Create a Python module suitable for some tests. + + *filename* is a string containing the filename to write to. + This function should be used with check_pyc_file and check_pyo_file, + see test_command_build_py.py for an example. + """ + # The documentation of -O mode is quite brief; experimentation + source + # code reading (Py_OptimizeFlag then c_optimize in Python/compile.c) + # find these optimizations: -O sets __debug__ to False and disables + # asserts, -OO also sets __doc__ attributes to None. The following code + # exercises all of these things. + with open(filename, 'w') as fp: + fp.write(textwrap.dedent('''\ + """Module docstring.""" + + print('debug:', __debug__) + + try: + assert False + except AssertionError: + print('assertion: failed as expected') + else: + print('assertion: disabled') + + print('docstring:', __doc__) + ''')) + + +def check_pyc_file(testcase, modname, basedir): + """Make sure that a pyc file is compiled with optimization off. + + *testcase* should be a unittest.TestCase instance. *modname* is the + name of a module for which there exists a .pyc file created by + byte-compiling a module written by create_py_module. *basedir* is + the parent directory of the .py file. + """ + # using a subprocess to avoid module caching issues and to be independent + # of the calling Python's -O option -> we can call check_pyc_file and + # check_pyo_file in a row, from a Python started with or without -O + with Popen([sys.executable, '-E', '-m', modname], cwd=basedir, + stdout=PIPE, stderr=PIPE) as process: + output = process.stdout.read() + + output = output.decode('ascii').splitlines() + testcase.assertEqual(process.returncode, 0) + testcase.assertEqual(output, ['debug: True', + 'assertion: failed as expected', + 'docstring: Module docstring.']) + + +def check_pyo_file(testcase, modname, basedir, level): + """Make sure that a pyo file is compiled with the right optimizations. + + *level* should be 1 or 2 (matching python -O or -OO). See + check_pyc_file for the other arguments. + """ + if level == 1: + flag = '-O' + docstring = 'docstring: Module docstring.' + elif level == 2: + flag = '-OO' + docstring = 'docstring: None' + else: + raise ValueError('level must be 1 or 2, not %r' % level) + + # an existing .pyc file is not considered for import, but if there is a + # .pyo file created with -O, a call with -OO will not recreate it, so + # pay attention to byte-compilation when writing tests (i.e. use different + # methods or module names to test code with different optimization levels) + + with Popen([sys.executable, '-E', flag, '-m', modname], cwd=basedir, + stdout=PIPE, stderr=PIPE) as process: + output = process.stdout.read() + + output = output.decode('ascii').splitlines() + testcase.assertEqual(process.returncode, 0) + testcase.assertEqual(output, ['debug: False', + 'assertion: disabled', + docstring]) + + try: from test.support import skip_unless_symlink except ImportError: diff --git a/Lib/packaging/tests/test_command_build_py.py b/Lib/packaging/tests/test_command_build_py.py --- a/Lib/packaging/tests/test_command_build_py.py +++ b/Lib/packaging/tests/test_command_build_py.py @@ -6,7 +6,7 @@ import imp from packaging.command.build_py import build_py from packaging.dist import Distribution -from packaging.errors import PackagingFileError +from packaging.errors import PackagingFileError, PackagingOptionError from packaging.tests import unittest, support @@ -88,10 +88,29 @@ class BuildPyTestCase(support.TempdirMan except PackagingFileError: self.fail("failed package_data test when package_dir is ''") + def test_finalize_options(self): + dist = self.create_dist()[1] + cmd = build_py(dist) + + cmd.finalize_options() + self.assertFalse(cmd.compile) + self.assertEqual(cmd.optimize, 0) + + # optimize must be 0, 1, or 2 + # FIXME the value is checked only if it's not an int + for valid in (0, 1, 2, '1'): + cmd.optimize = valid + cmd.finalize_options() + self.assertEqual(cmd.optimize, int(valid)) + + for invalid in ('foo', '4', '-1'): + cmd.optimize = invalid + self.assertRaises(PackagingOptionError, cmd.finalize_options) + def test_byte_compile(self): project_dir, dist = self.create_dist(py_modules=['boiledeggs']) os.chdir(project_dir) - self.write_file('boiledeggs.py', 'import antigravity') + support.create_py_module('boiledeggs.py') cmd = build_py(dist) cmd.compile = True cmd.build_lib = 'here' @@ -102,12 +121,14 @@ class BuildPyTestCase(support.TempdirMan self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) self.assertEqual(found, ['boiledeggs.%s.pyc' % imp.get_tag()]) + support.check_pyc_file(self, 'boiledeggs', cmd.build_lib) def test_byte_compile_optimized(self): project_dir, dist = self.create_dist(py_modules=['boiledeggs']) os.chdir(project_dir) - self.write_file('boiledeggs.py', 'import antigravity') + support.create_py_module('boiledeggs.py') cmd = build_py(dist) + # this also tests that --compile and --optimize work together cmd.compile = True cmd.optimize = 1 cmd.build_lib = 'here' @@ -119,6 +140,26 @@ class BuildPyTestCase(support.TempdirMan found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) self.assertEqual(sorted(found), ['boiledeggs.%s.pyc' % imp.get_tag(), 'boiledeggs.%s.pyo' % imp.get_tag()]) + support.check_pyc_file(self, 'boiledeggs', cmd.build_lib) + support.check_pyo_file(self, 'boiledeggs', cmd.build_lib, level=1) + + def test_byte_compile_more_optimized(self): + project_dir, dist = self.create_dist(py_modules=['boiledeggs']) + os.chdir(project_dir) + support.create_py_module('boiledeggs.py') + cmd = build_py(dist) + # this also tests that --optimize doesn't depend on --compile + cmd.compile = False + cmd.optimize = 2 + cmd.build_lib = 'here' + cmd.finalize_options() + cmd.run() + + found = os.listdir(cmd.build_lib) + self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) + found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) + self.assertEqual(sorted(found), ['boiledeggs.%s.pyo' % imp.get_tag()]) + support.check_pyo_file(self, 'boiledeggs', cmd.build_lib, level=2) def test_byte_compile_under_B(self): # make sure byte compilation works under -B (dont_write_bytecode) @@ -127,6 +168,7 @@ class BuildPyTestCase(support.TempdirMan sys.dont_write_bytecode = True self.test_byte_compile() self.test_byte_compile_optimized() + self.test_byte_compile_more_optimized() def test_suite(): diff --git a/Lib/packaging/tests/test_command_install_lib.py b/Lib/packaging/tests/test_command_install_lib.py --- a/Lib/packaging/tests/test_command_install_lib.py +++ b/Lib/packaging/tests/test_command_install_lib.py @@ -25,29 +25,74 @@ class InstallLibTestCase(support.Tempdir self.assertEqual(cmd.optimize, 0) # optimize must be 0, 1, or 2 - cmd.optimize = 'foo' - self.assertRaises(PackagingOptionError, cmd.finalize_options) - cmd.optimize = '4' - self.assertRaises(PackagingOptionError, cmd.finalize_options) + # FIXME the value is checked only if it's not an int + for valid in (0, 1, 2, '1'): + cmd.optimize = valid + cmd.finalize_options() + self.assertEqual(cmd.optimize, int(valid)) - cmd.optimize = '2' - cmd.finalize_options() - self.assertEqual(cmd.optimize, 2) + for invalid in ('foo', '4', '-1'): + cmd.optimize = invalid + self.assertRaises(PackagingOptionError, cmd.finalize_options) def test_byte_compile(self): - project_dir, dist = self.create_dist() + project_dir, dist = self.create_dist(py_modules=['boiledeggs']) + install_dir = self.mkdtemp() os.chdir(project_dir) + support.create_py_module('boiledeggs.py') cmd = install_lib(dist) + cmd.install_dir = install_dir + cmd.finalize_options() + cmd.run() + + py_file = os.path.join(install_dir, 'boiledeggs.py') + pyc_file = imp.cache_from_source(py_file, True) + pyo_file = imp.cache_from_source(py_file, False) + # --compile is enabled by default, --optimize disabled + self.assertTrue(os.path.exists(pyc_file)) + self.assertFalse(os.path.exists(pyo_file)) + support.check_pyc_file(self, 'boiledeggs', install_dir) + + def test_byte_compile_optimized(self): + project_dir, dist = self.create_dist(py_modules=['boiledeggs']) + install_dir = self.mkdtemp() + os.chdir(project_dir) + support.create_py_module('boiledeggs.py') + cmd = install_lib(dist) + # this also tests that --compile and --optimize work together cmd.compile = True cmd.optimize = 1 + cmd.install_dir = install_dir + cmd.finalize_options() + cmd.run() - f = os.path.join(project_dir, 'foo.py') - self.write_file(f, '# python file') - cmd.byte_compile([f]) - pyc_file = imp.cache_from_source('foo.py', True) - pyo_file = imp.cache_from_source('foo.py', False) + py_file = os.path.join(install_dir, 'boiledeggs.py') + pyc_file = imp.cache_from_source(py_file, True) + pyo_file = imp.cache_from_source(py_file, False) self.assertTrue(os.path.exists(pyc_file)) self.assertTrue(os.path.exists(pyo_file)) + support.check_pyc_file(self, 'boiledeggs', install_dir) + support.check_pyo_file(self, 'boiledeggs', install_dir, level=1) + + def test_byte_compile_more_optimized(self): + project_dir, dist = self.create_dist(py_modules=['boiledeggs']) + install_dir = self.mkdtemp() + os.chdir(project_dir) + support.create_py_module('boiledeggs.py') + cmd = install_lib(dist) + # this also tests that --optimize doesn't depend on --compile + cmd.compile = False + cmd.optimize = 2 + cmd.install_dir = install_dir + cmd.finalize_options() + cmd.run() + + py_file = os.path.join(install_dir, 'boiledeggs.py') + pyc_file = imp.cache_from_source(py_file, True) + pyo_file = imp.cache_from_source(py_file, False) + self.assertFalse(os.path.exists(pyc_file)) + self.assertTrue(os.path.exists(pyo_file)) + support.check_pyo_file(self, 'boiledeggs', install_dir, level=2) def test_byte_compile_under_B(self): # make sure byte compilation works under -B (dont_write_bytecode) @@ -55,6 +100,8 @@ class InstallLibTestCase(support.Tempdir sys.dont_write_bytecode) sys.dont_write_bytecode = True self.test_byte_compile() + self.test_byte_compile_optimized() + self.test_byte_compile_more_optimized() def test_get_outputs(self): project_dir, dist = self.create_dist() diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py --- a/Lib/packaging/tests/test_util.py +++ b/Lib/packaging/tests/test_util.py @@ -324,12 +324,36 @@ class UtilTestCase(support.EnvironRestor res = get_compiler_versions() self.assertEqual(res[2], None) - def test_byte_compile_under_B(self): + def test_byte_compile(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, 'spam.py') + support.create_py_module(filename) + byte_compile([filename]) + support.check_pyc_file(self, 'spam', tmpdir) + + filename = os.path.join(tmpdir, 'spamo.py') + support.create_py_module(filename) + byte_compile([filename], optimize=1) + support.check_pyo_file(self, 'spamo', tmpdir, level=1) + + filename = os.path.join(tmpdir, 'spamoo.py') + support.create_py_module(filename) + byte_compile([filename], optimize=2) + support.check_pyo_file(self, 'spamoo', tmpdir, level=2) + + # type and legal values for the optimize argument are well documented + # but not actually enforced + #self.assertRaises(ValueError, byte_compile, [filename], optimize=-1) + #self.assertRaises(ValueError, byte_compile, [filename], optimize=3) + # make sure byte compilation works under -B (dont_write_bytecode) self.addCleanup(setattr, sys, 'dont_write_bytecode', sys.dont_write_bytecode) sys.dont_write_bytecode = True - byte_compile([]) + filename = os.path.join(tmpdir, 'spamb.py') + support.create_py_module(filename) + byte_compile([filename]) + support.check_pyc_file(self, 'spamb', tmpdir) def test_newer(self): self.assertRaises(PackagingFileError, util.newer, 'xxx', 'xxx') diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -296,7 +296,7 @@ def strtobool(val): def byte_compile(py_files, optimize=0, force=False, prefix=None, - base_dir=None, dry_run=False, direct=None): + base_dir=None, dry_run=False): """Byte-compile a collection of Python source files to either .pyc or .pyo files in a __pycache__ subdirectory. @@ -320,122 +320,41 @@ def byte_compile(py_files, optimize=0, f If 'dry_run' is true, doesn't actually do anything that would affect the filesystem. + """ + # XXX see if compileall wouldn't be nicer + from py_compile import compile - Byte-compilation is either done directly in this interpreter process - with the standard py_compile module, or indirectly by writing a - temporary script and executing it. Normally, you should let - 'byte_compile()' figure out to use direct compilation or not (see - the source for details). The 'direct' flag is used by the script - generated in indirect mode; unless you know what you're doing, leave - it set to None. - """ - # FIXME use compileall + remove direct/indirect shenanigans + for file in py_files: + if file[-3:] != ".py": + # This lets us be lazy and not filter filenames in + # the "install_lib" command. + continue - # First, if the caller didn't force us into direct or indirect mode, - # figure out which mode we should be in. We take a conservative - # approach: choose direct mode *only* if the current interpreter is - # in debug mode and optimize is 0. If we're not in debug mode (-O - # or -OO), we don't know which level of optimization this - # interpreter is running with, so we can't do direct - # byte-compilation and be certain that it's the right thing. Thus, - # always compile indirectly if the current interpreter is in either - # optimize mode, or if either optimization level was requested by - # the caller. - if direct is None: - direct = (__debug__ and optimize == 0) + # Terminology from the py_compile module: + # cfile - byte-compiled file + # dfile - purported source filename (same as 'file' by default) + # The second argument to cache_from_source forces the extension to + # be .pyc (if true) or .pyo (if false); without it, the extension + # would depend on the calling Python's -O option + cfile = imp.cache_from_source(file, not optimize) + dfile = file - # "Indirect" byte-compilation: write a temporary script and then - # run it with the appropriate flags. - if not direct: - from tempfile import mkstemp - # XXX use something better than mkstemp - script_fd, script_name = mkstemp(".py") - os.close(script_fd) - script_fd = None - logger.info("writing byte-compilation script '%s'", script_name) - if not dry_run: - if script_fd is not None: - script = os.fdopen(script_fd, "w", encoding='utf-8') - else: - script = open(script_name, "w", encoding='utf-8') + if prefix: + if file[:len(prefix)] != prefix: + raise ValueError("invalid prefix: filename %r doesn't " + "start with %r" % (file, prefix)) + dfile = dfile[len(prefix):] + if base_dir: + dfile = os.path.join(base_dir, dfile) - with script: - script.write("""\ -from packaging.util import byte_compile -files = [ -""") - - # XXX would be nice to write absolute filenames, just for - # safety's sake (script should be more robust in the face of - # chdir'ing before running it). But this requires abspath'ing - # 'prefix' as well, and that breaks the hack in build_lib's - # 'byte_compile()' method that carefully tacks on a trailing - # slash (os.sep really) to make sure the prefix here is "just - # right". This whole prefix business is rather delicate -- the - # problem is that it's really a directory, but I'm treating it - # as a dumb string, so trailing slashes and so forth matter. - - #py_files = map(os.path.abspath, py_files) - #if prefix: - # prefix = os.path.abspath(prefix) - - script.write(",\n".join(map(repr, py_files)) + "]\n") - script.write(""" -byte_compile(files, optimize=%r, force=%r, - prefix=%r, base_dir=%r, - dry_run=False, - direct=True) -""" % (optimize, force, prefix, base_dir)) - - cmd = [sys.executable, script_name] - - env = os.environ.copy() - env['PYTHONPATH'] = os.path.pathsep.join(sys.path) - try: - spawn(cmd, env=env) - finally: - execute(os.remove, (script_name,), "removing %s" % script_name, - dry_run=dry_run) - - # "Direct" byte-compilation: use the py_compile module to compile - # right here, right now. Note that the script generated in indirect - # mode simply calls 'byte_compile()' in direct mode, a weird sort of - # cross-process recursion. Hey, it works! - else: - from py_compile import compile - - for file in py_files: - if file[-3:] != ".py": - # This lets us be lazy and not filter filenames in - # the "install_lib" command. - continue - - # Terminology from the py_compile module: - # cfile - byte-compiled file - # dfile - purported source filename (same as 'file' by default) - # The second argument to cache_from_source forces the extension to - # be .pyc (if true) or .pyo (if false); without it, the extension - # would depend on the calling Python's -O option - cfile = imp.cache_from_source(file, not optimize) - dfile = file - - if prefix: - if file[:len(prefix)] != prefix: - raise ValueError("invalid prefix: filename %r doesn't " - "start with %r" % (file, prefix)) - dfile = dfile[len(prefix):] - if base_dir: - dfile = os.path.join(base_dir, dfile) - - cfile_base = os.path.basename(cfile) - if direct: - if force or newer(file, cfile): - logger.info("byte-compiling %s to %s", file, cfile_base) - if not dry_run: - compile(file, cfile, dfile) - else: - logger.debug("skipping byte-compilation of %s to %s", - file, cfile_base) + cfile_base = os.path.basename(cfile) + if force or newer(file, cfile): + logger.info("byte-compiling %s to %s", file, cfile_base) + if not dry_run: + compile(file, cfile, dfile) + else: + logger.debug("skipping byte-compilation of %s to %s", + file, cfile_base) _RE_VERSION = re.compile('(\d+\.\d+(\.\d+)*)')