diff -r 1892f4567b07 Doc/packaging/commandref.rst --- a/Doc/packaging/commandref.rst Sat Feb 25 17:26:44 2012 +0100 +++ b/Doc/packaging/commandref.rst Mon Feb 27 13:21:12 2012 +0100 @@ -137,8 +137,9 @@ of bytecode files, only the options desc :command:`build_scripts` ------------------------ -Build the scripts (just copy them to the build directory and adjust their -shebang if they're Python scripts). +Build scripts (generate the wrapper scripts from the dotted path string): +1. on Posix platform: generate executable scripts without extension; +2. on Windows platform: generate executable .exe files (also support gui mode) :command:`clean` diff -r 1892f4567b07 Doc/packaging/setupscript.rst --- a/Doc/packaging/setupscript.rst Sat Feb 25 17:26:44 2012 +0100 +++ b/Doc/packaging/setupscript.rst Mon Feb 27 13:21:12 2012 +0100 @@ -440,23 +440,44 @@ Installing Scripts ================== So far we have been dealing with pure and non-pure Python modules, which are -usually not run by themselves but imported by scripts. +usually not run by themselves but imported by scripts. In old version of +:file:`Distutils`, there's no easy way to have such kind of script's filename match +local conventions on both Windows and POSIX platforms. What's more, one often +has to create a separate file just for the "main" script, when his actual "main" +is a function in a module somewhere. Scripts are files containing Python source code, intended to be started from the -command line. Scripts don't require Distutils to do anything very complicated. -The only clever feature is that if the first line of the script starts with -``#!`` and contains the word "python", the Distutils will adjust the first line -to refer to the current interpreter location. By default, it is replaced with -the current interpreter location. The :option:`--executable` (or :option:`-e`) -option will allow the interpreter path to be explicitly overridden. +command line. Current :file:`Packaging` module supports a new-style scripts which +are automatically generated from a dotted path string, and at the mean time, these +scripts' filename can match local conventions well - executable scripts with no +extension for POSIX and runnable .exe files for Windows. BTW, if you want another +version of Python to lauch these scripts rather than the default one, the +:option:`--executable` (or :option:`-e`) option will allow the interpreter path to +be explicitly overridden. -The :option:`scripts` option simply is a list of files to be handled in this -way. From the PyXML setup script:: +The :option:`scripts` option simply is a list of callable dotted path which is +pointing to a executed entry. Packaging now support two kinds of input dotted string +, one is very simple such as a.b.c, the other is a dotted string with an option. +With the first kind of dotted string, Packaging can generate executable scripts +with no extension for POSIX platform and .exe file for Windows platform. Because +some scripts may run in a GUI mode, then the second kind of dotted string is offered. +Currently, there only a 'window' or its abbreviation 'win' is supported, which means +a executable script running in GUI mode with .exe extension should be generated. +And please note, the second input will make no sense on POSIX. + +Here is a simple example to show these use cases:: setup(..., - scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val']) + scripts=['hello=a.b.main', 'hellowin=foo.bar.winmain -window']) -All the scripts will also be added to the ``MANIFEST`` file if no template is +If anyting is ok, then on POSIX platform, only a :file:`hello` script file will be +generated, while on Windows, two files - :file:`hello.exe` and :file:`hellowin.exe` will +be generated. We can run these scripts on specific command line tool on different +platforms, and thus can use scripts in a more easier way than old version of +:file:`Distutils`, especially for this example, after clicking the :file:`hellowin.exe` +file, we can see a window popping. + +All the generated scripts will also be added to the ``MANIFEST`` file if no template is provided. See :ref:`packaging-manifest`. .. _packaging-installing-package-data: diff -r 1892f4567b07 Lib/packaging/command/build_scripts.py --- a/Lib/packaging/command/build_scripts.py Sat Feb 25 17:26:44 2012 +0100 +++ b/Lib/packaging/command/build_scripts.py Mon Feb 27 13:21:12 2012 +0100 @@ -1,20 +1,24 @@ -"""Build scripts (copy to build dir and fix up shebang line).""" +"""Build scripts (generate the wrapper scripts from the dotted path string): +1 on Posix platform: generate executable scripts without extension +2 on Windows platform: generate executable .exe files (also support gui mode) +""" import os +import sys import re -import sysconfig -from tokenize import detect_encoding from packaging.command.cmd import Command -from packaging.util import convert_path, newer from packaging import logger -from packaging.compat import Mixin2to3 + +from packaging.errors import PackagingOptionError # check if Python is called on the first line with this expression first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$') -class build_scripts(Command, Mixin2to3): +sys_executable = os.path.normpath(sys.executable) + +class build_scripts(Command): description = "build scripts (copy and fix up shebang line)" @@ -33,122 +37,152 @@ class build_scripts(Command, Mixin2to3): self.force = None self.executable = None self.outfiles = None - self.use_2to3 = False - self.convert_2to3_doctests = None - self.use_2to3_fixers = None def finalize_options(self): self.set_undefined_options('build', ('build_scripts', 'build_dir'), - 'use_2to3', 'use_2to3_fixers', - 'convert_2to3_doctests', 'force', - 'executable') + 'executable', 'force') self.scripts = self.distribution.scripts def get_source_files(self): return self.scripts def run(self): - if not self.scripts: - return - copied_files = self.copy_scripts() - if self.use_2to3 and copied_files: - self._run_2to3(copied_files, fixers=self.use_2to3_fixers) + if self.scripts: + self._check_entries() + self.install_wrapper_scripts() - def copy_scripts(self): - """Copy each script listed in 'self.scripts'; if it's marked as a - Python script in the Unix way (first line matches 'first_line_re', - ie. starts with "\#!" and contains "python"), then adjust the first - line to refer to the current Python interpreter as we copy. + def _check_entries(self): + """Check the offered entries is correct or not: dotted path should always + match an existed multilevel directory path, for instance, if an entry + 'test=foo.bar.script' is offered, then there should be a script.py file + under the '{curdir}/foo/bar' path.""" + if self.scripts: + for entry in self.scripts: + en = entry.split('=') + dotted_string = en[1].strip() + pos = dotted_string.rfind('.') + module_path = dotted_string[:pos] + os_dir_path = module_path.replace('.', os.sep) + '.py' + + if not os.path.exists(os_dir_path): + raise PackagingOptionError("your specific entry '%s' does not exist!" % entry) + + tuples = dotted_string.split() # check if the dotted string has a following flag + if len(tuples) > 1 : + flag = tuples[-1] # currently we support only one flag + if flag not in ['gui']: # and only support the 'gui' flag + raise PackagingOptionError("your specific flag '%s' does not support" % flag) + + def install_wrapper_scripts(self): + """Create wrapper scripts for different entry points which have already + been listed in 'self.wrapper_scripts'. """ - self.mkpath(self.build_dir) - outfiles = [] - for script in self.scripts: - adjust = False - script = convert_path(script) - outfile = os.path.join(self.build_dir, os.path.basename(script)) - outfiles.append(outfile) + if self.scripts: + self.outfiles = [] + for args in self._get_script_args(): + self._write_script(*args) - if not self.force and not newer(script, outfile): - logger.debug("not copying %s (up-to-date)", script) - continue + def _get_script_args(self, executable=sys_executable, wininst=False): + header = get_script_header("", executable, wininst) + for entry in self.scripts: + en = entry.split('=') + script_name = en[0].strip() + dotted_string = en[1].strip() + pos = dotted_string.rfind('.') + from_package = dotted_string[:pos] + tuples = dotted_string.split() + main_func = tuples[0][pos+1:] - # Always open the file, but ignore failures in dry-run mode -- - # that way, we'll get accurate feedback if we can read the - # script. - try: - f = open(script, "rb") - except IOError: - if not self.dry_run: - raise - f = None + script_text = ( + "from %(from_package)s import %(main_func)s\n" + "%(main_func)s()\n") % locals() + + # check if the entry a Window main function or not + window_mode = False + if len(tuples) > 1 : + if tuples[-1] in ['gui']: + window_mode = True + + if sys.platform == 'win32' or wininst: + # should always use the corresponding extension on Windows platform + if window_mode: + ext, launcher = '-script.pyw', 'gui.exe' + new_header = re.sub('(?i)python.exe','pythonw.exe',header) + else: + ext, launcher = '-script.py', 'cli.exe' + new_header = re.sub('(?i)pythonw.exe','python.exe',header) + + if os.path.exists(new_header[2:-1]): + hdr = new_header + else: + hdr = header + + yield (script_name + ext, hdr+script_text, 't') + # reuse cli.exe copied from setuptools + yield (script_name + '.exe', get_resource_string(launcher), 'b') else: - encoding, lines = detect_encoding(f.readline) - f.seek(0) - first_line = f.readline() - if not first_line: - logger.warning('%s: %s is an empty file (skipping)', - self.get_command_name(), script) - continue + # on other platforms, generate script with no extension + yield(script_name, header+script_text) - match = first_line_re.match(first_line) - if match: - adjust = True - post_interp = match.group(1) or b'' + def _write_script(self, script_name, contents, mode="t"): + script_dir = self.build_dir # take build_dir as script installing dir + logger.info("installing %s script to %r", script_name, script_dir) + outfile = os.path.join(script_dir, script_name) + self.outfiles.append(outfile) + if not self.dry_run: + ensure_directory(outfile) + with open(outfile, "w" + mode) as f: + f.write(contents) + f.close() + logger.debug("changing mode of %r to %o", outfile, mode) + try: + os.chmod(outfile, 0o755) + except os.error: + logger.debug("changing mode of %r failed", outfile) - if adjust: - logger.info("copying and adjusting %s -> %s", script, - self.build_dir) - if not self.dry_run: - if not sysconfig.is_python_build(): - executable = self.executable - else: - executable = os.path.join( - sysconfig.get_config_var("BINDIR"), - "python%s%s" % (sysconfig.get_config_var("VERSION"), - sysconfig.get_config_var("EXE"))) - executable = os.fsencode(executable) - shebang = b"#!" + executable + post_interp + b"\n" - # Python parser starts to read a script using UTF-8 until - # it gets a #coding:xxx cookie. The shebang has to be the - # first line of a file, the #coding:xxx cookie cannot be - # written before. So the shebang has to be decodable from - # UTF-8. - try: - shebang.decode('utf-8') - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from utf-8".format(shebang)) - # If the script is encoded to a custom encoding (use a - # #coding:xxx cookie), the shebang has to be decodable from - # the script encoding too. - try: - shebang.decode(encoding) - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from the script encoding ({})" - .format(shebang, encoding)) - with open(outfile, "wb") as outf: - outf.write(shebang) - outf.writelines(f.readlines()) - if f: - f.close() - else: - if f: - f.close() - self.copy_file(script, outfile) - if os.name == 'posix': - for file in outfiles: - if self.dry_run: - logger.info("changing mode of %s", file) - else: - oldmode = os.stat(file).st_mode & 0o7777 - newmode = (oldmode | 0o555) & 0o7777 - if newmode != oldmode: - logger.info("changing mode of %s from %o to %o", - file, oldmode, newmode) - os.chmod(file, newmode) - return outfiles +# The following functions are copied from setuptools + +def get_script_header(script_text, executable=sys_executable, wininst=False): + """Create a #! line, getting options (if any) from script_text""" + first = (script_text.encode()+b'\n').splitlines()[0] + match = first_line_re.match(first) + options = '' + if match: + options = match.group(1).decode() or '' + if options: options = ' '+options + if wininst: + executable = "python.exe" + else: + # executable = nt_quote_arg(executable) + # what's the usage of nt_quote_arg? + pass + hdr = "#!%(executable)s%(options)s\n" % locals() + if options: + if options.strip().startswith('-'): + options = ' -x' + options.strip()[1:] + else: + options = ' -x' + # fix_jython_executable here? + hdr = "#!%(executable)s%(options)s\n" % locals() + return hdr + +def get_resource_string(resource_name): + """Get the resource string from a specific resource.""" + # first get the installed packaging path + import packaging + packaging_path = os.path.dirname(getattr(packaging, '__file__', '')) + # then get the resource path + resource_path = packaging_path + if resource_name: + resource_path = os.path.join(packaging_path, *resource_name.split('/')) + # read the source and return the content + with open(resource_path, 'rb') as f: + return f.read() + +def ensure_directory(path): + """Ensure that the parent directory of 'path' exists.""" + dirname = os.path.dirname(path) + if not os.path.isdir(dirname): + os.makedirs(dirname) diff -r 1892f4567b07 Lib/packaging/dist.py --- a/Lib/packaging/dist.py Sat Feb 25 17:26:44 2012 +0100 +++ b/Lib/packaging/dist.py Mon Feb 27 13:21:12 2012 +0100 @@ -147,6 +147,7 @@ Common commands: (see '--help-commands' self.include_dirs = [] self.extra_path = None self.scripts = [] + self.wrapper_scripts_entries = [] # entries of different wrapper scripts self.data_files = {} self.password = '' self.use_2to3 = False diff -r 1892f4567b07 Lib/packaging/tests/test_command_build_scripts.py --- a/Lib/packaging/tests/test_command_build_scripts.py Sat Feb 25 17:26:44 2012 +0100 +++ b/Lib/packaging/tests/test_command_build_scripts.py Mon Feb 27 13:21:12 2012 +0100 @@ -1,13 +1,16 @@ """Tests for distutils.command.build_scripts.""" import os +import re import sys -import sysconfig +import subprocess from packaging.dist import Distribution from packaging.command.build_scripts import build_scripts from packaging.tests import unittest, support +from packaging.errors import PackagingOptionError + class BuildScriptsTestCase(support.TempdirManager, support.LoggingCatcher, @@ -23,20 +26,74 @@ class BuildScriptsTestCase(support.Tempd self.assertTrue(cmd.force) self.assertEqual(cmd.build_dir, "/foo/bar") - def test_build(self): + def test_install_wrapper_scripts(self): source = self.mkdtemp() target = self.mkdtemp() - expected = self.write_sample_scripts(source) + # create a sample module first + tmp_mains_dir = self.create_sample_module(source, 'foo.bar') - cmd = self.get_build_scripts_cmd(target, - [os.path.join(source, fn) - for fn in expected]) + # first check a wrapper entry which doesn't actually exist + wrapper_entries = ['xxx=foo.bar.xxx'] + cmd = self.get_build_scripts_cmd(target, wrapper_entries) + cmd.finalize_options() + # should report error here: entries does not exist + self.assertRaises(PackagingOptionError, cmd.run) + + # create sample functions under the sample module + mains = self.create_sample_main_functions(tmp_mains_dir) + wrapper_entries = ["script%d=foo.bar.%s.main"%(i+1,main[:-3]) for i,main in enumerate(mains)] + + # should raise error if a not-supported flag offered + wrappers_with_flag = [wrapper_entries[0] + ' abcd'] + cmd = self.get_build_scripts_cmd(target, wrappers_with_flag) + cmd.finalize_options() + self.assertRaises(PackagingOptionError, cmd.run) + + wrapper_entries[1] += ' gui' # another entry is in GUI mode + + # test when correct dotted string offered + cmd = self.get_build_scripts_cmd(target, wrapper_entries) cmd.finalize_options() cmd.run() - built = os.listdir(target) - for name in expected: - self.assertIn(name, built) + # scripts should be created and the shebang line of each script should be correct + wrapper_exes = None + if sys.platform == 'win32': + wrapper_scripts = [os.path.join(target, entry[:7] + '-script.py') for entry in wrapper_entries] + wrapper_scripts[1] += 'w' + wrapper_exes = [os.path.join(target, entry[:7] + '.exe') for entry in wrapper_entries] + else: + # no extension + wrapper_scripts = [os.path.join(target, entry[:7]) for entry in wrapper_entries] + + # we should create a more tolerated regular expression than 'first_line_re' in build_scripts + # in our test, because we may run tests in debug mode + first_line_re = re.compile(b'^#!.*python[0-9.]*.*([ \t].*)?$') + for script in wrapper_scripts: + self.assertTrue(os.path.isfile(script)) + with open(script, "rb") as f: + self.assertIsNotNone(first_line_re.match(f.readline().rstrip())) + + # append the path of our 'source' into sys.path, thus script can import the module normally + self.addCleanup(sys.path.remove, source) + sys.path.append(source) + + # check if all of the wrapper scripts are runnable and can output the correct content + # xxx : how to check the output of generated GUI exe file here? + for script in wrapper_scripts: + pass + # XXX should call the script in a subprocess + # self.assertEqual(stdout.strip(), "Hello World!") + + # check if packaging also generates .exe files on windows and can generate correct output + env = dict(os.environ) + env["PYTHONPATH"] = source + if wrapper_exes: + for exe in wrapper_exes: + self.assertTrue(os.path.exists(exe)) + p = subprocess.Popen(exe, stdout = subprocess.PIPE, env = env) + out = p.communicate()[0].strip() + self.assertEqual(out, b'Hello World!') def get_build_scripts_cmd(self, target, scripts): dist = Distribution() @@ -45,63 +102,34 @@ class BuildScriptsTestCase(support.Tempd build_scripts=target, force=True, executable=sys.executable, - use_2to3=False, - use_2to3_fixers=None, - convert_2to3_doctests=None ) return build_scripts(dist) - def write_sample_scripts(self, dir): - expected = [] - expected.append("script1.py") - self.write_script(dir, "script1.py", - ("#! /usr/bin/env python2.3\n" - "# bogus script w/ Python sh-bang\n" - "pass\n")) - expected.append("script2.py") - self.write_script(dir, "script2.py", - ("#!/usr/bin/python\n" - "# bogus script w/ Python sh-bang\n" - "pass\n")) - expected.append("shell.sh") - self.write_script(dir, "shell.sh", - ("#!/bin/sh\n" - "# bogus shell script w/ sh-bang\n" - "exit 0\n")) - return expected + def create_sample_module(self, source, dotted_path): + dirs = dotted_path.split('.') + os.chdir(source) # should always first in source dir + for dir in dirs: + os.mkdir(dir) + os.chdir(dir) + open('__init__.py', 'w').close() + module_path = os.path.realpath(os.curdir) + os.chdir(source) + return module_path + + def create_sample_main_functions(self, dir): + mains = [] + mains.append("main1.py") + self.write_script(dir, "main1.py", + ("def main():\n print('Hello World!')\n")) + mains.append("main2.py") + self.write_script(dir, "main2.py", + ("def main():\n print('Hello World!')\n")) + return mains def write_script(self, dir, name, text): with open(os.path.join(dir, name), "w") as f: f.write(text) - def test_version_int(self): - source = self.mkdtemp() - target = self.mkdtemp() - expected = self.write_sample_scripts(source) - - - cmd = self.get_build_scripts_cmd(target, - [os.path.join(source, fn) - for fn in expected]) - cmd.finalize_options() - - # http://bugs.python.org/issue4524 - # - # On linux-g++-32 with command line `./configure --enable-ipv6 - # --with-suffix=3`, python is compiled okay but the build scripts - # failed when writing the name of the executable - old = sysconfig.get_config_vars().get('VERSION') - sysconfig._CONFIG_VARS['VERSION'] = 4 - try: - cmd.run() - finally: - if old is not None: - sysconfig._CONFIG_VARS['VERSION'] = old - - built = os.listdir(target) - for name in expected: - self.assertIn(name, built) - def test_suite(): return unittest.makeSuite(BuildScriptsTestCase) diff -r 1892f4567b07 Lib/packaging/tests/test_command_install_dist.py --- a/Lib/packaging/tests/test_command_install_dist.py Sat Feb 25 17:26:44 2012 +0100 +++ b/Lib/packaging/tests/test_command_install_dist.py Mon Feb 27 13:21:12 2012 +0100 @@ -182,10 +182,9 @@ class InstallTestCase(support.TempdirMan # test pre-PEP 376 --record option (outside dist-info dir) install_dir = self.mkdtemp() project_dir, dist = self.create_dist(py_modules=['hello'], - scripts=['sayhi']) + scripts=['sayhi = hello.main']) os.chdir(project_dir) self.write_file('hello.py', "def main(): print('o hai')") - self.write_file('sayhi', 'from hello import main; main()') cmd = install_dist(dist) dist.command_obj['install_dist'] = cmd