diff -r 953d8ec1aeb3 Lib/runpy.py --- a/Lib/runpy.py Sat Dec 14 21:07:51 2013 +0200 +++ b/Lib/runpy.py Sun Dec 15 10:58:51 2013 +1000 @@ -14,7 +14,9 @@ import sys import importlib.machinery # importlib first so we can test #15386 via -m import types -from pkgutil import read_code, get_loader, get_importer +from importlib import find_spec +from importlib.util import spec_from_loader +from pkgutil import read_code, get_importer __all__ = [ "run_module", "run_path", @@ -58,51 +60,74 @@ self.value = self._sentinel sys.argv[0] = self._saved_value +# TODO: Replace these helpers with importlib._bootstrap._SpecMethods def _run_code(code, run_globals, init_globals=None, - mod_name=None, mod_fname=None, - mod_loader=None, pkg_name=None): + mod_name=None, mod_spec=None, pkg_name=None): """Helper to run code in nominated namespace""" if init_globals is not None: run_globals.update(init_globals) + if mod_spec is None: + loader = None + fname = None + cached = None + else: + loader = mod_spec.loader + fname = mod_spec.origin + cached = mod_spec.cached + if pkg_name is None: + pkg_name = mod_spec.parent run_globals.update(__name__ = mod_name, - __file__ = mod_fname, - __cached__ = None, + __file__ = fname, + __cached__ = cached, __doc__ = None, - __loader__ = mod_loader, - __package__ = pkg_name) + __loader__ = loader, + __package__ = pkg_name, + __spec__ = mod_spec) exec(code, run_globals) return run_globals def _run_module_code(code, init_globals=None, - mod_name=None, mod_fname=None, - mod_loader=None, pkg_name=None): + mod_name=None, mod_spec=None, pkg_name=None): """Helper to run code in new namespace with sys modified""" - with _TempModule(mod_name) as temp_module, _ModifiedArgv0(mod_fname): + fname = None if mod_spec is None else mod_spec.origin + with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname): mod_globals = temp_module.module.__dict__ _run_code(code, mod_globals, init_globals, - mod_name, mod_fname, mod_loader, pkg_name) + mod_name, mod_spec, pkg_name) # Copy the globals of the temporary module, as they # may be cleared when the temporary module goes away return mod_globals.copy() -# This helper is needed due to a missing component in the PEP 302 -# loader protocol (specifically, "get_filename" is non-standard) -# Since we can't introduce new features in maintenance releases, -# support was added to zipimporter under the name '_get_filename' -def _get_filename(loader, mod_name): - for attr in ("get_filename", "_get_filename"): - meth = getattr(loader, attr, None) - if meth is not None: - return os.path.abspath(meth(mod_name)) - return None +def _fixed_find_spec(mod_name): + # find_spec has the same annoying behaviour as find_loader did (it + # fails to work properly for dotted names), so this is a fixed version + # ala pkgutil.get_loader + if mod_name.startswith('.'): + msg = "Relative module name {!r} not supported".format(mod_name) + raise ImportError(msg) + path = None + pkg_name = mod_name.rpartition(".")[0] + if pkg_name: + pkg = importlib.import_module(pkg_name) + path = getattr(pkg, "__path__", None) + if path is None: + return None + try: + return importlib.find_spec(mod_name, path) + except (ImportError, AttributeError, TypeError, ValueError) as ex: + # This hack fixes an impedance mismatch between pkgutil and + # importlib, where the latter raises other errors for cases where + # pkgutil previously raised ImportError + msg = "Error while finding spec for {!r} ({}: {})" + raise ImportError(msg.format(mod_name, type(ex), ex)) from ex # Helper to get the loader, code and filename for a module def _get_module_details(mod_name): - loader = get_loader(mod_name) - if loader is None: + spec = _fixed_find_spec(mod_name) + if spec is None: raise ImportError("No module named %s" % mod_name) - if loader.is_package(mod_name): + if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): raise ImportError("Cannot use package as __main__ module") try: @@ -111,11 +136,14 @@ except ImportError as e: raise ImportError(("%s; %r is a package and cannot " + "be directly executed") %(e, mod_name)) + loader = spec.loader + if loader is None: + raise ImportError("%r is a namespace package and cannot be executed" + % mod_name) code = loader.get_code(mod_name) if code is None: raise ImportError("No code object available for %s" % mod_name) - filename = _get_filename(loader, mod_name) - return mod_name, loader, code, filename + return mod_name, spec, code # XXX ncoghlan: Should this be documented and made public? # (Current thoughts: don't repeat the mistake that lead to its @@ -137,9 +165,9 @@ """ try: if alter_argv or mod_name != "__main__": # i.e. -m switch - mod_name, loader, code, fname = _get_module_details(mod_name) + mod_name, mod_spec, code = _get_module_details(mod_name) else: # i.e. directory or zipfile execution - mod_name, loader, code, fname = _get_main_module_details() + mod_name, mod_spec, code = _get_main_module_details() except ImportError as exc: # Try to provide a good error message # for directories, zip files and the -m switch @@ -152,12 +180,11 @@ info = "can't find '__main__' module in %r" % sys.argv[0] msg = "%s: %s" % (sys.executable, info) sys.exit(msg) - pkg_name = mod_name.rpartition('.')[0] main_globals = sys.modules["__main__"].__dict__ if alter_argv: - sys.argv[0] = fname + sys.argv[0] = mod_spec.origin return _run_code(code, main_globals, None, - "__main__", fname, loader, pkg_name) + "__main__", mod_spec) def run_module(mod_name, init_globals=None, run_name=None, alter_sys=False): @@ -165,17 +192,14 @@ Returns the resulting top level namespace dictionary """ - mod_name, loader, code, fname = _get_module_details(mod_name) + mod_name, mod_spec, code = _get_module_details(mod_name) if run_name is None: run_name = mod_name - pkg_name = mod_name.rpartition('.')[0] if alter_sys: - return _run_module_code(code, init_globals, run_name, - fname, loader, pkg_name) + return _run_module_code(code, init_globals, run_name, mod_spec) else: # Leave the sys module alone - return _run_code(code, {}, init_globals, run_name, - fname, loader, pkg_name) + return _run_code(code, {}, init_globals, run_name, mod_spec) def _get_main_module_details(): # Helper that gives a nicer error message when attempting to @@ -207,7 +231,7 @@ loader = importlib.machinery.SourceFileLoader(run_name, fname) else: loader = importlib.machinery.SourcelessFileLoader(run_name, fname) - return code, loader + return code, spec_from_loader(run_name, loader) def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location @@ -231,9 +255,8 @@ if isinstance(importer, type(None)) or is_NullImporter: # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code, mod_loader = _get_code_from_file(run_name, path_name) - return _run_module_code(code, init_globals, run_name, path_name, - mod_loader, pkg_name) + code, mod_spec = _get_code_from_file(run_name, path_name) + return _run_module_code(code, init_globals, run_name, mod_spec) else: # Importer is defined for path, so add it to # the start of sys.path @@ -245,12 +268,12 @@ # have no choice and we have to remove it even while we read the # code. If we don't do this, a __loader__ attribute in the # existing __main__ module may prevent location of the new module. - mod_name, loader, code, fname = _get_main_module_details() + mod_name, mod_spec, code = _get_main_module_details() with _TempModule(run_name) as temp_module, \ _ModifiedArgv0(path_name): mod_globals = temp_module.module.__dict__ return _run_code(code, mod_globals, init_globals, - run_name, fname, loader, pkg_name).copy() + run_name, mod_spec, pkg_name).copy() finally: try: sys.path.remove(path_name) diff -r 953d8ec1aeb3 Lib/test/test_cmd_line_script.py --- a/Lib/test/test_cmd_line_script.py Sat Dec 14 21:07:51 2013 +0200 +++ b/Lib/test/test_cmd_line_script.py Sun Dec 15 10:58:51 2013 +1000 @@ -41,11 +41,28 @@ _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__) print('__loader__==%a' % _loader) print('__file__==%a' % __file__) -assertEqual(__cached__, None) +print('__cached__==%a' % __cached__) print('__package__==%r' % __package__) +# Check PEP 451 details +import os.path +if __package__ is not None: + print('__main__ was located through the import system') + assertIdentical(__spec__.loader, __loader__) + expected_spec_name = os.path.splitext(os.path.basename(__file__))[0] + if __package__: + expected_spec_name = __package__ + "." + expected_spec_name + assertEqual(__spec__.name, expected_spec_name) + assertEqual(__spec__.parent, __package__) + assertIdentical(__spec__.submodule_search_locations, None) + assertEqual(__spec__.origin, __file__) + if __spec__.cached is not None: + assertEqual(__spec__.cached, __cached__) # Check the sys module import sys assertIdentical(globals(), sys.modules[__name__].__dict__) +if __spec__ is not None: + # XXX: We're not currently making __main__ available under its real name + pass # assertIdentical(globals(), sys.modules[__spec__.name].__dict__) from test import test_cmd_line_script example_args_list = test_cmd_line_script.example_args assertEqual(sys.argv[1:], example_args_list) diff -r 953d8ec1aeb3 Lib/test/test_runpy.py --- a/Lib/test/test_runpy.py Sat Dec 14 21:07:51 2013 +0200 +++ b/Lib/test/test_runpy.py Sun Dec 15 10:58:51 2013 +1000 @@ -5,7 +5,7 @@ import sys import re import tempfile -import importlib +import importlib, importlib.machinery, importlib.util import py_compile from test.support import ( forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing, @@ -47,7 +47,7 @@ "__cached__": None, "__package__": None, "__doc__": None, -# "__spec__": None, # XXX Uncomment. + "__spec__": None } example_namespace = { "sys": sys, @@ -57,7 +57,7 @@ "run_name_in_sys_modules": False, "module_in_sys_modules": False, "nested": dict(implicit_namespace, - x=1, __name__="", __loader__=None, __spec__=None), + x=1, __name__="", __loader__=None), } example_namespace.update(implicit_namespace) @@ -68,11 +68,19 @@ # testing occurs at those upper layers as well, not just at the utility # layer + # Figuring out the loader details in advance is hard to do, so we skip + # checking loader and loader_state + CHECKED_SPEC_ATTRIBUTES = ["name", "parent", "origin", "cached", + "has_location", "submodule_search_locations"] + def assertNamespaceMatches(self, result_ns, expected_ns): """Check two namespaces match. Ignores any unspecified interpreter created names """ + # Avoid side effects + result_ns = result_ns.copy() + expected_ns = expected_ns.copy() # Impls are permitted to add extra names, so filter them out for k in list(result_ns): if k.startswith("__") and k.endswith("__"): @@ -80,7 +88,22 @@ result_ns.pop(k) if k not in expected_ns["nested"]: result_ns["nested"].pop(k) - # Don't use direct dict comparison - the diffs are too hard to debug + # Spec equality includes the loader, so we take the spec out of the + # result namespace and check that separately + result_spec = result_ns.pop("__spec__") + expected_spec = expected_ns.pop("__spec__") + if expected_spec is None: + self.assertIsNone(result_spec) + else: + self.assertEqual(type(result_spec.loader), + type(expected_spec.loader)) + for attr in self.CHECKED_SPEC_ATTRIBUTES: + k = "__spec__." + attr + actual = (k, getattr(result_spec, attr)) + expected = (k, getattr(expected_spec, attr)) + self.assertEqual(actual, expected) + # For the rest, we still don't use direct dict comparison on the + # namespace, as the diffs are too hard to debug if anything breaks self.assertEqual(set(result_ns), set(expected_ns)) for k in result_ns: actual = (k, result_ns[k]) @@ -131,12 +154,16 @@ mod_fname = "Some other nonsense" mod_loader = "Now you're just being silly" mod_package = '' # Treat as a top level module + mod_spec = importlib.machinery.ModuleSpec(mod_name, + origin=mod_fname, + loader=mod_loader) expected_ns = example_namespace.copy() expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, "__loader__": mod_loader, "__package__": mod_package, + "__spec__": mod_spec, "run_argv0": mod_fname, "run_name_in_sys_modules": True, "module_in_sys_modules": True, @@ -145,9 +172,7 @@ return _run_module_code(example_source, init_globals, mod_name, - mod_fname, - mod_loader, - mod_package) + mod_spec) self.check_code_execution(create_ns, expected_ns) # TODO: Use self.addCleanup to get rid of a lot of try-finally blocks @@ -177,31 +202,43 @@ def test_library_module(self): self.assertEqual(run_module("runpy")["__name__"], "runpy") - def _add_pkg_dir(self, pkg_dir): + def _add_pkg_dir(self, pkg_dir, namespace=False): os.mkdir(pkg_dir) + if namespace: + return None pkg_fname = os.path.join(pkg_dir, "__init__.py") create_empty_file(pkg_fname) return pkg_fname - def _make_pkg(self, source, depth, mod_base="runpy_test"): + def _make_pkg(self, source, depth, mod_base="runpy_test", + *, namespace=False, parent_namespaces=False): + # Enforce a couple of internal sanity checks on test cases + if (namespace or parent_namespaces) and not depth: + raise RuntimeError("Can't mark top level module as a " + "namespace package") pkg_name = "__runpy_pkg__" test_fname = mod_base+os.extsep+"py" pkg_dir = sub_dir = os.path.realpath(tempfile.mkdtemp()) if verbose > 1: print(" Package tree in:", sub_dir) sys.path.insert(0, pkg_dir) if verbose > 1: print(" Updated sys.path:", sys.path[0]) - for i in range(depth): - sub_dir = os.path.join(sub_dir, pkg_name) - pkg_fname = self._add_pkg_dir(sub_dir) - if verbose > 1: print(" Next level in:", sub_dir) - if verbose > 1: print(" Created:", pkg_fname) + if depth: + namespace_flags = [parent_namespaces] * depth + namespace_flags[-1] = namespace + for namespace_flag in namespace_flags: + sub_dir = os.path.join(sub_dir, pkg_name) + pkg_fname = self._add_pkg_dir(sub_dir, namespace_flag) + if verbose > 1: print(" Next level in:", sub_dir) + if verbose > 1: print(" Created:", pkg_fname) mod_fname = os.path.join(sub_dir, test_fname) mod_file = open(mod_fname, "w") mod_file.write(source) mod_file.close() if verbose > 1: print(" Created:", mod_fname) mod_name = (pkg_name+".")*depth + mod_base - return pkg_dir, mod_fname, mod_name + mod_spec = importlib.util.spec_from_file_location(mod_name, + mod_fname) + return pkg_dir, mod_fname, mod_name, mod_spec def _del_pkg(self, top, depth, mod_name): for entry in list(sys.modules): @@ -231,20 +268,29 @@ def _fix_ns_for_legacy_pyc(self, ns, alter_sys): char_to_add = "c" if __debug__ else "o" ns["__file__"] += char_to_add + ns["__cached__"] = ns["__file__"] + spec = ns["__spec__"] + new_spec = importlib.util.spec_from_file_location(spec.name, + ns["__file__"]) + ns["__spec__"] = new_spec if alter_sys: ns["run_argv0"] += char_to_add - def _check_module(self, depth, alter_sys=False): - pkg_dir, mod_fname, mod_name = ( - self._make_pkg(example_source, depth)) + def _check_module(self, depth, alter_sys=False, + *, namespace=False, parent_namespaces=False): + pkg_dir, mod_fname, mod_name, mod_spec = ( + self._make_pkg(example_source, depth, + namespace=namespace, + parent_namespaces=parent_namespaces)) forget(mod_name) expected_ns = example_namespace.copy() expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, + "__cached__": mod_spec.cached, "__package__": mod_name.rpartition(".")[0], -# "__spec__": None, # XXX Needs to be set. + "__spec__": mod_spec, }) if alter_sys: expected_ns.update({ @@ -271,17 +317,21 @@ self._del_pkg(pkg_dir, depth, mod_name) if verbose > 1: print("Module executed successfully") - def _check_package(self, depth, alter_sys=False): - pkg_dir, mod_fname, mod_name = ( - self._make_pkg(example_source, depth, "__main__")) + def _check_package(self, depth, alter_sys=False, + *, namespace=False, parent_namespaces=False): + pkg_dir, mod_fname, mod_name, mod_spec = ( + self._make_pkg(example_source, depth, "__main__", + namespace=namespace, + parent_namespaces=parent_namespaces)) pkg_name = mod_name.rpartition(".")[0] forget(mod_name) expected_ns = example_namespace.copy() expected_ns.update({ "__name__": mod_name, "__file__": mod_fname, + "__cached__": importlib.util.cache_from_source(mod_fname), "__package__": pkg_name, -# "__spec__": None, # XXX Needs to be set. + "__spec__": mod_spec, }) if alter_sys: expected_ns.update({ @@ -337,7 +387,7 @@ from . import sibling from ..uncle.cousin import nephew """ - pkg_dir, mod_fname, mod_name = ( + pkg_dir, mod_fname, mod_name, mod_spec = ( self._make_pkg(contents, depth)) if run_name is None: expected_name = mod_name @@ -376,11 +426,31 @@ if verbose > 1: print("Testing package depth:", depth) self._check_module(depth) + def test_run_module_in_namespace_package(self): + for depth in range(1, 4): + if verbose > 1: print("Testing package depth:", depth) + self._check_module(depth, namespace=True, parent_namespaces=True) + def test_run_package(self): for depth in range(1, 4): if verbose > 1: print("Testing package depth:", depth) self._check_package(depth) + def test_run_package_in_namespace_package(self): + for depth in range(1, 4): + if verbose > 1: print("Testing package depth:", depth) + self._check_package(depth, parent_namespaces=True) + + def test_run_namespace_package(self): + for depth in range(1, 4): + if verbose > 1: print("Testing package depth:", depth) + self._check_package(depth, namespace=True) + + def test_run_namespace_package_in_namespace_package(self): + for depth in range(1, 4): + if verbose > 1: print("Testing package depth:", depth) + self._check_package(depth, namespace=True, parent_namespaces=True) + def test_run_module_alter_sys(self): for depth in range(4): if verbose > 1: print("Testing package depth:", depth) @@ -404,14 +474,16 @@ def test_run_name(self): depth = 1 run_name = "And now for something completely different" - pkg_dir, mod_fname, mod_name = ( + pkg_dir, mod_fname, mod_name, mod_spec = ( self._make_pkg(example_source, depth)) forget(mod_name) expected_ns = example_namespace.copy() expected_ns.update({ "__name__": run_name, "__file__": mod_fname, + "__cached__": importlib.util.cache_from_source(mod_fname), "__package__": mod_name.rpartition(".")[0], + "__spec__": mod_spec, }) def create_ns(init_globals): return run_module(mod_name, init_globals, run_name) @@ -440,7 +512,7 @@ pkg_name = ".".join([base_name] * max_depth) expected_packages.add(pkg_name) expected_modules.add(pkg_name + ".runpy_test") - pkg_dir, mod_fname, mod_name = ( + pkg_dir, mod_fname, mod_name, mod_spec = ( self._make_pkg("", max_depth)) self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name) for depth in range(2, max_depth+1): @@ -464,15 +536,28 @@ return make_script(script_dir, script_basename, source) def _check_script(self, script_name, expected_name, expected_file, - expected_argv0): + expected_argv0, mod_name=None, + precompiled=False): # First check is without run_name def create_ns(init_globals): return run_path(script_name, init_globals) expected_ns = example_namespace.copy() + if mod_name is None: + spec_name = expected_name + else: + spec_name = mod_name + mod_spec = importlib.util.spec_from_file_location(spec_name, + expected_file) + if precompiled: + cached = expected_file + else: + cached = importlib.util.cache_from_source(expected_file) expected_ns.update({ "__name__": expected_name, "__file__": expected_file, + "__cached__": cached, "__package__": "", + "__spec__": mod_spec, "run_argv0": expected_argv0, "run_name_in_sys_modules": True, "module_in_sys_modules": True, @@ -482,6 +567,10 @@ run_name = "prove.issue15230.is.fixed" def create_ns(init_globals): return run_path(script_name, init_globals, run_name) + if mod_name is None: + mod_spec = importlib.util.spec_from_file_location(run_name, + expected_file) + expected_ns["__spec__"] = mod_spec expected_ns["__name__"] = run_name expected_ns["__package__"] = run_name.rpartition(".")[0] self.check_code_execution(create_ns, expected_ns) @@ -504,14 +593,14 @@ compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) self._check_script(compiled_name, "", compiled_name, - compiled_name) + compiled_name, precompiled=True) def test_directory(self): with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) self._check_script(script_dir, "", script_name, - script_dir) + script_dir, mod_name=mod_name) def test_directory_compiled(self): with temp_dir() as script_dir: @@ -522,7 +611,8 @@ if not sys.dont_write_bytecode: legacy_pyc = make_legacy_pyc(script_name) self._check_script(script_dir, "", legacy_pyc, - script_dir) + script_dir, mod_name=mod_name, + precompiled=True) def test_directory_error(self): with temp_dir() as script_dir: @@ -545,7 +635,8 @@ compiled_name = py_compile.compile(script_name, doraise=True) zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name) - self._check_script(zip_name, "", fname, zip_name) + self._check_script(zip_name, "", fname, + zip_name, precompiled=True) def test_zipfile_error(self): with temp_dir() as script_dir: