diff -r 802ed5baffb3 Doc/library/importlib.rst --- a/Doc/library/importlib.rst Sat Mar 22 20:39:08 2014 +0100 +++ b/Doc/library/importlib.rst Sat Mar 22 22:15:00 2014 -0400 @@ -1191,3 +1191,32 @@ module will be file-based. .. versionadded:: 3.4 + +.. class:: LazyLoader(loader) + + A class which postpones the execution of the loader of a module until the + module has an attribute accessed. For projects where startup time is + critical, this allows for potentially eliminating the cost of loading a + module if it is never used. For projects where startup time is not + essential then use of this class is **heavily** discouraged due to error + messages created during loading being postponed and thus occurring out of + context. + + This class **only** works with loaders that define + :meth:`importlib.abc.Loader.exec_module` as control over what module type + is used for the module is required. For the same reasons, the loader + **cannot** define :meth:`importlib.abc.Loader.create_module`. + + .. versionadded:: 3.5 + + .. classmethod:: factory(loader) + + A static method which returns a callable that creates a lazy loader. This + is meant to be used in situations where the loader is passed by class + instead of by instance. + :: + + suffixes = importlib.machinery.SOURCE_SUFFIXES + loader = importlib.machinery.SourceFileLoader + lazy_loader = importlib.util.LazyLoader.factory(loader) + finder = importlib.machinery.FileFinder(path, [(lazy_loader, suffixes)]) diff -r 802ed5baffb3 Lib/importlib/util.py --- a/Lib/importlib/util.py Sat Mar 22 20:39:08 2014 +0100 +++ b/Lib/importlib/util.py Sat Mar 22 22:15:00 2014 -0400 @@ -1,5 +1,5 @@ """Utility code for constructing importers, etc.""" - +from . import abc from ._bootstrap import MAGIC_NUMBER from ._bootstrap import cache_from_source from ._bootstrap import decode_source @@ -12,6 +12,7 @@ from contextlib import contextmanager import functools import sys +import types import warnings @@ -200,3 +201,80 @@ return fxn(self, module, *args, **kwargs) return module_for_loader_wrapper + + +class _Module(types.ModuleType): + + """A subclass of the module type to allow __class__ manipulation.""" + + +class _LazyModule(types.ModuleType): + + """A subclass of the module type which triggers loading upon attribute access.""" + + def __getattribute__(self, attr): + """Trigger the load of the module and return the attribute.""" + # Stop triggering this method. + self.__class__ = _Module + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = self.__spec__.loader_state + attrs_now = self.__dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + if key not in attrs_then or attrs_now[key] != attrs_then[key]: + attrs_updated[key] = value + # Using __spec__.loader over __loader__ in order to avoid using a + # mutated __loader__ value. + self.__spec__.loader.exec_module(self) + # Update after loading since that's what would happen in an eager + # loading situation. + self.__dict__.update(attrs_updated) + return getattr(self, attr) + + def __delattr__(self, attr): + """Trigger the load and then perform the deletion.""" + # To trigger the load and raise an if the attribute + # doesn't exist. + self.__getattribute__(attr) + delattr(self, attr) + + +class LazyLoader(abc.Loader): + + """A loader that creates a module which defers loading until attribute access.""" + + @staticmethod + def __check_eager_loader(loader): + if not hasattr(loader, 'exec_module'): + raise TypeError('loader must define exec_module()') + elif hasattr(loader.__class__, 'create_module'): + if abc.Loader.create_module != loader.__class__.create_module: + # Only care if create_module() is overridden in a subclass of + # importlib.abc.Loader. + raise TypeError('loader cannot define create_module()') + + @classmethod + def factory(cls, loader): + """Construct a callable which returns the eager loader made lazy.""" + cls.__check_eager_loader(loader) + return lambda *args, **kwargs: cls(loader(*args, **kwargs)) + + def __init__(self, loader): + self.__check_eager_loader(loader) + self.loader = loader + + def create_module(self, spec): + """Create a module which can have its __class__ manipulated.""" + return _Module(spec.name) + + def exec_module(self, module): + """Make the module load lazily.""" + module.__spec__.loader = self.loader + module.__loader__ = self.loader + # Don't need to worry about deep-copying as trying to set an attribute + # on an object would have triggered the load, + # e.g. ``module.__spec__.loader = None`` would trigger a load from + # trying to access module.__spec__. + module.__spec__.loader_state = module.__dict__.copy() + module.__class__ = _LazyModule diff -r 802ed5baffb3 Lib/test/test_importlib/test_lazy.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_importlib/test_lazy.py Sat Mar 22 22:15:00 2014 -0400 @@ -0,0 +1,123 @@ +import importlib +from importlib import abc +from importlib import util +import unittest + +from . import util as test_util + + +class CollectInit: + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def exec_module(self, module): + return self + + +class LazyLoaderFactoryTests(unittest.TestCase): + + def test_init(self): + factory = util.LazyLoader.factory(CollectInit) + # E.g. what importlib.machinery.FileFinder instantiates loaders with + # plus keyword arguments. + lazy_loader = factory('module name', 'module path', kw='kw') + loader = lazy_loader.loader + self.assertEqual(('module name', 'module path'), loader.args) + self.assertEqual({'kw': 'kw'}, loader.kwargs) + + def test_validation(self): + # No exec_module(), no lazy loading. + with self.assertRaises(TypeError): + util.LazyLoader.factory(object) + + +class TestingImporter(abc.MetaPathFinder, abc.Loader): + + module_name = 'lazy_loader_test' + mutated_name = 'changed' + loaded = None + + def find_spec(self, name, path, target=None): + if name != self.module_name: + return None + return util.spec_from_loader(name, util.LazyLoader(self)) + + def exec_module(self, module): + exec('attr = 42\n__name__ = {!r}'.format(self.mutated_name), + module.__dict__) + self.loaded = module + + +class LazyLoaderTests(unittest.TestCase): + + def test_init(self): + with self.assertRaises(TypeError): + util.LazyLoader(object) + + def new_module(self): + loader = TestingImporter() + spec = util.spec_from_loader(TestingImporter.module_name, + util.LazyLoader(loader)) + module = spec.loader.create_module(spec) + module.__spec__ = spec + module.__loader__ = spec.loader + spec.loader.exec_module(module) + # Module is now lazy. + self.assertIsNone(loader.loaded) + return module + + def test_e2e(self): + # End-to-end test to verify the load is in fact lazy. + importer = TestingImporter() + assert importer.loaded is None + with test_util.uncache(importer.module_name): + with test_util.import_state(meta_path=[importer]): + module = importlib.import_module(importer.module_name) + self.assertIsNone(importer.loaded) + # Trigger load. + self.assertEqual(module.__loader__, importer) + self.assertIsNotNone(importer.loaded) + self.assertEqual(module, importer.loaded) + + def test_attr_unchanged(self): + # An attribute only mutated as a side-effect of import should not be + # changed needlessly. + module = self.new_module() + self.assertEqual(TestingImporter.mutated_name, module.__name__) + + def test_new_attr(self): + # A new attribute should persist. + module = self.new_module() + module.new_attr = 42 + self.assertEqual(42, module.new_attr) + + def test_mutated_preexisting_attr(self): + # Changing an attribute that already existed on the module -- + # e.g. __name__ -- should persist. + module = self.new_module() + module.__name__ = 'bogus' + self.assertEqual('bogus', module.__name__) + + def test_mutated_attr(self): + # Changing an attribute that comes into existence after an import + # should persist. + module = self.new_module() + module.attr = 6 + self.assertEqual(6, module.attr) + + def test_delete_eventual_attr(self): + # Deleting an attribute should stay deleted. + module = self.new_module() + del module.attr + self.assertFalse(hasattr(module, 'attr')) + + def test_delete_preexisting_attr(self): + module = self.new_module() + del module.__name__ + self.assertFalse(hasattr(module, '__name__')) + + +if __name__ == '__main__': + unittest.main()