# HG changeset patch
# Parent  8e1b5bbff4c79742c0d4974339a676569e053ccb
Issue #25533: Add builtins=True flag to iterate built-in modules

diff -r 8e1b5bbff4c7 Doc/library/pkgutil.rst
--- a/Doc/library/pkgutil.rst	Fri May 20 13:37:40 2016 +0200
+++ b/Doc/library/pkgutil.rst	Fri May 20 14:11:15 2016 +0000
@@ -137,10 +137,12 @@
       on the package internal PEP 302 import emulation.
 
 
-.. function:: iter_modules(path=None, prefix='')
+.. function:: iter_modules(path=None, prefix='', *, builtins=False)
 
    Yields ``(module_finder, name, ispkg)`` for all submodules on *path*, or, if
    *path* is ``None``, all top-level modules on ``sys.path``.
+   If *builtins* is true and *path* is ``None``, built-in modules are
+   included.
 
    *path* should be either ``None`` or a list of paths to look for modules in.
 
@@ -150,18 +152,25 @@
 
       Only works for a :term:`finder` which defines an ``iter_modules()``
       method. This interface is non-standard, so the module also provides
-      implementations for :class:`importlib.machinery.FileFinder` and
+      implementations for :class:`importlib.machinery.FileFinder`,
+      :class:`~importlib.machinery.BuiltinImporter` and
       :class:`zipimport.zipimporter`.
 
    .. versionchanged:: 3.3
       Updated to be based directly on :mod:`importlib` rather than relying
       on the package internal PEP 302 import emulation.
 
+   .. versionadded:: 3.6
+      The *builtins* flag.
 
-.. function:: walk_packages(path=None, prefix='', onerror=None)
+
+.. function:: walk_packages(path=None, prefix='', onerror=None, *, \
+      builtins=False)
 
    Yields ``(module_finder, name, ispkg)`` for all modules recursively on
    *path*, or, if *path* is ``None``, all accessible modules.
+   If *builtins* is true and *path* is ``None``, built-in modules are
+   included.
 
    *path* should be either ``None`` or a list of paths to look for modules in.
 
@@ -180,7 +189,7 @@
    Examples::
 
       # list all modules python can access
-      walk_packages()
+      walk_packages(builtins=True)
 
       # list all submodules of ctypes
       walk_packages(ctypes.__path__, ctypes.__name__ + '.')
@@ -189,13 +198,17 @@
 
       Only works for a :term:`finder` which defines an ``iter_modules()``
       method. This interface is non-standard, so the module also provides
-      implementations for :class:`importlib.machinery.FileFinder` and
+      implementations for :class:`importlib.machinery.FileFinder`,
+      :class:`~importlib.machinery.BuiltinImporter` and
       :class:`zipimport.zipimporter`.
 
    .. versionchanged:: 3.3
       Updated to be based directly on :mod:`importlib` rather than relying
       on the package internal PEP 302 import emulation.
 
+   .. versionadded:: 3.6
+      The *builtins* flag.
+
 
 .. function:: get_data(package, resource)
 
diff -r 8e1b5bbff4c7 Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst	Fri May 20 13:37:40 2016 +0200
+++ b/Doc/whatsnew/3.6.rst	Fri May 20 14:11:15 2016 +0000
@@ -292,6 +292,14 @@
 Storchaka in :issue:`24164`.)
 
 
+pkgutil
+-------
+
+The :func:`~pkgutil.iter_modules` and :func:`~pkgutil.walk_packages`
+functions now accept a *builtins* flag which enables yielding
+built-in modules.  (Contributed by Martin Panter in :issue:`25533`.)
+
+
 readline
 --------
 
diff -r 8e1b5bbff4c7 Lib/pkgutil.py
--- a/Lib/pkgutil.py	Fri May 20 13:37:40 2016 +0200
+++ b/Lib/pkgutil.py	Fri May 20 14:11:15 2016 +0000
@@ -1,6 +1,6 @@
 """Utilities to support packages."""
 
-from functools import singledispatch as simplegeneric
+from functools import singledispatch
 import importlib
 import importlib.util
 import importlib.machinery
@@ -44,9 +44,10 @@
     return marshal.load(stream)
 
 
-def walk_packages(path=None, prefix='', onerror=None):
+def walk_packages(path=None, prefix='', onerror=None, *, builtins=False):
     """Yields (module_loader, name, ispkg) for all modules recursively
-    on path, or, if path is None, all accessible modules.
+    on path, or, if path is None, all accessible modules.  If 'builtins' is
+    true and 'path' is None, built-in modules are included.
 
     'path' should be either None or a list of paths to look for
     modules in.
@@ -67,7 +68,7 @@
     Examples:
 
     # list all modules python can access
-    walk_packages()
+    walk_packages(builtins=True)
 
     # list all submodules of ctypes
     walk_packages(ctypes.__path__, ctypes.__name__+'.')
@@ -78,7 +79,8 @@
             return True
         m[p] = True
 
-    for importer, name, ispkg in iter_modules(path, prefix):
+    modules = iter_modules(path, prefix, builtins=builtins)
+    for importer, name, ispkg in modules:
         yield importer, name, ispkg
 
         if ispkg:
@@ -101,9 +103,10 @@
                 yield from walk_packages(path, name+'.', onerror)
 
 
-def iter_modules(path=None, prefix=''):
+def iter_modules(path=None, prefix='', *, builtins=False):
     """Yields (module_loader, name, ispkg) for all submodules on path,
-    or, if path is None, all top-level modules on sys.path.
+    or, if path is None, all top-level modules on sys.path.  If 'builtins' is
+    true and 'path' is None, built-in modules are included.
 
     'path' should be either None or a list of paths to look for
     modules in.
@@ -119,21 +122,27 @@
 
     yielded = {}
     for i in importers:
-        for name, ispkg in iter_importer_modules(i, prefix):
+        modules = iter_importer_modules(i, prefix, builtins=builtins)
+        for name, ispkg in modules:
             if name not in yielded:
                 yielded[name] = 1
                 yield i, name, ispkg
 
 
-@simplegeneric
-def iter_importer_modules(importer, prefix=''):
-    if not hasattr(importer, 'iter_modules'):
-        return []
-    return importer.iter_modules(prefix)
+@singledispatch
+def iter_importer_modules(importer, prefix='', *, builtins=False):
+    if hasattr(importer, 'iter_modules'):
+        return importer.iter_modules(prefix)
+    if importer is importlib.machinery.BuiltinImporter:
+        # importer is not instantiated, but it has no instance methods.  This
+        # case is not handled by @singledispatch.
+        return _iter_builtin_modules(importer, prefix, builtins=builtins)
+    return []
 
 
 # Implement a file walker for the normal importlib path hook
-def _iter_file_finder_modules(importer, prefix=''):
+@iter_importer_modules.register(importlib.machinery.FileFinder)
+def _iter_file_finder_modules(importer, prefix='', *, builtins=False):
     if importer.path is None or not os.path.isdir(importer.path):
         return
 
@@ -173,8 +182,13 @@
             yielded[modname] = 1
             yield prefix + modname, ispkg
 
-iter_importer_modules.register(
-    importlib.machinery.FileFinder, _iter_file_finder_modules)
+
+@iter_importer_modules.register(importlib.machinery.BuiltinImporter)
+def _iter_builtin_modules(importer, prefix='', *, builtins=False):
+    if not builtins:
+        return
+    for modname in sys.builtin_module_names:
+        yield (prefix + modname, False)
 
 
 def _import_imp():
@@ -360,7 +374,8 @@
     import zipimport
     from zipimport import zipimporter
 
-    def iter_zipimport_modules(importer, prefix=''):
+    @iter_importer_modules.register(zipimporter)
+    def iter_zipimport_modules(importer, prefix='', *, builtins=False):
         dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
         _prefix = importer.prefix
         plen = len(_prefix)
@@ -388,8 +403,6 @@
                 yielded[modname] = 1
                 yield prefix + modname, False
 
-    iter_importer_modules.register(zipimporter, iter_zipimport_modules)
-
 except ImportError:
     pass
 
diff -r 8e1b5bbff4c7 Lib/pydoc.py
--- a/Lib/pydoc.py	Fri May 20 13:37:40 2016 +0200
+++ b/Lib/pydoc.py	Fri May 20 14:11:15 2016 +0000
@@ -2060,21 +2060,9 @@
     def run(self, callback, key=None, completer=None, onerror=None):
         if key: key = key.lower()
         self.quit = False
-        seen = {}
 
-        for modname in sys.builtin_module_names:
-            if modname != '__main__':
-                seen[modname] = 1
-                if key is None:
-                    callback(None, modname, '')
-                else:
-                    name = __import__(modname).__doc__ or ''
-                    desc = name.split('\n')[0]
-                    name = modname + ' - ' + desc
-                    if name.lower().find(key) >= 0:
-                        callback(None, modname, desc)
-
-        for importer, modname, ispkg in pkgutil.walk_packages(onerror=onerror):
+        modules = pkgutil.walk_packages(builtins=True, onerror=onerror)
+        for importer, modname, ispkg in modules:
             if self.quit:
                 break
 
@@ -2087,6 +2075,7 @@
                     # raised by tests for bad coding cookies or BOM
                     continue
                 loader = spec.loader
+                source = None
                 if hasattr(loader, 'get_source'):
                     try:
                         source = loader.get_source(modname)
@@ -2094,12 +2083,13 @@
                         if onerror:
                             onerror(modname)
                         continue
+                if source is not None:
                     desc = source_synopsis(io.StringIO(source)) or ''
                     if hasattr(loader, 'get_filename'):
                         path = loader.get_filename(modname)
                     else:
                         path = None
-                else:
+                else:  # No get_source(), or source is None (built-in module)
                     try:
                         module = importlib._bootstrap._load(spec)
                     except ImportError:
diff -r 8e1b5bbff4c7 Lib/test/test_pkgutil.py
--- a/Lib/test/test_pkgutil.py	Fri May 20 13:37:40 2016 +0200
+++ b/Lib/test/test_pkgutil.py	Fri May 20 14:11:15 2016 +0000
@@ -2,6 +2,7 @@
 import unittest
 import sys
 import importlib
+from importlib.machinery import BuiltinImporter
 from importlib.util import spec_from_file_location
 import pkgutil
 import os
@@ -101,6 +102,26 @@
         for t in pkgutil.walk_packages(path=[self.dirname]):
             self.fail("unexpected package found")
 
+    def test_iter_builtins(self):
+        self.check_builtin_iter(pkgutil.iter_modules)
+
+    def test_walk_builtins(self):
+        self.check_builtin_iter(pkgutil.walk_packages)
+
+    def check_builtin_iter(self, func):
+        # Stop iterating early to avoid scanning arbitrary sys.path entries
+        for [finder, name, ispkg] in func(builtins=True):
+            if name == "sys":
+                break
+        else:
+            self.fail("sys module missing")
+        if isinstance(finder, type):
+            # BuiltinImporter only has class methods; uninstantiated is okay
+            self.assertTrue(issubclass(finder, BuiltinImporter))
+        else:
+            self.assertIsInstance(finder, BuiltinImporter)
+        self.assertFalse(ispkg)
+
 class PkgutilPEP302Tests(unittest.TestCase):
 
     class MyTestLoader(object):
diff -r 8e1b5bbff4c7 Lib/test/test_pydoc.py
--- a/Lib/test/test_pydoc.py	Fri May 20 13:37:40 2016 +0200
+++ b/Lib/test/test_pydoc.py	Fri May 20 14:11:15 2016 +0000
@@ -382,8 +382,8 @@
         a given path.
         """
         default_path = path or [os.path.dirname(__file__)]
-        def wrapper(path=None, prefix='', onerror=None):
-            return walk_packages(path or default_path, prefix, onerror)
+        def wrapper(path=None, *pos, **kw):
+            return walk_packages(path or default_path, *pos, **kw)
         return wrapper
 
     @contextlib.contextmanager
diff -r 8e1b5bbff4c7 Misc/NEWS
--- a/Misc/NEWS	Fri May 20 13:37:40 2016 +0200
+++ b/Misc/NEWS	Fri May 20 14:11:15 2016 +0000
@@ -16,6 +16,10 @@
 Library
 -------
 
+- Issue #25533: Added "builtins" flag for pkgutil.iter_modules() and
+  walk_packages() to include built-in modules.  Simplified pydoc to use the
+  new functionality.
+
 - Issue #26741: subprocess.Popen destructor now emits a ResourceWarning warning
   if the child process is still running.