bpo-1284670: Allow to restrict ModuleFinder to get "direct" dependencies

MaxwellDupre · Dec 1, 2021 · 797dfef4d70ce5cb838a2fa35faf7d85b45db12b · 797dfef
1 parent d90a125
commit 797dfef4d70ce5cb838a2fa35faf7d85b45db12b
Showing with 201 additions and 8 deletions.

+9 −6 Lib/modulefinder.py

+21 −2 Lib/test/test_modulefinder.py

+171 −0 issue1284670.patch
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
@@ -89,7 +89,7 @@ def _find_module(name, path=None):
 
 class Module:
 
-    def __init__(self, name, file=None, path=None):
+    def __init__(self, name, file=None, path=None, recurse=True):
         self.__name__ = name
         self.__file__ = file
         self.__path__ = path
@@ -113,17 +113,19 @@ def __repr__(self):
 
 class ModuleFinder:
 
-    def __init__(self, path=None, debug=0, excludes=None, replace_paths=None):
+    def __init__(self, path=None, debug=0, excludes=None, replace_paths=None, recurse=1):
         if path is None:
             path = sys.path
         self.path = path
         self.modules = {}
         self.badmodules = {}
         self.debug = debug
+        self.recurse = 0
         self.indent = 0
         self.excludes = excludes if excludes is not None else []
         self.replace_paths = replace_paths if replace_paths is not None else []
         self.processed_paths = []   # Used in debugging only
+        self.recurse = recurse
 
     def msg(self, level, str, *args):
         if level <= self.debug:
@@ -439,10 +441,11 @@ def scan_code(self, co, m):
                 # We don't expect anything else from the generator.
                 raise RuntimeError(what)
 
-        for c in co.co_consts:
-            if isinstance(c, type(co)):
-                self.scan_code(c, m)
-
+        if self.recurse:
+            for c in co.co_consts:
+                if isinstance(c, type(co)):
+                    self.scan_code(c, m)
+
     def load_package(self, fqname, pathname):
         self.msgin(2, "load_package", fqname, pathname)
         newname = replacePackageMap.get(fqname)

diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py
@@ -199,6 +199,21 @@ def foo(): pass
                                 from . import bar
 """]
 
+non_recursive_import_test_1 = [
+    "a",
+    ["a", "b"],
+    [],
+    [],
+    """\
+a.py
+                                import b
+b.py
+                                import c
+c.py
+                                import sys
+"""]
+
+
 relative_import_test_4 = [
     "a.module",
     ["a", "a.module"],
@@ -319,12 +334,13 @@ def create_package(source):
             ofi.close()
 
 class ModuleFinderTest(unittest.TestCase):
-    def _do_test(self, info, report=False, debug=0, replace_paths=[], modulefinder_class=modulefinder.ModuleFinder):
+    def _do_test(self, info, report=False, debug=0, replace_paths=[], modulefinder_class=modulefinder.ModuleFinder,
+                    **kwargs):
         import_this, modules, missing, maybe_missing, source = info
         create_package(source)
         try:
             mf = modulefinder_class(path=TEST_PATH, debug=debug,
-                                           replace_paths=replace_paths)
+                                           replace_paths=replace_paths, **kwargs)
             mf.import_hook(import_this)
             if report:
                 mf.report()
@@ -433,5 +449,8 @@ def load_module(self, fqname, fp, pathname, file_info):
 
         self._do_test(absolute_import_test, modulefinder_class=CheckLoadModuleApi)
 
+    def test_non_recursive_1(self):
+        self._do_test(non_recursive_import_test_1, recurse=False, debug=10)    
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/issue1284670.patch b/issue1284670.patch
@@ -0,0 +1,171 @@
+--- a/Lib/modulefinder.py
++++ b/Lib/modulefinder.py
+@@ -1,6 +1,5 @@
+ """Find modules used by a script, using introspection."""
+
+-from __future__ import generators
+ import dis
+ import imp
+ import marshal
+@@ -9,8 +8,6 @@ import sys
+ import types
+ import struct
+
+-READ_MODE = "rU"
+-
+ # XXX Clean up once str8's cstor matches bytes.
+ LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')])
+ IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')])
+@@ -29,8 +26,7 @@ packagePathMap = {}
+
+ # A Public interface
+ def AddPackagePath(packagename, path):
+-    paths = packagePathMap.get(packagename, [])
+-    paths.append(path)
++    paths = packagePathMap.setdefault(packagename, []).append(path)
+     packagePathMap[packagename] = paths
+
+ replacePackageMap = {}
+@@ -106,14 +102,14 @@ class ModuleFinder:
+
+     def run_script(self, pathname):
+         self.msg(2, "run_script", pathname)
+-        with open(pathname, READ_MODE) as fp:
++        with open(pathname) as fp:
+             stuff = ("", "r", imp.PY_SOURCE)
+             self.load_module('__main__', fp, pathname, stuff)
+
+     def load_file(self, pathname):
+         dir, name = os.path.split(pathname)
+         name, ext = os.path.splitext(name)
+-        with open(pathname, READ_MODE) as fp:
++        with open(pathname) as fp:
+             stuff = (ext, "r", imp.PY_SOURCE)
+             self.load_module(name, fp, pathname, stuff)
+
+@@ -270,7 +266,8 @@ class ModuleFinder:
+         try:
+             m = self.load_module(fqname, fp, pathname, stuff)
+         finally:
+-            if fp: fp.close()
++            if fp:
++                fp.close()
+         if parent:
+             setattr(parent, partname, m)
+         self.msgout(3, "import_module ->", m)
+@@ -662,4 +659,4 @@ if __name__ == '__main__':
+     try:
+         mf = test()
+     except KeyboardInterrupt:
+-        print("\n[interrupt]")
++        print("\n[interrupted]")
+--- a/Lib/test/test_modulefinder.py
++++ b/Lib/test/test_modulefinder.py
+@@ -1,7 +1,7 @@
+-import __future__
+ import os
++import errno
++import shutil
+ import unittest
+-import distutils.dir_util
+ import tempfile
+
+ from test import support
+@@ -9,7 +9,7 @@ from test import support
+ import modulefinder
+
+ TEST_DIR = tempfile.mkdtemp()
+-TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)]
++TEST_PATH = [TEST_DIR, os.path.dirname(tempfile.__file__)]
+
+ # Each test description is a list of 5 items:
+ #
+@@ -196,12 +196,17 @@ a/module.py
+                                 from . import bar
+ """]
+
++
+ def open_file(path):
+-    ##print "#", os.path.abspath(path)
+     dirname = os.path.dirname(path)
+-    distutils.dir_util.mkpath(dirname)
++    try:
++        os.makedirs(dirname)
++    except OSError as e:
++        if e.errno != errno.EEXIST:
++            raise
+     return open(path, "w")
+
++
+ def create_package(source):
+     ofi = None
+     try:
+@@ -216,6 +221,7 @@ def create_package(source):
+         if ofi:
+             ofi.close()
+
++
+ class ModuleFinderTest(unittest.TestCase):
+     def _do_test(self, info, report=False):
+         import_this, modules, missing, maybe_missing, source = info
+@@ -234,19 +240,17 @@ class ModuleFinderTest(unittest.TestCase
+ ##                    import traceback; traceback.print_exc()
+ ##                sys.path = opath
+ ##                return
+-            modules = set(modules)
+-            found = set(mf.modules.keys())
+-            more = list(found - modules)
+-            less = list(modules - found)
++            modules = sorted(set(modules))
++            found = sorted(mf.modules)
+             # check if we found what we expected, not more, not less
+-            self.assertEqual((more, less), ([], []))
++            self.assertEqual(found, modules)
+
+             # check for missing and maybe missing modules
+             bad, maybe = mf.any_missing_maybe()
+             self.assertEqual(bad, missing)
+             self.assertEqual(maybe, maybe_missing)
+         finally:
+-            distutils.dir_util.remove_tree(TEST_DIR)
++            shutil.rmtree(TEST_DIR)
+
+     def test_package(self):
+         self._do_test(package_test)
+@@ -254,25 +258,23 @@ class ModuleFinderTest(unittest.TestCase
+     def test_maybe(self):
+         self._do_test(maybe_test)
+
+-    if getattr(__future__, "absolute_import", None):
++    def test_maybe_new(self):
++        self._do_test(maybe_test_new)
+
+-        def test_maybe_new(self):
+-            self._do_test(maybe_test_new)
+-
+-        def test_absolute_imports(self):
+-            self._do_test(absolute_import_test)
++    def test_absolute_imports(self):
++        self._do_test(absolute_import_test)
+
+-        def test_relative_imports(self):
+-            self._do_test(relative_import_test)
++    def test_relative_imports(self):
++        self._do_test(relative_import_test)
+
+-        def test_relative_imports_2(self):
+-            self._do_test(relative_import_test_2)
++    def test_relative_imports_2(self):
++        self._do_test(relative_import_test_2)
+
+-        def test_relative_imports_3(self):
+-            self._do_test(relative_import_test_3)
++    def test_relative_imports_3(self):
++        self._do_test(relative_import_test_3)
++
+
+ def test_main():
+-    distutils.log.set_threshold(distutils.log.WARN)
+     support.run_unittest(ModuleFinderTest)
+
+ if __name__ == "__main__":