Modulefinder now handles absolute and relative imports, including

tests.

Will backport to release25-maint.

Lib/modulefinder.py

@@ -1,13 +1,14 @@
 """Find modules used by a script, using introspection."""
-
 # This module should be kept compatible with Python 2.2, see PEP 291.
 
+from __future__ import generators
 import dis
 import imp
 import marshal
 import os
 import sys
 import new
+import struct
 
 if hasattr(sys.__stdout__, "newlines"):
     READ_MODE = "U"  # universal line endings
@@ -15,11 +16,12 @@ else:
     # remain compatible with Python  < 2.3
     READ_MODE = "r"
 
-LOAD_CONST = dis.opname.index('LOAD_CONST')
+LOAD_CONST = chr(dis.opname.index('LOAD_CONST'))
-IMPORT_NAME = dis.opname.index('IMPORT_NAME')
+IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME'))
-STORE_NAME = dis.opname.index('STORE_NAME')
+STORE_NAME = chr(dis.opname.index('STORE_NAME'))
-STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')
+STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL'))
 STORE_OPS = [STORE_NAME, STORE_GLOBAL]
+HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT)
 
 # Modulefinder does a good job at simulating Python's, but it can not
 # handle __path__ modifications packages make at runtime.  Therefore there
@@ -118,9 +120,9 @@ class ModuleFinder:
         stuff = (ext, "r", imp.PY_SOURCE)
         self.load_module(name, fp, pathname, stuff)
 
-    def import_hook(self, name, caller=None, fromlist=None):
+    def import_hook(self, name, caller=None, fromlist=None, level=-1):
-        self.msg(3, "import_hook", name, caller, fromlist)
+        self.msg(3, "import_hook", name, caller, fromlist, level)
-        parent = self.determine_parent(caller)
+        parent = self.determine_parent(caller, level=level)
         q, tail = self.find_head_package(parent, name)
         m = self.load_tail(q, tail)
         if not fromlist:
@@ -129,12 +131,26 @@ class ModuleFinder:
             self.ensure_fromlist(m, fromlist)
         return None
 
-    def determine_parent(self, caller):
+    def determine_parent(self, caller, level=-1):
-        self.msgin(4, "determine_parent", caller)
+        self.msgin(4, "determine_parent", caller, level)
-        if not caller:
+        if not caller or level == 0:
             self.msgout(4, "determine_parent -> None")
             return None
         pname = caller.__name__
+        if level >= 1: # relative import
+            if caller.__path__:
+                level -= 1
+            if level == 0:
+                parent = self.modules[pname]
+                assert parent is caller
+                self.msgout(4, "determine_parent ->", parent)
+                return parent
+            if pname.count(".") < level:
+                raise ImportError, "relative importpath too deep"
+            pname = ".".join(pname.split(".")[:-level])
+            parent = self.modules[pname]
+            self.msgout(4, "determine_parent ->", parent)
+            return parent
         if caller.__path__:
             parent = self.modules[pname]
             assert caller is parent
@@ -294,13 +310,13 @@ class ModuleFinder:
             self.badmodules[name] = {}
         self.badmodules[name][caller.__name__] = 1
 
-    def _safe_import_hook(self, name, caller, fromlist):
+    def _safe_import_hook(self, name, caller, fromlist, level=-1):
         # wrapper for self.import_hook() that won't raise ImportError
         if name in self.badmodules:
             self._add_badmodule(name, caller)
             return
         try:
-            self.import_hook(name, caller)
+            self.import_hook(name, caller, level=level)
         except ImportError, msg:
             self.msg(2, "ImportError:", str(msg))
             self._add_badmodule(name, caller)
@@ -311,38 +327,87 @@ class ModuleFinder:
                         self._add_badmodule(sub, caller)
                         continue
                     try:
-                        self.import_hook(name, caller, [sub])
+                        self.import_hook(name, caller, [sub], level=level)
                     except ImportError, msg:
                         self.msg(2, "ImportError:", str(msg))
                         fullname = name + "." + sub
                         self._add_badmodule(fullname, caller)
 
+    def scan_opcodes(self, co,
+                     unpack = struct.unpack):
+        # Scan the code, and yield 'interesting' opcode combinations
+        # Version for Python 2.4 and older
+        code = co.co_code
+        names = co.co_names
+        consts = co.co_consts
+        while code:
+            c = code[0]
+            if c in STORE_OPS:
+                oparg, = unpack('<H', code[1:3])
+                yield "store", (names[oparg],)
+                code = code[3:]
+                continue
+            if c == LOAD_CONST and code[3] == IMPORT_NAME:
+                oparg_1, oparg_2 = unpack('<xHxH', code[:6])
+                yield "import", (consts[oparg_1], names[oparg_2])
+                code = code[6:]
+                continue
+            if c >= HAVE_ARGUMENT:
+                code = code[3:]
+            else:
+                code = code[1:]
+
+    def scan_opcodes_25(self, co,
+                     unpack = struct.unpack):
+        # Scan the code, and yield 'interesting' opcode combinations
+        # Python 2.5 version (has absolute and relative imports)
+        code = co.co_code
+        names = co.co_names
+        consts = co.co_consts
+        LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME
+        while code:
+            c = code[0]
+            if c in STORE_OPS:
+                oparg, = unpack('<H', code[1:3])
+                yield "store", (names[oparg],)
+                code = code[3:]
+                continue
+            if code[:9:3] == LOAD_LOAD_AND_IMPORT:
+                oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9])
+                level = consts[oparg_1]
+                if level == -1: # normal import
+                    yield "import", (consts[oparg_2], names[oparg_3])
+                elif level == 0: # absolute import
+                    yield "absolute_import", (consts[oparg_2], names[oparg_3])
+                else: # relative import
+                    yield "relative_import", (level, consts[oparg_2], names[oparg_3])
+                code = code[9:]
+                continue
+            if c >= HAVE_ARGUMENT:
+                code = code[3:]
+            else:
+                code = code[1:]
+
     def scan_code(self, co, m):
         code = co.co_code
-        n = len(code)
+        if sys.version_info >= (2, 5):
-        i = 0
+            scanner = self.scan_opcodes_25
-        fromlist = None
+        else:
-        while i < n:
+            scanner = self.scan_opcodes
-            c = code[i]
+        for what, args in scanner(co):
-            i = i+1
+            if what == "store":
-            op = ord(c)
+                name, = args
-            if op >= dis.HAVE_ARGUMENT:
+                m.globalnames[name] = 1
-                oparg = ord(code[i]) + ord(code[i+1])*256
+            elif what in ("import", "absolute_import"):
-                i = i+2
+                fromlist, name = args
-            if op == LOAD_CONST:
-                # An IMPORT_NAME is always preceded by a LOAD_CONST, it's
-                # a tuple of "from" names, or None for a regular import.
-                # The tuple may contain "*" for "from <mod> import *"
-                fromlist = co.co_consts[oparg]
-            elif op == IMPORT_NAME:
-                assert fromlist is None or type(fromlist) is tuple
-                name = co.co_names[oparg]
                 have_star = 0
                 if fromlist is not None:
                     if "*" in fromlist:
                         have_star = 1
                     fromlist = [f for f in fromlist if f != "*"]
-                self._safe_import_hook(name, m, fromlist)
+                if what == "absolute_import": level = 0
+                else: level = -1
+                self._safe_import_hook(name, m, fromlist, level=level)
                 if have_star:
                     # We've encountered an "import *". If it is a Python module,
                     # the code has already been parsed and we can suck out the
@@ -362,10 +427,17 @@ class ModuleFinder:
                             m.starimports[name] = 1
                     else:
                         m.starimports[name] = 1
-            elif op in STORE_OPS:
+            elif what == "relative_import":
-                # keep track of all global names that are assigned to
+                level, fromlist, name = args
-                name = co.co_names[oparg]
+                if name:
-                m.globalnames[name] = 1
+                    self._safe_import_hook(name, m, fromlist, level=level)
+                else:
+                    parent = self.determine_parent(m, level=level)
+                    self._safe_import_hook(parent.__name__, None, fromlist, level=0)
+            else:
+                # We don't expect anything else from the generator.
+                raise RuntimeError(what)
+
         for c in co.co_consts:
             if isinstance(c, type(co)):
                 self.scan_code(c, m)

Lib/test/test_modulefinder.py

@@ -0,0 +1,263 @@
+import __future__
+import sys, os
+import unittest
+import distutils.dir_util
+import tempfile
+
+from test import test_support
+
+try: set
+except NameError: from sets import Set as set
+
+import modulefinder
+
+# Note: To test modulefinder with Python 2.2, sets.py and
+# modulefinder.py must be available - they are not in the standard
+# library.
+
+TEST_DIR = tempfile.mkdtemp()
+TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)]
+
+# Each test description is a list of 5 items:
+#
+# 1. a module name that will be imported by modulefinder
+# 2. a list of module names that modulefinder is required to find
+# 3. a list of module names that modulefinder should complain
+#    about because they are not found
+# 4. a list of module names that modulefinder should complain
+#    about because they MAY be not found
+# 5. a string specifying packages to create; the format is obvious imo.
+#
+# Each package will be created in TEST_DIR, and TEST_DIR will be
+# removed after the tests again.
+# Modulefinder searches in a path that contains TEST_DIR, plus
+# the standard Lib directory.
+
+maybe_test = [
+    "a.module",
+    ["a", "a.module", "sys",
+     "b"],
+    ["c"], ["b.something"],
+    """\
+a/__init__.py
+a/module.py
+                                from b import something
+                                from c import something
+b/__init__.py
+                                from sys import *
+"""]
+
+maybe_test_new = [
+    "a.module",
+    ["a", "a.module", "sys",
+     "b", "__future__"],
+    ["c"], ["b.something"],
+    """\
+a/__init__.py
+a/module.py
+                                from b import something
+                                from c import something
+b/__init__.py
+                                from __future__ import absolute_import
+                                from sys import *
+"""]
+
+package_test = [
+    "a.module",
+    ["a", "a.b", "a.c", "a.module", "mymodule", "sys"],
+    ["blahblah"], [],
+    """\
+mymodule.py
+a/__init__.py
+                                import blahblah
+                                from a import b
+                                import c
+a/module.py
+                                import sys
+                                from a import b as x
+                                from a.c import sillyname
+a/b.py
+a/c.py
+                                from a.module import x
+                                import mymodule as sillyname
+                                from sys import version_info
+"""]
+
+absolute_import_test = [
+    "a.module",
+    ["a", "a.module",
+     "b", "b.x", "b.y", "b.z",
+     "__future__", "sys", "exceptions"],
+    ["blahblah"], [],
+    """\
+mymodule.py
+a/__init__.py
+a/module.py
+                                from __future__ import absolute_import
+                                import sys # sys
+                                import blahblah # fails
+                                import exceptions # exceptions
+                                import b.x # b.x
+                                from b import y # b.y
+                                from b.z import * # b.z.*
+a/exceptions.py
+a/sys.py
+                                import mymodule
+a/b/__init__.py
+a/b/x.py
+a/b/y.py
+a/b/z.py
+b/__init__.py
+                                import z
+b/unused.py
+b/x.py
+b/y.py
+b/z.py
+"""]
+
+relative_import_test = [
+    "a.module",
+    ["__future__",
+     "a", "a.module",
+     "a.b", "a.b.y", "a.b.z",
+     "a.b.c", "a.b.c.moduleC",
+     "a.b.c.d", "a.b.c.e",
+     "a.b.x",
+     "exceptions"],
+    [], [],
+    """\
+mymodule.py
+a/__init__.py
+                                from .b import y, z # a.b.y, a.b.z
+a/module.py
+                                from __future__ import absolute_import # __future__
+                                import exceptions # exceptions
+a/exceptions.py
+a/sys.py
+a/b/__init__.py
+                                from ..b import x # a.b.x
+                                #from a.b.c import moduleC
+                                from .c import moduleC # a.b.moduleC
+a/b/x.py
+a/b/y.py
+a/b/z.py
+a/b/g.py
+a/b/c/__init__.py
+                                from ..c import e # a.b.c.e
+a/b/c/moduleC.py
+                                from ..c import d # a.b.c.d
+a/b/c/d.py
+a/b/c/e.py
+a/b/c/x.py
+"""]
+
+relative_import_test_2 = [
+    "a.module",
+    ["a", "a.module",
+     "a.sys",
+     "a.b", "a.b.y", "a.b.z",
+     "a.b.c", "a.b.c.d",
+     "a.b.c.e",
+     "a.b.c.moduleC",
+     "a.b.c.f",
+     "a.b.x",
+     "a.another"],
+    [], [],
+    """\
+mymodule.py
+a/__init__.py
+                                from . import sys # a.sys
+a/another.py
+a/module.py
+                                from .b import y, z # a.b.y, a.b.z
+a/exceptions.py
+a/sys.py
+a/b/__init__.py
+                                from .c import moduleC # a.b.c.moduleC
+                                from .c import d # a.b.c.d
+a/b/x.py
+a/b/y.py
+a/b/z.py
+a/b/c/__init__.py
+                                from . import e # a.b.c.e
+a/b/c/moduleC.py
+                                #
+                                from . import f   # a.b.c.f
+                                from .. import x  # a.b.x
+                                from ... import another # a.another
+a/b/c/d.py
+a/b/c/e.py
+a/b/c/f.py
+"""]
+
+def open_file(path):
+    ##print "#", os.path.abspath(path)
+    dirname = os.path.dirname(path)
+    distutils.dir_util.mkpath(dirname)
+    return open(path, "w")
+
+def create_package(source):
+    ofi = None
+    for line in source.splitlines():
+        if line.startswith(" ") or line.startswith("\t"):
+            ofi.write(line.strip() + "\n")
+        else:
+            ofi = open_file(os.path.join(TEST_DIR, line.strip()))
+
+class ModuleFinderTest(unittest.TestCase):
+    def _do_test(self, info, report=False):
+        import_this, modules, missing, maybe_missing, source = info
+        create_package(source)
+        try:
+            mf = modulefinder.ModuleFinder(path=TEST_PATH)
+            mf.import_hook(import_this)
+            if report:
+                mf.report()
+##                # This wouldn't work in general when executed several times:
+##                opath = sys.path[:]
+##                sys.path = TEST_PATH
+##                try:
+##                    __import__(import_this)
+##                except:
+##                    import traceback; traceback.print_exc()
+##                sys.path = opath
+##                return
+            modules = set(modules)
+            found = set(mf.modules.keys())
+            more = list(found - modules)
+            less = list(modules - found)
+            # check if we found what we expected, not more, not less
+            self.failUnlessEqual((more, less), ([], []))
+
+            # check for missing and maybe missing modules
+            bad, maybe = mf.any_missing_maybe()
+            self.failUnlessEqual(bad, missing)
+            self.failUnlessEqual(maybe, maybe_missing)
+        finally:
+            distutils.dir_util.remove_tree(TEST_DIR)
+
+    def test_package(self):
+        self._do_test(package_test)
+
+    def test_maybe(self):
+        self._do_test(maybe_test)
+
+    if getattr(__future__, "absolute_import", None):
+
+        def test_maybe_new(self):
+            self._do_test(maybe_test_new)
+
+        def test_absolute_imports(self):
+            self._do_test(absolute_import_test)
+
+        def test_relative_imports(self):
+            self._do_test(relative_import_test)
+
+        def test_relative_imports_2(self):
+            self._do_test(relative_import_test_2)
+
+def test_main():
+    test_support.run_unittest(ModuleFinderTest)
+
+if __name__ == "__main__":
+    unittest.main()

Misc/NEWS

@@ -87,6 +87,9 @@ Core and builtins
 Library
 -------
 
+- Lib/modulefinder.py now handles absolute and relative imports
+  correctly.
+
 - Patch #1567274: Support SMTP over TLS.
 
 - Patch #1560695: Add .note.GNU-stack to ctypes' sysv.S so that
@@ -199,6 +202,8 @@ Extension Modules
 Tests
 -----
 
+- Added some tests for modulefinder.
+
 - Converted test_imp to use unittest.
 
 - Fix bsddb test_basics.test06_Transactions to check the version