Modulefinder now handles absolute and relative imports, including
tests. Will backport to release25-maint.
This commit is contained in:
Thomas Heller
parent
df08f0b9a0
commit
112d1a64ac
|
|
@ -1,13 +1,14 @@
|
|||
"""Find modules used by a script, using introspection."""
|
||||
|
||||
# This module should be kept compatible with Python 2.2, see PEP 291.
|
||||
|
||||
from __future__ import generators
|
||||
import dis
|
||||
import imp
|
||||
import marshal
|
||||
import os
|
||||
import sys
|
||||
import new
|
||||
import struct
|
||||
|
||||
if hasattr(sys.__stdout__, "newlines"):
|
||||
READ_MODE = "U" # universal line endings
|
||||
|
|
@ -15,11 +16,12 @@ else:
|
|||
# remain compatible with Python < 2.3
|
||||
READ_MODE = "r"
|
||||
|
||||
LOAD_CONST = dis.opname.index('LOAD_CONST')
|
||||
IMPORT_NAME = dis.opname.index('IMPORT_NAME')
|
||||
STORE_NAME = dis.opname.index('STORE_NAME')
|
||||
STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')
|
||||
LOAD_CONST = chr(dis.opname.index('LOAD_CONST'))
|
||||
IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME'))
|
||||
STORE_NAME = chr(dis.opname.index('STORE_NAME'))
|
||||
STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL'))
|
||||
STORE_OPS = [STORE_NAME, STORE_GLOBAL]
|
||||
HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT)
|
||||
|
||||
# Modulefinder does a good job at simulating Python's, but it can not
|
||||
# handle __path__ modifications packages make at runtime. Therefore there
|
||||
|
|
@ -118,9 +120,9 @@ class ModuleFinder:
|
|||
stuff = (ext, "r", imp.PY_SOURCE)
|
||||
self.load_module(name, fp, pathname, stuff)
|
||||
|
||||
def import_hook(self, name, caller=None, fromlist=None):
|
||||
self.msg(3, "import_hook", name, caller, fromlist)
|
||||
parent = self.determine_parent(caller)
|
||||
def import_hook(self, name, caller=None, fromlist=None, level=-1):
|
||||
self.msg(3, "import_hook", name, caller, fromlist, level)
|
||||
parent = self.determine_parent(caller, level=level)
|
||||
q, tail = self.find_head_package(parent, name)
|
||||
m = self.load_tail(q, tail)
|
||||
if not fromlist:
|
||||
|
|
@ -129,12 +131,26 @@ class ModuleFinder:
|
|||
self.ensure_fromlist(m, fromlist)
|
||||
return None
|
||||
|
||||
def determine_parent(self, caller):
|
||||
self.msgin(4, "determine_parent", caller)
|
||||
if not caller:
|
||||
def determine_parent(self, caller, level=-1):
|
||||
self.msgin(4, "determine_parent", caller, level)
|
||||
if not caller or level == 0:
|
||||
self.msgout(4, "determine_parent -> None")
|
||||
return None
|
||||
pname = caller.__name__
|
||||
if level >= 1: # relative import
|
||||
if caller.__path__:
|
||||
level -= 1
|
||||
if level == 0:
|
||||
parent = self.modules[pname]
|
||||
assert parent is caller
|
||||
self.msgout(4, "determine_parent ->", parent)
|
||||
return parent
|
||||
if pname.count(".") < level:
|
||||
raise ImportError, "relative importpath too deep"
|
||||
pname = ".".join(pname.split(".")[:-level])
|
||||
parent = self.modules[pname]
|
||||
self.msgout(4, "determine_parent ->", parent)
|
||||
return parent
|
||||
if caller.__path__:
|
||||
parent = self.modules[pname]
|
||||
assert caller is parent
|
||||
|
|
@ -294,13 +310,13 @@ class ModuleFinder:
|
|||
self.badmodules[name] = {}
|
||||
self.badmodules[name][caller.__name__] = 1
|
||||
|
||||
def _safe_import_hook(self, name, caller, fromlist):
|
||||
def _safe_import_hook(self, name, caller, fromlist, level=-1):
|
||||
# wrapper for self.import_hook() that won't raise ImportError
|
||||
if name in self.badmodules:
|
||||
self._add_badmodule(name, caller)
|
||||
return
|
||||
try:
|
||||
self.import_hook(name, caller)
|
||||
self.import_hook(name, caller, level=level)
|
||||
except ImportError, msg:
|
||||
self.msg(2, "ImportError:", str(msg))
|
||||
self._add_badmodule(name, caller)
|
||||
|
|
@ -311,38 +327,87 @@ class ModuleFinder:
|
|||
self._add_badmodule(sub, caller)
|
||||
continue
|
||||
try:
|
||||
self.import_hook(name, caller, [sub])
|
||||
self.import_hook(name, caller, [sub], level=level)
|
||||
except ImportError, msg:
|
||||
self.msg(2, "ImportError:", str(msg))
|
||||
fullname = name + "." + sub
|
||||
self._add_badmodule(fullname, caller)
|
||||
|
||||
def scan_opcodes(self, co,
|
||||
unpack = struct.unpack):
|
||||
# Scan the code, and yield 'interesting' opcode combinations
|
||||
# Version for Python 2.4 and older
|
||||
code = co.co_code
|
||||
names = co.co_names
|
||||
consts = co.co_consts
|
||||
while code:
|
||||
c = code[0]
|
||||
if c in STORE_OPS:
|
||||
oparg, = unpack('<H', code[1:3])
|
||||
yield "store", (names[oparg],)
|
||||
code = code[3:]
|
||||
continue
|
||||
if c == LOAD_CONST and code[3] == IMPORT_NAME:
|
||||
oparg_1, oparg_2 = unpack('<xHxH', code[:6])
|
||||
yield "import", (consts[oparg_1], names[oparg_2])
|
||||
code = code[6:]
|
||||
continue
|
||||
if c >= HAVE_ARGUMENT:
|
||||
code = code[3:]
|
||||
else:
|
||||
code = code[1:]
|
||||
|
||||
def scan_opcodes_25(self, co,
|
||||
unpack = struct.unpack):
|
||||
# Scan the code, and yield 'interesting' opcode combinations
|
||||
# Python 2.5 version (has absolute and relative imports)
|
||||
code = co.co_code
|
||||
names = co.co_names
|
||||
consts = co.co_consts
|
||||
LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME
|
||||
while code:
|
||||
c = code[0]
|
||||
if c in STORE_OPS:
|
||||
oparg, = unpack('<H', code[1:3])
|
||||
yield "store", (names[oparg],)
|
||||
code = code[3:]
|
||||
continue
|
||||
if code[:9:3] == LOAD_LOAD_AND_IMPORT:
|
||||
oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9])
|
||||
level = consts[oparg_1]
|
||||
if level == -1: # normal import
|
||||
yield "import", (consts[oparg_2], names[oparg_3])
|
||||
elif level == 0: # absolute import
|
||||
yield "absolute_import", (consts[oparg_2], names[oparg_3])
|
||||
else: # relative import
|
||||
yield "relative_import", (level, consts[oparg_2], names[oparg_3])
|
||||
code = code[9:]
|
||||
continue
|
||||
if c >= HAVE_ARGUMENT:
|
||||
code = code[3:]
|
||||
else:
|
||||
code = code[1:]
|
||||
|
||||
def scan_code(self, co, m):
|
||||
code = co.co_code
|
||||
n = len(code)
|
||||
i = 0
|
||||
fromlist = None
|
||||
while i < n:
|
||||
c = code[i]
|
||||
i = i+1
|
||||
op = ord(c)
|
||||
if op >= dis.HAVE_ARGUMENT:
|
||||
oparg = ord(code[i]) + ord(code[i+1])*256
|
||||
i = i+2
|
||||
if op == LOAD_CONST:
|
||||
# An IMPORT_NAME is always preceded by a LOAD_CONST, it's
|
||||
# a tuple of "from" names, or None for a regular import.
|
||||
# The tuple may contain "*" for "from <mod> import *"
|
||||
fromlist = co.co_consts[oparg]
|
||||
elif op == IMPORT_NAME:
|
||||
assert fromlist is None or type(fromlist) is tuple
|
||||
name = co.co_names[oparg]
|
||||
if sys.version_info >= (2, 5):
|
||||
scanner = self.scan_opcodes_25
|
||||
else:
|
||||
scanner = self.scan_opcodes
|
||||
for what, args in scanner(co):
|
||||
if what == "store":
|
||||
name, = args
|
||||
m.globalnames[name] = 1
|
||||
elif what in ("import", "absolute_import"):
|
||||
fromlist, name = args
|
||||
have_star = 0
|
||||
if fromlist is not None:
|
||||
if "*" in fromlist:
|
||||
have_star = 1
|
||||
fromlist = [f for f in fromlist if f != "*"]
|
||||
self._safe_import_hook(name, m, fromlist)
|
||||
if what == "absolute_import": level = 0
|
||||
else: level = -1
|
||||
self._safe_import_hook(name, m, fromlist, level=level)
|
||||
if have_star:
|
||||
# We've encountered an "import *". If it is a Python module,
|
||||
# the code has already been parsed and we can suck out the
|
||||
|
|
@ -362,10 +427,17 @@ class ModuleFinder:
|
|||
m.starimports[name] = 1
|
||||
else:
|
||||
m.starimports[name] = 1
|
||||
elif op in STORE_OPS:
|
||||
# keep track of all global names that are assigned to
|
||||
name = co.co_names[oparg]
|
||||
m.globalnames[name] = 1
|
||||
elif what == "relative_import":
|
||||
level, fromlist, name = args
|
||||
if name:
|
||||
self._safe_import_hook(name, m, fromlist, level=level)
|
||||
else:
|
||||
parent = self.determine_parent(m, level=level)
|
||||
self._safe_import_hook(parent.__name__, None, fromlist, level=0)
|
||||
else:
|
||||
# We don't expect anything else from the generator.
|
||||
raise RuntimeError(what)
|
||||
|
||||
for c in co.co_consts:
|
||||
if isinstance(c, type(co)):
|
||||
self.scan_code(c, m)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,263 @@
|
|||
import __future__
|
||||
import sys, os
|
||||
import unittest
|
||||
import distutils.dir_util
|
||||
import tempfile
|
||||
|
||||
from test import test_support
|
||||
|
||||
try: set
|
||||
except NameError: from sets import Set as set
|
||||
|
||||
import modulefinder
|
||||
|
||||
# Note: To test modulefinder with Python 2.2, sets.py and
|
||||
# modulefinder.py must be available - they are not in the standard
|
||||
# library.
|
||||
|
||||
TEST_DIR = tempfile.mkdtemp()
|
||||
TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)]
|
||||
|
||||
# Each test description is a list of 5 items:
|
||||
#
|
||||
# 1. a module name that will be imported by modulefinder
|
||||
# 2. a list of module names that modulefinder is required to find
|
||||
# 3. a list of module names that modulefinder should complain
|
||||
# about because they are not found
|
||||
# 4. a list of module names that modulefinder should complain
|
||||
# about because they MAY be not found
|
||||
# 5. a string specifying packages to create; the format is obvious imo.
|
||||
#
|
||||
# Each package will be created in TEST_DIR, and TEST_DIR will be
|
||||
# removed after the tests again.
|
||||
# Modulefinder searches in a path that contains TEST_DIR, plus
|
||||
# the standard Lib directory.
|
||||
|
||||
maybe_test = [
|
||||
"a.module",
|
||||
["a", "a.module", "sys",
|
||||
"b"],
|
||||
["c"], ["b.something"],
|
||||
"""\
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
from b import something
|
||||
from c import something
|
||||
b/__init__.py
|
||||
from sys import *
|
||||
"""]
|
||||
|
||||
maybe_test_new = [
|
||||
"a.module",
|
||||
["a", "a.module", "sys",
|
||||
"b", "__future__"],
|
||||
["c"], ["b.something"],
|
||||
"""\
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
from b import something
|
||||
from c import something
|
||||
b/__init__.py
|
||||
from __future__ import absolute_import
|
||||
from sys import *
|
||||
"""]
|
||||
|
||||
package_test = [
|
||||
"a.module",
|
||||
["a", "a.b", "a.c", "a.module", "mymodule", "sys"],
|
||||
["blahblah"], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
import blahblah
|
||||
from a import b
|
||||
import c
|
||||
a/module.py
|
||||
import sys
|
||||
from a import b as x
|
||||
from a.c import sillyname
|
||||
a/b.py
|
||||
a/c.py
|
||||
from a.module import x
|
||||
import mymodule as sillyname
|
||||
from sys import version_info
|
||||
"""]
|
||||
|
||||
absolute_import_test = [
|
||||
"a.module",
|
||||
["a", "a.module",
|
||||
"b", "b.x", "b.y", "b.z",
|
||||
"__future__", "sys", "exceptions"],
|
||||
["blahblah"], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
from __future__ import absolute_import
|
||||
import sys # sys
|
||||
import blahblah # fails
|
||||
import exceptions # exceptions
|
||||
import b.x # b.x
|
||||
from b import y # b.y
|
||||
from b.z import * # b.z.*
|
||||
a/exceptions.py
|
||||
a/sys.py
|
||||
import mymodule
|
||||
a/b/__init__.py
|
||||
a/b/x.py
|
||||
a/b/y.py
|
||||
a/b/z.py
|
||||
b/__init__.py
|
||||
import z
|
||||
b/unused.py
|
||||
b/x.py
|
||||
b/y.py
|
||||
b/z.py
|
||||
"""]
|
||||
|
||||
relative_import_test = [
|
||||
"a.module",
|
||||
["__future__",
|
||||
"a", "a.module",
|
||||
"a.b", "a.b.y", "a.b.z",
|
||||
"a.b.c", "a.b.c.moduleC",
|
||||
"a.b.c.d", "a.b.c.e",
|
||||
"a.b.x",
|
||||
"exceptions"],
|
||||
[], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
from .b import y, z # a.b.y, a.b.z
|
||||
a/module.py
|
||||
from __future__ import absolute_import # __future__
|
||||
import exceptions # exceptions
|
||||
a/exceptions.py
|
||||
a/sys.py
|
||||
a/b/__init__.py
|
||||
from ..b import x # a.b.x
|
||||
#from a.b.c import moduleC
|
||||
from .c import moduleC # a.b.moduleC
|
||||
a/b/x.py
|
||||
a/b/y.py
|
||||
a/b/z.py
|
||||
a/b/g.py
|
||||
a/b/c/__init__.py
|
||||
from ..c import e # a.b.c.e
|
||||
a/b/c/moduleC.py
|
||||
from ..c import d # a.b.c.d
|
||||
a/b/c/d.py
|
||||
a/b/c/e.py
|
||||
a/b/c/x.py
|
||||
"""]
|
||||
|
||||
relative_import_test_2 = [
|
||||
"a.module",
|
||||
["a", "a.module",
|
||||
"a.sys",
|
||||
"a.b", "a.b.y", "a.b.z",
|
||||
"a.b.c", "a.b.c.d",
|
||||
"a.b.c.e",
|
||||
"a.b.c.moduleC",
|
||||
"a.b.c.f",
|
||||
"a.b.x",
|
||||
"a.another"],
|
||||
[], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
from . import sys # a.sys
|
||||
a/another.py
|
||||
a/module.py
|
||||
from .b import y, z # a.b.y, a.b.z
|
||||
a/exceptions.py
|
||||
a/sys.py
|
||||
a/b/__init__.py
|
||||
from .c import moduleC # a.b.c.moduleC
|
||||
from .c import d # a.b.c.d
|
||||
a/b/x.py
|
||||
a/b/y.py
|
||||
a/b/z.py
|
||||
a/b/c/__init__.py
|
||||
from . import e # a.b.c.e
|
||||
a/b/c/moduleC.py
|
||||
#
|
||||
from . import f # a.b.c.f
|
||||
from .. import x # a.b.x
|
||||
from ... import another # a.another
|
||||
a/b/c/d.py
|
||||
a/b/c/e.py
|
||||
a/b/c/f.py
|
||||
"""]
|
||||
|
||||
def open_file(path):
|
||||
##print "#", os.path.abspath(path)
|
||||
dirname = os.path.dirname(path)
|
||||
distutils.dir_util.mkpath(dirname)
|
||||
return open(path, "w")
|
||||
|
||||
def create_package(source):
|
||||
ofi = None
|
||||
for line in source.splitlines():
|
||||
if line.startswith(" ") or line.startswith("\t"):
|
||||
ofi.write(line.strip() + "\n")
|
||||
else:
|
||||
ofi = open_file(os.path.join(TEST_DIR, line.strip()))
|
||||
|
||||
class ModuleFinderTest(unittest.TestCase):
|
||||
def _do_test(self, info, report=False):
|
||||
import_this, modules, missing, maybe_missing, source = info
|
||||
create_package(source)
|
||||
try:
|
||||
mf = modulefinder.ModuleFinder(path=TEST_PATH)
|
||||
mf.import_hook(import_this)
|
||||
if report:
|
||||
mf.report()
|
||||
## # This wouldn't work in general when executed several times:
|
||||
## opath = sys.path[:]
|
||||
## sys.path = TEST_PATH
|
||||
## try:
|
||||
## __import__(import_this)
|
||||
## except:
|
||||
## import traceback; traceback.print_exc()
|
||||
## sys.path = opath
|
||||
## return
|
||||
modules = set(modules)
|
||||
found = set(mf.modules.keys())
|
||||
more = list(found - modules)
|
||||
less = list(modules - found)
|
||||
# check if we found what we expected, not more, not less
|
||||
self.failUnlessEqual((more, less), ([], []))
|
||||
|
||||
# check for missing and maybe missing modules
|
||||
bad, maybe = mf.any_missing_maybe()
|
||||
self.failUnlessEqual(bad, missing)
|
||||
self.failUnlessEqual(maybe, maybe_missing)
|
||||
finally:
|
||||
distutils.dir_util.remove_tree(TEST_DIR)
|
||||
|
||||
def test_package(self):
|
||||
self._do_test(package_test)
|
||||
|
||||
def test_maybe(self):
|
||||
self._do_test(maybe_test)
|
||||
|
||||
if getattr(__future__, "absolute_import", None):
|
||||
|
||||
def test_maybe_new(self):
|
||||
self._do_test(maybe_test_new)
|
||||
|
||||
def test_absolute_imports(self):
|
||||
self._do_test(absolute_import_test)
|
||||
|
||||
def test_relative_imports(self):
|
||||
self._do_test(relative_import_test)
|
||||
|
||||
def test_relative_imports_2(self):
|
||||
self._do_test(relative_import_test_2)
|
||||
|
||||
def test_main():
|
||||
test_support.run_unittest(ModuleFinderTest)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -87,6 +87,9 @@ Core and builtins
|
|||
Library
|
||||
-------
|
||||
|
||||
- Lib/modulefinder.py now handles absolute and relative imports
|
||||
correctly.
|
||||
|
||||
- Patch #1567274: Support SMTP over TLS.
|
||||
|
||||
- Patch #1560695: Add .note.GNU-stack to ctypes' sysv.S so that
|
||||
|
|
@ -199,6 +202,8 @@ Extension Modules
|
|||
Tests
|
||||
-----
|
||||
|
||||
- Added some tests for modulefinder.
|
||||
|
||||
- Converted test_imp to use unittest.
|
||||
|
||||
- Fix bsddb test_basics.test06_Transactions to check the version
|
||||
|
|
|
|||
Loading…
Reference in New Issue