issue 14660: Implement PEP 420, namespace packages.

Lib/importlib/_bootstrap.py

@@ -467,6 +467,10 @@ class BuiltinImporter:
 
     """
 
+    @classmethod
+    def module_repr(cls, module):
+        return "<module '{}' (built-in)>".format(module.__name__)
+
     @classmethod
     def find_module(cls, fullname, path=None):
         """Find the built-in module.
@@ -520,6 +524,10 @@ class FrozenImporter:
 
     """
 
+    @classmethod
+    def module_repr(cls, m):
+        return "<module '{}' (frozen)>".format(m.__name__)
+
     @classmethod
     def find_module(cls, fullname, path=None):
         """Find a frozen module."""
@@ -533,7 +541,10 @@ class FrozenImporter:
         """Load a frozen module."""
         is_reload = fullname in sys.modules
         try:
-            return _imp.init_frozen(fullname)
+            m = _imp.init_frozen(fullname)
+            # Let our own module_repr() method produce a suitable repr.
+            del m.__file__
+            return m
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -875,6 +886,79 @@ class ExtensionFileLoader:
         return None
 
 
+class _NamespacePath:
+    """Represents a namespace package's path.  It uses the module name
+    to find its parent module, and from there it looks up the parent's
+    __path__.  When this changes, the module's own path is recomputed,
+    using path_finder.  For top-leve modules, the parent module's path
+    is sys.path."""
+
+    def __init__(self, name, path, path_finder):
+        self._name = name
+        self._path = path
+        self._last_parent_path = tuple(self._get_parent_path())
+        self._path_finder = path_finder
+
+    def _find_parent_path_names(self):
+        """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
+        parent, dot, me = self._name.rpartition('.')
+        if dot == '':
+            # This is a top-level module. sys.path contains the parent path.
+            return 'sys', 'path'
+        # Not a top-level module. parent-module.__path__ contains the
+        #  parent path.
+        return parent, '__path__'
+
+    def _get_parent_path(self):
+        parent_module_name, path_attr_name = self._find_parent_path_names()
+        return getattr(sys.modules[parent_module_name], path_attr_name)
+
+    def _recalculate(self):
+        # If the parent's path has changed, recalculate _path
+        parent_path = tuple(self._get_parent_path()) # Make a copy
+        if parent_path != self._last_parent_path:
+            loader, new_path = self._path_finder(self._name, parent_path)
+            # Note that no changes are made if a loader is returned, but we
+            #  do remember the new parent path
+            if loader is None:
+                self._path = new_path
+            self._last_parent_path = parent_path     # Save the copy
+        return self._path
+
+    def __iter__(self):
+        return iter(self._recalculate())
+
+    def __len__(self):
+        return len(self._recalculate())
+
+    def __repr__(self):
+        return "_NamespacePath({0!r})".format(self._path)
+
+    def __contains__(self, item):
+        return item in self._recalculate()
+
+    def append(self, item):
+        self._path.append(item)
+
+
+class NamespaceLoader:
+    def __init__(self, name, path, path_finder):
+        self._path = _NamespacePath(name, path, path_finder)
+
+    @classmethod
+    def module_repr(cls, module):
+        return "<module '{}' (namespace)>".format(module.__name__)
+
+    @set_package
+    @set_loader
+    @module_for_loader
+    def load_module(self, module):
+        """Load a namespace module."""
+        _verbose_message('namespace module loaded with path {!r}', self._path)
+        module.__path__ = self._path
+        return module
+
+
 # Finders #####################################################################
 
 class PathFinder:
@@ -915,20 +999,47 @@ class PathFinder:
             sys.path_importer_cache[path] = finder
         return finder
 
+    @classmethod
+    def _get_loader(cls, fullname, path):
+        """Find the loader or namespace_path for this module/package name."""
+        # If this ends up being a namespace package, namespace_path is
+        #  the list of paths that will become its __path__
+        namespace_path = []
+        for entry in path:
+            finder = cls._path_importer_cache(entry)
+            if finder is not None:
+                if hasattr(finder, 'find_loader'):
+                    loader, portions = finder.find_loader(fullname)
+                else:
+                    loader = finder.find_module(fullname)
+                    portions = []
+                if loader is not None:
+                    # We found a loader: return it immediately.
+                    return (loader, namespace_path)
+                # This is possibly part of a namespace package.
+                #  Remember these path entries (if any) for when we
+                #  create a namespace package, and continue iterating
+                #  on path.
+                namespace_path.extend(portions)
+        else:
+            return (None, namespace_path)
+
     @classmethod
     def find_module(cls, fullname, path=None):
         """Find the module on sys.path or 'path' based on sys.path_hooks and
         sys.path_importer_cache."""
         if path is None:
             path = sys.path
-        for entry in path:
+        loader, namespace_path = cls._get_loader(fullname, path)
-            finder = cls._path_importer_cache(entry)
+        if loader is not None:
-            if finder is not None:
+            return loader
-                loader = finder.find_module(fullname)
-                if loader:
-                    return loader
         else:
-            return None
+            if namespace_path:
+                # We found at least one namespace path.  Return a
+                #  loader which can create the namespace package.
+                return NamespaceLoader(fullname, namespace_path, cls._get_loader)
+            else:
+                return None
 
 
 class FileFinder:
@@ -942,8 +1053,8 @@ class FileFinder:
 
     def __init__(self, path, *details):
         """Initialize with the path to search on and a variable number of
-        3-tuples containing the loader, file suffixes the loader recognizes, and
+        3-tuples containing the loader, file suffixes the loader recognizes,
-        a boolean of whether the loader handles packages."""
+        and a boolean of whether the loader handles packages."""
         packages = []
         modules = []
         for loader, suffixes, supports_packages in details:
@@ -964,6 +1075,19 @@ class FileFinder:
 
     def find_module(self, fullname):
         """Try to find a loader for the specified module."""
+        # Call find_loader(). If it returns a string (indicating this
+        # is a namespace package portion), generate a warning and
+        # return None.
+        loader, portions = self.find_loader(fullname)
+        assert len(portions) in [0, 1]
+        if loader is None and len(portions):
+            msg = "Not importing directory {}: missing __init__"
+            _warnings.warn(msg.format(portions[0]), ImportWarning)
+        return loader
+
+    def find_loader(self, fullname):
+        """Try to find a loader for the specified module, or the namespace
+        package portions. Returns (loader, list-of-portions)."""
         tail_module = fullname.rpartition('.')[2]
         try:
             mtime = _os.stat(self.path).st_mtime
@@ -987,17 +1111,17 @@ class FileFinder:
                     init_filename = '__init__' + suffix
                     full_path = _path_join(base_path, init_filename)
                     if _path_isfile(full_path):
-                        return loader(fullname, full_path)
+                        return (loader(fullname, full_path), [base_path])
                 else:
-                    msg = "Not importing directory {}: missing __init__"
+                    # A namespace package, return the path
-                    _warnings.warn(msg.format(base_path), ImportWarning)
+                    return (None, [base_path])
         # Check for a file w/ a proper suffix exists.
         for suffix, loader in self.modules:
             if cache_module + suffix in cache:
                 full_path = _path_join(self.path, tail_module + suffix)
                 if _path_isfile(full_path):
-                    return loader(fullname, full_path)
+                    return (loader(fullname, full_path), [])
-        return None
+        return (None, [])
 
     def _fill_cache(self):
         """Fill the cache of potential modules and packages for this directory."""

Lib/importlib/test/frozen/test_loader.py

@@ -10,38 +10,46 @@ class LoaderTests(abc.LoaderTests):
     def test_module(self):
         with util.uncache('__hello__'), captured_stdout() as stdout:
             module = machinery.FrozenImporter.load_module('__hello__')
-            check = {'__name__': '__hello__', '__file__': '<frozen>',
+            check = {'__name__': '__hello__',
-                    '__package__': '', '__loader__': machinery.FrozenImporter}
+                    '__package__': '', 
+                    '__loader__': machinery.FrozenImporter,
+                    }
             for attr, value in check.items():
                 self.assertEqual(getattr(module, attr), value)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
 
     def test_package(self):
         with util.uncache('__phello__'),  captured_stdout() as stdout:
             module = machinery.FrozenImporter.load_module('__phello__')
-            check = {'__name__': '__phello__', '__file__': '<frozen>',
+            check = {'__name__': '__phello__', 
-                     '__package__': '__phello__', '__path__': ['__phello__'],
+                     '__package__': '__phello__', 
-                     '__loader__': machinery.FrozenImporter}
+                     '__path__': ['__phello__'],
+                     '__loader__': machinery.FrozenImporter,
+                     }
             for attr, value in check.items():
                 attr_value = getattr(module, attr)
                 self.assertEqual(attr_value, value,
                                  "for __phello__.%s, %r != %r" %
                                  (attr, attr_value, value))
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
 
     def test_lacking_parent(self):
         with util.uncache('__phello__', '__phello__.spam'), \
              captured_stdout() as stdout:
             module = machinery.FrozenImporter.load_module('__phello__.spam')
-            check = {'__name__': '__phello__.spam', '__file__': '<frozen>',
+            check = {'__name__': '__phello__.spam',
                     '__package__': '__phello__',
-                    '__loader__': machinery.FrozenImporter}
+                    '__loader__': machinery.FrozenImporter,
+                    }
             for attr, value in check.items():
                 attr_value = getattr(module, attr)
                 self.assertEqual(attr_value, value,
                                  "for __phello__.spam.%s, %r != %r" %
                                  (attr, attr_value, value))
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
 
     def test_module_reuse(self):
         with util.uncache('__hello__'), captured_stdout() as stdout:
@@ -51,6 +59,12 @@ class LoaderTests(abc.LoaderTests):
             self.assertEqual(stdout.getvalue(),
                              'Hello world!\nHello world!\n')
 
+    def test_module_repr(self):
+        with util.uncache('__hello__'), captured_stdout():
+            module = machinery.FrozenImporter.load_module('__hello__')
+            self.assertEqual(repr(module), 
+                             "<module '__hello__' (frozen)>")
+
     def test_state_after_failure(self):
         # No way to trigger an error in a frozen module.
         pass

Lib/importlib/test/source/test_finder.py

@@ -106,36 +106,17 @@ class FinderTests(abc.FinderTests):
             loader = self.import_(pkg_dir, 'pkg.sub')
             self.assertTrue(hasattr(loader, 'load_module'))
 
-    # [sub empty]
-    def test_empty_sub_directory(self):
-        context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__')
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", ImportWarning)
-            with context as mapping:
-                os.unlink(mapping['pkg.sub.__init__'])
-                pkg_dir = os.path.dirname(mapping['pkg.__init__'])
-                with self.assertRaises(ImportWarning):
-                    self.import_(pkg_dir, 'pkg.sub')
-
     # [package over modules]
     def test_package_over_module(self):
         name = '_temp'
         loader = self.run_test(name, {'{0}.__init__'.format(name), name})
         self.assertTrue('__init__' in loader.get_filename(name))
 
-
     def test_failure(self):
         with source_util.create_modules('blah') as mapping:
             nothing = self.import_(mapping['.root'], 'sdfsadsadf')
             self.assertTrue(nothing is None)
 
-    # [empty dir]
-    def test_empty_dir(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", ImportWarning)
-            with self.assertRaises(ImportWarning):
-                self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'})
-
     def test_empty_string_for_dir(self):
         # The empty string from sys.path means to search in the cwd.
         finder = machinery.FileFinder('', (machinery.SourceFileLoader,

Lib/pkgutil.py

@@ -515,19 +515,29 @@ def extend_path(path, name):
 
     pname = os.path.join(*name.split('.')) # Reconstitute as relative path
     sname_pkg = name + ".pkg"
-    init_py = "__init__.py"
 
     path = path[:] # Start with a copy of the existing path
 
     for dir in sys.path:
-        if not isinstance(dir, str) or not os.path.isdir(dir):
+        if not isinstance(dir, str):
             continue
-        subdir = os.path.join(dir, pname)
+
-        # XXX This may still add duplicate entries to path on
+        finder = get_importer(dir)
-        # case-insensitive filesystems
+        if finder is not None:
-        initfile = os.path.join(subdir, init_py)
+            # Is this finder PEP 420 compliant?
-        if subdir not in path and os.path.isfile(initfile):
+            if hasattr(finder, 'find_loader'):
-            path.append(subdir)
+                loader, portions = finder.find_loader(name)
+            else:
+                # No, no need to call it
+                loader = None
+                portions = []
+
+            for portion in portions:
+                # XXX This may still add duplicate entries to path on
+                # case-insensitive filesystems
+                if portion not in path:
+                    path.append(portion)
+
         # XXX Is this the right thing for subpackages like zope.app?
         # It looks for a file named "zope.app.pkg"
         pkgfile = os.path.join(dir, sname_pkg)

Lib/test/namespace_pkgs/both_portions/foo/one.py

@@ -0,0 +1 @@
+attr = 'both_portions foo one'

Lib/test/namespace_pkgs/both_portions/foo/two.py

@@ -0,0 +1 @@
+attr = 'both_portions foo two'

Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py

@@ -0,0 +1 @@
+attr = 'portion1 foo one'

Lib/test/namespace_pkgs/portion1/foo/one.py

@@ -0,0 +1 @@
+attr = 'portion1 foo one'

Lib/test/namespace_pkgs/portion2/foo/two.py

@@ -0,0 +1 @@
+attr = 'portion2 foo two'

Lib/test/namespace_pkgs/project1/parent/child/one.py

@@ -0,0 +1 @@
+attr = 'parent child one'

Lib/test/namespace_pkgs/project2/parent/child/two.py

@@ -0,0 +1 @@
+attr = 'parent child two'

Lib/test/namespace_pkgs/project3/parent/child/three.py

@@ -0,0 +1 @@
+attr = 'parent child three'

Lib/test/test_frozen.py

@@ -7,7 +7,7 @@ import sys
 class FrozenTests(unittest.TestCase):
 
     module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
-                              '__file__', '__loader__', '__name__',
+                              '__loader__', '__name__',
                               '__package__'])
     package_attrs = frozenset(list(module_attrs) + ['__path__'])
 

Lib/test/test_import.py

@@ -286,12 +286,6 @@ class ImportTests(unittest.TestCase):
         import test.support as y
         self.assertIs(y, test.support, y.__name__)
 
-    def test_import_initless_directory_warning(self):
-        with check_warnings(('', ImportWarning)):
-            # Just a random non-package directory we always expect to be
-            # somewhere in sys.path...
-            self.assertRaises(ImportError, __import__, "site-packages")
-
     def test_import_by_filename(self):
         path = os.path.abspath(TESTFN)
         encoding = sys.getfilesystemencoding()

Lib/test/test_module.py

@@ -5,6 +5,15 @@ from test.support import run_unittest, gc_collect
 import sys
 ModuleType = type(sys)
 
+class FullLoader:
+    @classmethod
+    def module_repr(cls, m):
+        return "<module '{}' (crafted)>".format(m.__name__)
+
+class BareLoader:
+    pass
+
+
 class ModuleTests(unittest.TestCase):
     def test_uninitialized(self):
         # An uninitialized module has no __dict__ or __name__,
@@ -80,8 +89,90 @@ a = A(destroyed)"""
         gc_collect()
         self.assertEqual(destroyed, [1])
 
+    def test_module_repr_minimal(self):
+        # reprs when modules have no __file__, __name__, or __loader__
+        m = ModuleType('foo')
+        del m.__name__
+        self.assertEqual(repr(m), "<module '?'>")
+
+    def test_module_repr_with_name(self):
+        m = ModuleType('foo')
+        self.assertEqual(repr(m), "<module 'foo'>")
+
+    def test_module_repr_with_name_and_filename(self):
+        m = ModuleType('foo')
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module 'foo' from '/tmp/foo.py'>")
+
+    def test_module_repr_with_filename_only(self):
+        m = ModuleType('foo')
+        del m.__name__
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module '?' from '/tmp/foo.py'>")
+
+    def test_module_repr_with_bare_loader_but_no_name(self):
+        m = ModuleType('foo')
+        del m.__name__
+        # Yes, a class not an instance.
+        m.__loader__ = BareLoader
+        self.assertEqual(
+            repr(m), "<module '?' (<class 'test.test_module.BareLoader'>)>")
+
+    def test_module_repr_with_full_loader_but_no_name(self):
+        # m.__loader__.module_repr() will fail because the module has no
+        # m.__name__.  This exception will get suppressed and instead the
+        # loader's repr will be used.
+        m = ModuleType('foo')
+        del m.__name__
+        # Yes, a class not an instance.
+        m.__loader__ = FullLoader
+        self.assertEqual(
+            repr(m), "<module '?' (<class 'test.test_module.FullLoader'>)>")
+
+    def test_module_repr_with_bare_loader(self):
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = BareLoader
+        self.assertEqual(
+            repr(m), "<module 'foo' (<class 'test.test_module.BareLoader'>)>")
+
+    def test_module_repr_with_full_loader(self):
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = FullLoader
+        self.assertEqual(
+            repr(m), "<module 'foo' (crafted)>")
+
+    def test_module_repr_with_bare_loader_and_filename(self):
+        # Because the loader has no module_repr(), use the file name.
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = BareLoader
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module 'foo' from '/tmp/foo.py'>")
+
+    def test_module_repr_with_full_loader_and_filename(self):
+        # Even though the module has an __file__, use __loader__.module_repr()
+        m = ModuleType('foo')
+        # Yes, a class not an instance.
+        m.__loader__ = FullLoader
+        m.__file__ = '/tmp/foo.py'
+        self.assertEqual(repr(m), "<module 'foo' (crafted)>")
+
+    def test_module_repr_builtin(self):
+        self.assertEqual(repr(sys), "<module 'sys' (built-in)>")
+
+    def test_module_repr_source(self):
+        r = repr(unittest)
+        self.assertEqual(r[:25], "<module 'unittest' from '")
+        self.assertEqual(r[-13:], "__init__.py'>")
+
+    # frozen and namespace module reprs are tested in importlib.
+
+
 def test_main():
     run_unittest(ModuleTests)
 
+
 if __name__ == '__main__':
     test_main()

Lib/test/test_namespace_pkgs.py

@@ -0,0 +1,239 @@
+import sys
+import contextlib
+import unittest
+import os
+
+import importlib.test.util
+from test.support import run_unittest
+
+# needed tests:
+#
+# need to test when nested, so that the top-level path isn't sys.path
+# need to test dynamic path detection, both at top-level and nested
+# with dynamic path, check when a loader is returned on path reload (that is,
+#  trying to switch from a namespace package to a regular package)
+
+
+@contextlib.contextmanager
+def sys_modules_context():
+    """
+    Make sure sys.modules is the same object and has the same content
+    when exiting the context as when entering.
+
+    Similar to importlib.test.util.uncache, but doesn't require explicit
+    names.
+    """
+    sys_modules_saved = sys.modules
+    sys_modules_copy = sys.modules.copy()
+    try:
+        yield
+    finally:
+        sys.modules = sys_modules_saved
+        sys.modules.clear()
+        sys.modules.update(sys_modules_copy)
+
+
+@contextlib.contextmanager
+def namespace_tree_context(**kwargs):
+    """
+    Save import state and sys.modules cache and restore it on exit.
+    Typical usage:
+
+    >>> with namespace_tree_context(path=['/tmp/xxyy/portion1',
+    ...         '/tmp/xxyy/portion2']):
+    ...     pass
+    """
+    # use default meta_path and path_hooks unless specified otherwise
+    kwargs.setdefault('meta_path', sys.meta_path)
+    kwargs.setdefault('path_hooks', sys.path_hooks)
+    import_context = importlib.test.util.import_state(**kwargs)
+    with import_context, sys_modules_context():
+        yield
+
+class NamespacePackageTest(unittest.TestCase):
+    """
+    Subclasses should define self.root and self.paths (under that root)
+    to be added to sys.path.
+    """
+    root = os.path.join(os.path.dirname(__file__), 'namespace_pkgs')
+
+    def setUp(self):
+        self.resolved_paths = [
+            os.path.join(self.root, path) for path in self.paths
+        ]
+        self.ctx = namespace_tree_context(path=self.resolved_paths)
+        self.ctx.__enter__()
+
+    def tearDown(self):
+        # TODO: will we ever want to pass exc_info to __exit__?
+        self.ctx.__exit__(None, None, None)
+
+class SingleNamespacePackage(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+    def test_module_repr(self):
+        import foo.one
+        self.assertEqual(repr(foo), "<module 'foo' (namespace)>")
+
+
+class DynamicPatheNamespacePackage(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_dynamic_path(self):
+        # Make sure only 'foo.one' can be imported
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+        with self.assertRaises(ImportError):
+            import foo.two
+
+        # Now modify sys.path
+        sys.path.append(os.path.join(self.root, 'portion2'))
+
+        # And make sure foo.two is now importable
+        import foo.two
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class CombinedNamespacePackages(NamespacePackageTest):
+    paths = ['both_portions']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'both_portions foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SeparatedNamespacePackages(NamespacePackageTest):
+    paths = ['portion1', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
+    paths = ['portion1', 'both_portions']
+
+    def test_first_path_wins(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+    def test_first_path_wins_again(self):
+        sys.path.reverse()
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'both_portions foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+    def test_first_path_wins_importing_second_first(self):
+        import foo.two
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SingleZipNamespacePackage(NamespacePackageTest):
+    paths = ['top_level_portion1.zip']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+
+class SeparatedZipNamespacePackages(NamespacePackageTest):
+    paths = ['top_level_portion1.zip', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+        self.assertIn('top_level_portion1.zip', foo.one.__file__)
+        self.assertNotIn('.zip', foo.two.__file__)
+
+
+class SingleNestedZipNamespacePackage(NamespacePackageTest):
+    paths = ['nested_portion1.zip/nested_portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+
+class SeparatedNestedZipNamespacePackages(NamespacePackageTest):
+    paths = ['nested_portion1.zip/nested_portion1', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+        fn = os.path.join('nested_portion1.zip', 'nested_portion1')
+        self.assertIn(fn, foo.one.__file__)
+        self.assertNotIn('.zip', foo.two.__file__)
+
+
+class LegacySupport(NamespacePackageTest):
+    paths = ['not_a_namespace_pkg', 'portion1', 'portion2', 'both_portions']
+
+    def test_non_namespace_package_takes_precedence(self):
+        import foo.one
+        with self.assertRaises(ImportError):
+            import foo.two
+        self.assertIn('__init__', foo.__file__)
+        self.assertNotIn('namespace', str(foo.__loader__).lower())
+
+
+class ZipWithMissingDirectory(NamespacePackageTest):
+    paths = ['missing_directory.zip']
+
+    @unittest.expectedFailure
+    def test_missing_directory(self):
+        # This will fail because missing_directory.zip contains:
+        #   Length      Date    Time    Name
+        # ---------  ---------- -----   ----
+        #        29  2012-05-03 18:13   foo/one.py
+        #         0  2012-05-03 20:57   bar/
+        #        38  2012-05-03 20:57   bar/two.py
+        # ---------                     -------
+        #        67                     3 files
+
+        # Because there is no 'foo/', the zipimporter currently doesn't
+        #  know that foo is a namespace package
+
+        import foo.one
+
+    def test_present_directory(self):
+        # This succeeds because there is a "bar/" in the zip file
+        import bar.two
+        self.assertEqual(bar.two.attr, 'missing_directory foo two')
+
+
+def test_main():
+    run_unittest(*NamespacePackageTest.__subclasses__())
+
+
+if __name__ == "__main__":
+    test_main()

Lib/test/test_pkgutil.py

@@ -138,10 +138,11 @@ class PkgutilPEP302Tests(unittest.TestCase):
         del sys.modules['foo']
 
 
+# These tests, especially the setup and cleanup, are hideous. They
+# need to be cleaned up once issue 14715 is addressed.
 class ExtendPathTests(unittest.TestCase):
     def create_init(self, pkgname):
         dirname = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, dirname)
         sys.path.insert(0, dirname)
 
         pkgdir = os.path.join(dirname, pkgname)
@@ -156,22 +157,12 @@ class ExtendPathTests(unittest.TestCase):
         with open(module_name, 'w') as fl:
             print('value={}'.format(value), file=fl)
 
-    def setUp(self):
-        # Create 2 directories on sys.path
-        self.pkgname = 'foo'
-        self.dirname_0 = self.create_init(self.pkgname)
-        self.dirname_1 = self.create_init(self.pkgname)
-
-    def tearDown(self):
-        del sys.path[0]
-        del sys.path[0]
-        del sys.modules['foo']
-        del sys.modules['foo.bar']
-        del sys.modules['foo.baz']
-
     def test_simple(self):
-        self.create_submodule(self.dirname_0, self.pkgname, 'bar', 0)
+        pkgname = 'foo'
-        self.create_submodule(self.dirname_1, self.pkgname, 'baz', 1)
+        dirname_0 = self.create_init(pkgname)
+        dirname_1 = self.create_init(pkgname)
+        self.create_submodule(dirname_0, pkgname, 'bar', 0)
+        self.create_submodule(dirname_1, pkgname, 'baz', 1)
         import foo.bar
         import foo.baz
         # Ensure we read the expected values
@@ -180,8 +171,45 @@ class ExtendPathTests(unittest.TestCase):
 
         # Ensure the path is set up correctly
         self.assertEqual(sorted(foo.__path__),
-                         sorted([os.path.join(self.dirname_0, self.pkgname),
+                         sorted([os.path.join(dirname_0, pkgname),
-                                 os.path.join(self.dirname_1, self.pkgname)]))
+                                 os.path.join(dirname_1, pkgname)]))
+
+        # Cleanup
+        shutil.rmtree(dirname_0)
+        shutil.rmtree(dirname_1)
+        del sys.path[0]
+        del sys.path[0]
+        del sys.modules['foo']
+        del sys.modules['foo.bar']
+        del sys.modules['foo.baz']
+
+    def test_mixed_namespace(self):
+        pkgname = 'foo'
+        dirname_0 = self.create_init(pkgname)
+        dirname_1 = self.create_init(pkgname)
+        self.create_submodule(dirname_0, pkgname, 'bar', 0)
+        # Turn this into a PEP 420 namespace package
+        os.unlink(os.path.join(dirname_0, pkgname, '__init__.py'))
+        self.create_submodule(dirname_1, pkgname, 'baz', 1)
+        import foo.bar
+        import foo.baz
+        # Ensure we read the expected values
+        self.assertEqual(foo.bar.value, 0)
+        self.assertEqual(foo.baz.value, 1)
+
+        # Ensure the path is set up correctly
+        self.assertEqual(sorted(foo.__path__),
+                         sorted([os.path.join(dirname_0, pkgname),
+                                 os.path.join(dirname_1, pkgname)]))
+
+        # Cleanup
+        shutil.rmtree(dirname_0)
+        shutil.rmtree(dirname_1)
+        del sys.path[0]
+        del sys.path[0]
+        del sys.modules['foo']
+        del sys.modules['foo.bar']
+        del sys.modules['foo.baz']
 
     # XXX: test .pkg files
 

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.3.0 Alpha 4?
 Core and Builtins
 -----------------
 
+- Issue #14660 (PEP 420): Namespace packages. Implemented by Eric Smith.
+
 - Issue #14494: Fix __future__.py and its documentation to note that
   absolute imports are the default behavior in 3.0 instead of 2.7.
   Patch by Sven Marnach.

Modules/zipimport.c

@@ -259,6 +259,29 @@ enum zi_module_info {
     MI_PACKAGE
 };
 
+/* Does this path represent a directory?
+   on error, return < 0
+   if not a dir, return 0
+   if a dir, return 1
+*/
+static int
+check_is_directory(ZipImporter *self, PyObject* prefix, PyObject *path)
+{
+    PyObject *dirpath;
+    PyObject *item;
+
+    /* See if this is a "directory". If so, it's eligible to be part
+       of a namespace package. We test by seeing if the name, with an
+       appended path separator, exists. */
+    dirpath = PyUnicode_FromFormat("%U%U%c", prefix, path, SEP);
+    if (dirpath == NULL)
+        return -1;
+    /* If dirpath is present in self->files, we have a directory. */
+    item = PyDict_GetItem(self->files, dirpath);
+    Py_DECREF(dirpath);
+    return item != NULL;
+}
+
 /* Return some information about a module. */
 static enum zi_module_info
 get_module_info(ZipImporter *self, PyObject *fullname)
@@ -296,6 +319,46 @@ get_module_info(ZipImporter *self, PyObject *fullname)
     return MI_NOT_FOUND;
 }
 
+/* The guts of "find_loader" and "find_module". Return values:
+   -1: error
+    0: no loader or namespace portions found
+    1: module/package found
+    2: namespace portion found: *namespace_portion will point to the name
+*/
+static int
+find_loader(ZipImporter *self, PyObject *fullname, PyObject **namespace_portion)
+{
+    enum zi_module_info mi;
+
+    *namespace_portion = NULL;
+
+    mi = get_module_info(self, fullname);
+    if (mi == MI_ERROR)
+        return -1;
+    if (mi == MI_NOT_FOUND) {
+        /* Not a module or regular package. See if this is a directory, and
+           therefore possibly a portion of a namespace package. */
+        int is_dir = check_is_directory(self, self->prefix, fullname);
+        if (is_dir < 0)
+            return -1;
+        if (is_dir) {
+            /* This is possibly a portion of a namespace
+               package. Return the string representing its path,
+               without a trailing separator. */
+            *namespace_portion = PyUnicode_FromFormat("%U%c%U%U",
+                                                      self->archive, SEP,
+                                                      self->prefix, fullname);
+            if (*namespace_portion == NULL)
+                return -1;
+            return 2;
+        }
+        return 0;
+    }
+    /* This is a module or package. */
+    return 1;
+}
+
+
 /* Check whether we can satisfy the import of the module named by
    'fullname'. Return self if we can, None if we can't. */
 static PyObject *
@@ -304,21 +367,78 @@ zipimporter_find_module(PyObject *obj, PyObject *args)
     ZipImporter *self = (ZipImporter *)obj;
     PyObject *path = NULL;
     PyObject *fullname;
-    enum zi_module_info mi;
+    PyObject* namespace_portion = NULL;
 
     if (!PyArg_ParseTuple(args, "U|O:zipimporter.find_module",
                           &fullname, &path))
-        return NULL;
+        goto error;
 
-    mi = get_module_info(self, fullname);
+    switch (find_loader(self, fullname, &namespace_portion)) {
-    if (mi == MI_ERROR)
+    case -1:            /* Error */
-        return NULL;
+        goto error;
-    if (mi == MI_NOT_FOUND) {
+    case 0:             /* Not found, return None */
+        Py_INCREF(Py_None);
+        return Py_None;
+    case 1:             /* Return self */
+        Py_INCREF(self);
+        return (PyObject *)self;
+    case 2:             /* A namespace portion, but not allowed via
+                           find_module, so return None */
+        Py_DECREF(namespace_portion);
         Py_INCREF(Py_None);
         return Py_None;
     }
-    Py_INCREF(self);
+    /* Can't get here. */
-    return (PyObject *)self;
+    assert(0);
+    return NULL;
+error:
+    Py_XDECREF(namespace_portion);
+    return NULL;
+}
+
+
+/* Check whether we can satisfy the import of the module named by
+   'fullname', or whether it could be a portion of a namespace
+   package. Return self if we can load it, a string containing the
+   full path if it's a possible namespace portion, None if we
+   can't load it. */
+static PyObject *
+zipimporter_find_loader(PyObject *obj, PyObject *args)
+{
+    ZipImporter *self = (ZipImporter *)obj;
+    PyObject *path = NULL;
+    PyObject *fullname;
+    PyObject *result = NULL;
+    PyObject *namespace_portion = NULL;
+
+    if (!PyArg_ParseTuple(args, "U|O:zipimporter.find_module",
+                          &fullname, &path))
+        goto error;
+
+    switch (find_loader(self, fullname, &namespace_portion)) {
+    case -1:            /* Error */
+        goto error;
+    case 0:             /* Not found, return (None, []) */
+        if (!(result = Py_BuildValue("O[]", Py_None)))
+            goto error;
+        return result;
+    case 1:             /* Return (self, []) */
+        if (!(result = Py_BuildValue("O[]", self)))
+            goto error;
+        return result;
+    case 2:             /* Return (None, [namespace_portion]) */
+        if (!(result = Py_BuildValue("O[O]", Py_None, namespace_portion)))
+            goto error;
+        return result;
+    }
+    /* Can't get here. */
+    assert(0);
+        return NULL;
+
+error:
+    Py_XDECREF(namespace_portion);
+    Py_XDECREF(result);
+        return NULL;
 }
 
 /* Load and return the module named by 'fullname'. */
@@ -558,6 +678,16 @@ instance itself if the module was found, or None if it wasn't.\n\
 The optional 'path' argument is ignored -- it's there for compatibility\n\
 with the importer protocol.");
 
+PyDoc_STRVAR(doc_find_loader,
+"find_loader(fullname, path=None) -> self, str or None.\n\
+\n\
+Search for a module specified by 'fullname'. 'fullname' must be the\n\
+fully qualified (dotted) module name. It returns the zipimporter\n\
+instance itself if the module was found, a string containing the\n\
+full path name if it's possibly a portion of a namespace package,\n\
+or None otherwise. The optional 'path' argument is ignored -- it's\n\
+ there for compatibility with the importer protocol.");
+
 PyDoc_STRVAR(doc_load_module,
 "load_module(fullname) -> module.\n\
 \n\
@@ -599,6 +729,8 @@ Return the filename for the specified module.");
 static PyMethodDef zipimporter_methods[] = {
     {"find_module", zipimporter_find_module, METH_VARARGS,
      doc_find_module},
+    {"find_loader", zipimporter_find_loader, METH_VARARGS,
+     doc_find_loader},
     {"load_module", zipimporter_load_module, METH_VARARGS,
      doc_load_module},
     {"get_data", zipimporter_get_data, METH_VARARGS,

Objects/moduleobject.c

@@ -366,8 +366,28 @@ module_dealloc(PyModuleObject *m)
 static PyObject *
 module_repr(PyModuleObject *m)
 {
-    PyObject *name, *filename, *repr;
+    PyObject *name, *filename, *repr, *loader = NULL;
 
+    /* See if the module has an __loader__.  If it does, give the loader the
+     * first shot at producing a repr for the module.
+     */
+    if (m->md_dict != NULL) {
+        loader = PyDict_GetItemString(m->md_dict, "__loader__");
+    }
+    if (loader != NULL) {
+        repr = PyObject_CallMethod(loader, "module_repr", "(O)",
+                                   (PyObject *)m, NULL);
+        if (repr == NULL) {
+            PyErr_Clear();
+        }
+        else {
+            return repr;
+        }
+    }
+    /* __loader__.module_repr(m) did not provide us with a repr.  Next, see if
+     * the module has an __file__.  If it doesn't then use repr(__loader__) if
+     * it exists, otherwise, just use module.__name__.
+     */
     name = PyModule_GetNameObject((PyObject *)m);
     if (name == NULL) {
         PyErr_Clear();
@@ -378,8 +398,17 @@ module_repr(PyModuleObject *m)
     filename = PyModule_GetFilenameObject((PyObject *)m);
     if (filename == NULL) {
         PyErr_Clear();
-        repr = PyUnicode_FromFormat("<module %R (built-in)>", name);
+        /* There's no m.__file__, so if there was an __loader__, use that in
+         * the repr, otherwise, the only thing you can use is m.__name__
+         */
+        if (loader == NULL) {
+            repr = PyUnicode_FromFormat("<module %R>", name);
+        }
+        else {
+            repr = PyUnicode_FromFormat("<module %R (%R)>", name, loader);
+        }
     }
+    /* Finally, use m.__file__ */
     else {
         repr = PyUnicode_FromFormat("<module %R from %R>", name, filename);
         Py_DECREF(filename);