Issue #15576: Allow extension modules to be a package's __init__

module again. Also took the opportunity to stop accidentally exporting
_imp.extension_suffixes() as public.

Doc/library/importlib.rst

@@ -671,9 +671,8 @@ find and load modules.
    The *path* argument is the directory for which the finder is in charge of
    searching.
 
-   The *loader_details* argument is a variable number of 3-item tuples each
+   The *loader_details* argument is a variable number of 2-item tuples each
-   containing a loader, file suffixes the loader recognizes, and a boolean
+   containing a loader and a sequence of file suffixes the loader recognizes.
-   representing whether the loader handles packages.
 
    The finder will cache the directory contents as necessary, making stat calls
    for each module search to verify the cache is not outdated. Because cache
@@ -798,7 +797,8 @@ find and load modules.
 
    .. method:: is_package(fullname)
 
-      Returns ``False`` as extension modules can never be packages.
+      Returns ``True`` if the file path points to a package's ``__init__``
+      module based on :attr:`EXTENSION_SUFFIXES`.
 
    .. method:: get_code(fullname)
 

Lib/imp.py

@@ -9,7 +9,7 @@ functionality over this module.
 from _imp import (lock_held, acquire_lock, release_lock,
                   load_dynamic, get_frozen_object, is_frozen_package,
                   init_builtin, init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename, extension_suffixes)
+                  _fix_co_filename)
 
 # Directly exposed by this module
 from importlib._bootstrap import new_module
@@ -51,7 +51,7 @@ def get_suffixes():
     warnings.warn('imp.get_suffixes() is deprecated; use the constants '
                   'defined on importlib.machinery instead',
                   DeprecationWarning, 2)
-    extensions = [(s, 'rb', C_EXTENSION) for s in extension_suffixes()]
+    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
     source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
     bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
 

Lib/importlib/_bootstrap.py

@@ -1067,6 +1067,10 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics):
         return None
 
 
+# Filled in by _setup().
+EXTENSION_SUFFIXES = []
+
+
 class ExtensionFileLoader:
 
     """Loader for extension modules.
@@ -1089,6 +1093,8 @@ class ExtensionFileLoader:
             module = _call_with_frames_removed(_imp.load_dynamic,
                                                fullname, self.path)
             _verbose_message('extension module loaded from {!r}', self.path)
+            if self.is_package(fullname):
+                module.__path__ = [_path_split(self.path)[0]]
             return module
         except:
             if not is_reload and fullname in sys.modules:
@@ -1097,7 +1103,12 @@ class ExtensionFileLoader:
 
     def is_package(self, fullname):
         """Return False as an extension module can never be a package."""
-        return False
+        file_name = _path_split(self.path)[1]
+        for suffix in EXTENSION_SUFFIXES:
+            if file_name == '__init__' + suffix:
+                return True
+        else:
+            return False
 
     def get_code(self, fullname):
         """Return None as an extension module cannot create a code object."""
@@ -1283,14 +1294,10 @@ class FileFinder:
         """Initialize with the path to search on and a variable number of
         3-tuples containing the loader, file suffixes the loader recognizes,
         and a boolean of whether the loader handles packages."""
-        packages = []
+        loaders = []
-        modules = []
+        for loader, suffixes in details:
-        for loader, suffixes, supports_packages in details:
+            loaders.extend((suffix, loader) for suffix in suffixes)
-            modules.extend((suffix, loader) for suffix in suffixes)
+        self._loaders = loaders
-            if supports_packages:
-                packages.extend((suffix, loader) for suffix in suffixes)
-        self.packages = packages
-        self.modules = modules
         # Base (directory) path
         self.path = path or '.'
         self._path_mtime = -1
@@ -1336,7 +1343,7 @@ class FileFinder:
         if cache_module in cache:
             base_path = _path_join(self.path, tail_module)
             if _path_isdir(base_path):
-                for suffix, loader in self.packages:
+                for suffix, loader in self._loaders:
                     init_filename = '__init__' + suffix
                     full_path = _path_join(base_path, init_filename)
                     if _path_isfile(full_path):
@@ -1346,7 +1353,7 @@ class FileFinder:
                     #  find a module in the next section.
                     is_namespace = True
         # Check for a file w/ a proper suffix exists.
-        for suffix, loader in self.modules:
+        for suffix, loader in self._loaders:
             if cache_module + suffix in cache:
                 full_path = _path_join(self.path, tail_module + suffix)
                 if _path_isfile(full_path):
@@ -1589,9 +1596,9 @@ def _get_supported_file_loaders():
 
     Each item is a tuple (loader, suffixes, allow_packages).
     """
-    extensions = ExtensionFileLoader, _imp.extension_suffixes(), False
+    extensions = ExtensionFileLoader, _imp.extension_suffixes()
-    source = SourceFileLoader, SOURCE_SUFFIXES, True
+    source = SourceFileLoader, SOURCE_SUFFIXES
-    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True
+    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
     return [extensions, source, bytecode]
 
 
@@ -1689,9 +1696,10 @@ def _setup(sys_module, _imp_module):
     setattr(self_module, 'path_separators', set(path_separators))
     # Constants
     setattr(self_module, '_relax_case', _make_relax_case())
+    EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
     if builtin_os == 'nt':
         SOURCE_SUFFIXES.append('.pyw')
-        if '_d.pyd' in _imp.extension_suffixes():
+        if '_d.pyd' in EXTENSION_SUFFIXES:
             WindowsRegistryFinder.DEBUG_BUILD = True
 
 

Lib/importlib/machinery.py

@@ -3,7 +3,8 @@
 import _imp
 
 from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
-                         OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES)
+                         OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
+                         EXTENSION_SUFFIXES)
 from ._bootstrap import BuiltinImporter
 from ._bootstrap import FrozenImporter
 from ._bootstrap import WindowsRegistryFinder
@@ -13,7 +14,6 @@ from ._bootstrap import SourceFileLoader
 from ._bootstrap import SourcelessFileLoader
 from ._bootstrap import ExtensionFileLoader
 
-EXTENSION_SUFFIXES = _imp.extension_suffixes()
 
 def all_suffixes():
     """Returns a list of all recognized module suffixes for this process"""

Lib/test/test_importlib/extension/test_case_sensitivity.py

@@ -16,8 +16,7 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
         assert good_name != bad_name
         finder = _bootstrap.FileFinder(ext_util.PATH,
                                         (_bootstrap.ExtensionFileLoader,
-                                            imp.extension_suffixes(),
+                                         _bootstrap.EXTENSION_SUFFIXES))
-                                            False))
         return finder.find_module(bad_name)
 
     def test_case_sensitive(self):

Lib/test/test_importlib/extension/test_finder.py

@@ -1,8 +1,7 @@
-from importlib import _bootstrap
+from importlib import machinery
 from .. import abc
 from . import util
 
-import imp
 import unittest
 
 class FinderTests(abc.FinderTests):
@@ -10,17 +9,16 @@ class FinderTests(abc.FinderTests):
     """Test the finder for extension modules."""
 
     def find_module(self, fullname):
-        importer = _bootstrap.FileFinder(util.PATH,
+        importer = machinery.FileFinder(util.PATH,
-                                          (_bootstrap.ExtensionFileLoader,
+                                        (machinery.ExtensionFileLoader,
-                                              imp.extension_suffixes(),
+                                         machinery.EXTENSION_SUFFIXES))
-                                              False))
         return importer.find_module(fullname)
 
     def test_module(self):
         self.assertTrue(self.find_module(util.NAME))
 
     def test_package(self):
-        # Extension modules cannot be an __init__ for a package.
+        # No extension module as an __init__ available for testing.
         pass
 
     def test_module_in_package(self):
@@ -28,7 +26,7 @@ class FinderTests(abc.FinderTests):
         pass
 
     def test_package_in_package(self):
-        # Extension modules cannot be an __init__ for a package.
+        # No extension module as an __init__ available for testing.
         pass
 
     def test_package_over_module(self):
@@ -38,8 +36,6 @@ class FinderTests(abc.FinderTests):
     def test_failure(self):
         self.assertIsNone(self.find_module('asdfjkl;'))
 
-    # XXX Raise an exception if someone tries to use the 'path' argument?
-
 
 def test_main():
     from test.support import run_unittest

Lib/test/test_importlib/extension/test_loader.py

@@ -3,6 +3,7 @@ from . import util as ext_util
 from .. import abc
 from .. import util
 
+import os.path
 import sys
 import unittest
 
@@ -38,11 +39,11 @@ class LoaderTests(abc.LoaderTests):
                                   machinery.ExtensionFileLoader)
 
     def test_package(self):
-        # Extensions are not found in packages.
+        # No extension module as __init__ available for testing.
         pass
 
     def test_lacking_parent(self):
-        # Extensions are not found in packages.
+        # No extension module in a package available for testing.
         pass
 
     def test_module_reuse(self):
@@ -61,6 +62,13 @@ class LoaderTests(abc.LoaderTests):
             self.load_module(name)
         self.assertEqual(cm.exception.name, name)
 
+    def test_is_package(self):
+        self.assertFalse(self.loader.is_package(ext_util.NAME))
+        for suffix in machinery.EXTENSION_SUFFIXES:
+            path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
+            loader = machinery.ExtensionFileLoader('pkg', path)
+            self.assertTrue(loader.is_package('pkg'))
+
 
 def test_main():
     from test.support import run_unittest

Lib/test/test_importlib/extension/test_path_hook.py

@@ -1,4 +1,4 @@
-from importlib import _bootstrap
+from importlib import machinery
 from . import util
 
 import collections
@@ -14,8 +14,8 @@ class PathHookTests(unittest.TestCase):
     # XXX Should it only work for directories containing an extension module?
 
     def hook(self, entry):
-        return _bootstrap.FileFinder.path_hook((_bootstrap.ExtensionFileLoader,
+        return machinery.FileFinder.path_hook((machinery.ExtensionFileLoader,
-            imp.extension_suffixes(), False))(entry)
+            machinery.EXTENSION_SUFFIXES))(entry)
 
     def test_success(self):
         # Path hook should handle a directory where a known extension module

Lib/test/test_importlib/source/test_case_sensitivity.py

@@ -23,11 +23,9 @@ class CaseSensitivityTest(unittest.TestCase):
     def find(self, path):
         finder = machinery.FileFinder(path,
                                       (machinery.SourceFileLoader,
-                                            machinery.SOURCE_SUFFIXES,
+                                            machinery.SOURCE_SUFFIXES),
-                                            True),
                                         (machinery.SourcelessFileLoader,
-                                            machinery.BYTECODE_SUFFIXES,
+                                            machinery.BYTECODE_SUFFIXES))
-                                            True))
         return finder.find_module(self.name)
 
     def sensitivity_test(self):

Lib/test/test_importlib/source/test_finder.py

@@ -37,9 +37,9 @@ class FinderTests(abc.FinderTests):
 
     def import_(self, root, module):
         loader_details = [(machinery.SourceFileLoader,
-                            machinery.SOURCE_SUFFIXES, True),
+                            machinery.SOURCE_SUFFIXES),
                           (machinery.SourcelessFileLoader,
-                            machinery.BYTECODE_SUFFIXES, True)]
+                            machinery.BYTECODE_SUFFIXES)]
         finder = machinery.FileFinder(root, *loader_details)
         return finder.find_module(module)
 
@@ -120,7 +120,7 @@ class FinderTests(abc.FinderTests):
     def test_empty_string_for_dir(self):
         # The empty string from sys.path means to search in the cwd.
         finder = machinery.FileFinder('', (machinery.SourceFileLoader,
-            machinery.SOURCE_SUFFIXES, True))
+            machinery.SOURCE_SUFFIXES))
         with open('mod.py', 'w') as file:
             file.write("# test file for importlib")
         try:
@@ -132,7 +132,7 @@ class FinderTests(abc.FinderTests):
     def test_invalidate_caches(self):
         # invalidate_caches() should reset the mtime.
         finder = machinery.FileFinder('', (machinery.SourceFileLoader,
-            machinery.SOURCE_SUFFIXES, True))
+            machinery.SOURCE_SUFFIXES))
         finder._path_mtime = 42
         finder.invalidate_caches()
         self.assertEqual(finder._path_mtime, -1)

Lib/test/test_importlib/source/test_path_hook.py

@@ -11,7 +11,7 @@ class PathHookTest(unittest.TestCase):
 
     def path_hook(self):
         return machinery.FileFinder.path_hook((machinery.SourceFileLoader,
-            machinery.SOURCE_SUFFIXES, True))
+            machinery.SOURCE_SUFFIXES))
 
     def test_success(self):
         with source_util.create_modules('dummy') as mapping:

Misc/NEWS

@@ -80,6 +80,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #15576: Allow extension modules to act as a package's __init__ module.
+
 - Issue #15502: Have importlib.invalidate_caches() work on sys.meta_path
   instead of sys.path_importer_cache.