Issue #2377: Make importlib the implementation of __import__().

importlib._bootstrap is now frozen into Python/importlib.h and stored
as _frozen_importlib in sys.modules. Py_Initialize() loads the frozen
code along with sys and imp and then uses _frozen_importlib._install()
to set builtins.__import__() w/ _frozen_importlib.__import__().

Include/abstract.h

@@ -339,6 +339,10 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 
      PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(PyObject *o,
                                                        PyObject *method, ...);
+     PyAPI_FUNC(PyObject *) _PyObject_CallMethodObjIdArgs(PyObject *o,
+                                               struct _Py_Identifier *method,
+                                               ...);
+
 
        /*
      Call the method named m of object o with a variable number of

Include/dictobject.h

@@ -109,6 +109,8 @@ PyAPI_DATA(PyTypeObject) PyDictValues_Type;
 PyAPI_FUNC(PyObject *) PyDict_New(void);
 PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
+PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
+                                                  struct _Py_Identifier *key);
 PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
 PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);

Include/import.h

@@ -7,6 +7,9 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(void) _PyImportZip_Init(void);
+
+PyMODINIT_FUNC PyInit_imp(void);
 PyAPI_FUNC(long) PyImport_GetMagicNumber(void);
 PyAPI_FUNC(const char *) PyImport_GetMagicTag(void);
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(

Include/pystate.h

@@ -25,6 +25,7 @@ typedef struct _is {
     PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;
+    PyObject *importlib;
     PyObject *modules_reloading;
 
     PyObject *codec_search_path;
@@ -33,6 +34,7 @@ typedef struct _is {
     int codecs_initialized;
     int fscodec_initialized;
 
+
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif

Include/pythonrun.h

@@ -188,7 +188,7 @@ PyAPI_FUNC(const char *) _Py_hgversion(void);
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
 PyAPI_FUNC(PyObject *) _PySys_Init(void);
 PyAPI_FUNC(void) _PyImport_Init(void);
-PyAPI_FUNC(void) _PyExc_Init(void);
+PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
 PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(void) _PyFloat_Init(void);

Lib/importlib/_bootstrap.py

@@ -160,6 +160,13 @@ code_type = type(_wrap.__code__)
 
 # Finder/loader utility code ##################################################
 
+def verbose_message(message, *args):
+    """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
+    if sys.flags.verbose:
+        if not message.startswith('#') and not message.startswith('import '):
+            message = '# ' + message
+        print(message.format(*args), file=sys.stderr)
+
 
 def set_package(fxn):
     """Set __package__ on the returned module."""
@@ -388,9 +395,13 @@ class _LoaderBasics:
             raise ImportError("bad magic number in {}".format(fullname),
                               name=fullname, path=bytecode_path)
         elif len(raw_timestamp) != 4:
-            raise EOFError("bad timestamp in {}".format(fullname))
+            message = 'bad timestamp in {}'.format(fullname)
+            verbose_message(message)
+            raise EOFError(message)
         elif len(raw_size) != 4:
-            raise EOFError("bad size in {}".format(fullname))
+            message = 'bad size in {}'.format(fullname)
+            verbose_message(message)
+            raise EOFError(message)
         if source_stats is not None:
             try:
                 source_mtime = int(source_stats['mtime'])
@@ -398,9 +409,10 @@ class _LoaderBasics:
                 pass
             else:
                 if _r_long(raw_timestamp) != source_mtime:
-                    raise ImportError(
+                    message = 'bytecode is stale for {}'.format(fullname)
-                        "bytecode is stale for {}".format(fullname),
+                    verbose_message(message)
-                        name=fullname, path=bytecode_path)
+                    raise ImportError(message, name=fullname,
+                                      path=bytecode_path)
             try:
                 source_size = source_stats['size'] & 0xFFFFFFFF
             except KeyError:
@@ -506,9 +518,13 @@ class SourceLoader(_LoaderBasics):
                     except (ImportError, EOFError):
                         pass
                     else:
+                        verbose_message('{} matches {}', bytecode_path,
+                                        source_path)
                         found = marshal.loads(bytes_data)
                         if isinstance(found, code_type):
                             imp._fix_co_filename(found, source_path)
+                            verbose_message('code object from {}',
+                                            bytecode_path)
                             return found
                         else:
                             msg = "Non-code object in {}"
@@ -517,6 +533,7 @@ class SourceLoader(_LoaderBasics):
         source_bytes = self.get_data(source_path)
         code_object = compile(source_bytes, source_path, 'exec',
                                 dont_inherit=True)
+        verbose_message('code object from {}', source_path)
         if (not sys.dont_write_bytecode and bytecode_path is not None and
             source_mtime is not None):
             # If e.g. Jython ever implements imp.cache_from_source to have
@@ -528,6 +545,7 @@ class SourceLoader(_LoaderBasics):
             data.extend(marshal.dumps(code_object))
             try:
                 self.set_data(bytecode_path, data)
+                verbose_message('wrote {!r}', bytecode_path)
             except NotImplementedError:
                 pass
         return code_object
@@ -596,6 +614,7 @@ class _SourceFileLoader(_FileLoader, SourceLoader):
                 return
         try:
             _write_atomic(path, data)
+            verbose_message('created {!r}', path)
         except (PermissionError, FileExistsError):
             # Don't worry if you can't write bytecode or someone is writing
             # it at the same time.
@@ -615,6 +634,7 @@ class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
         bytes_data = self._bytes_from_bytecode(fullname, data, path, None)
         found = marshal.loads(bytes_data)
         if isinstance(found, code_type):
+            verbose_message('code object from {!r}', path)
             return found
         else:
             raise ImportError("Non-code object in {}".format(path),
@@ -644,7 +664,9 @@ class _ExtensionFileLoader:
         """Load an extension module."""
         is_reload = fullname in sys.modules
         try:
-            return imp.load_dynamic(fullname, self._path)
+            module = imp.load_dynamic(fullname, self._path)
+            verbose_message('extension module loaded from {!r}', self._path)
+            return module
         except:
             if not is_reload and fullname in sys.modules:
                 del sys.modules[fullname]
@@ -953,6 +975,7 @@ def _find_and_load(name, import_):
     elif name not in sys.modules:
         # The parent import may have already imported this module.
         loader.load_module(name)
+        verbose_message('import {!r} # {!r}', name, loader)
     # Backwards-compatibility; be nicer to skip the dict lookup.
     module = sys.modules[name]
     if parent:

Lib/importlib/test/import_/test_path.py

@@ -87,7 +87,7 @@ class FinderTests(unittest.TestCase):
 
 class DefaultPathFinderTests(unittest.TestCase):
 
-    """Test importlib._bootstrap._DefaultPathFinder."""
+    """Test _bootstrap._DefaultPathFinder."""
 
     def test_implicit_hooks(self):
         # Test that the implicit path hooks are used.

Lib/importlib/test/import_/util.py

@@ -1,6 +1,5 @@
 import functools
 import importlib
-import importlib._bootstrap
 import unittest
 
 

Lib/importlib/test/regrtest.py

@@ -13,16 +13,4 @@ from test import regrtest
 if __name__ == '__main__':
     __builtins__.__import__ = importlib.__import__
 
-    exclude = ['--exclude',
-                'test_frozen', # Does not expect __loader__ attribute
-                'test_pkg',  # Does not expect __loader__ attribute
-                'test_pydoc', # Does not expect __loader__ attribute
-              ]
-
-    # Switching on --exclude implies running all test but the ones listed, so
-    # only use it when one is not running an explicit test
-    if len(sys.argv) == 1:
-        # No programmatic way to specify tests to exclude
-        sys.argv.extend(exclude)
-
     regrtest.main(quiet=True, verbose2=True)

Lib/importlib/test/source/test_file_loader.py

@@ -1,5 +1,5 @@
+from ... import _bootstrap
 import importlib
-from importlib import _bootstrap
 from .. import abc
 from .. import util
 from . import util as source_util

Lib/importlib/test/source/test_finder.py

@@ -1,10 +1,11 @@
-from importlib import _bootstrap
 from .. import abc
 from . import util as source_util
-from test.support import make_legacy_pyc
+
-import os
+from importlib import _bootstrap
 import errno
+import os
 import py_compile
+from test.support import make_legacy_pyc
 import unittest
 import warnings
 

Lib/importlib/test/source/test_path_hook.py

@@ -1,5 +1,6 @@
-from importlib import _bootstrap
 from . import util as source_util
+
+from importlib import _bootstrap
 import unittest
 
 

Lib/importlib/test/source/test_source_encoding.py

@@ -1,6 +1,6 @@
-from importlib import _bootstrap
 from . import util as source_util
 
+from importlib import _bootstrap
 import codecs
 import re
 import sys

Lib/importlib/test/util.py

@@ -35,7 +35,7 @@ def uncache(*names):
     for name in names:
         if name in ('sys', 'marshal', 'imp'):
             raise ValueError(
-                "cannot uncache {0} as it will break _importlib".format(name))
+                "cannot uncache {0}".format(name))
         try:
             del sys.modules[name]
         except KeyError:

Lib/os.py

@@ -42,6 +42,8 @@ def _get_exports_list(module):
     except AttributeError:
         return [n for n in dir(module) if n[0] != '_']
 
+# Any new dependencies of the os module and/or changes in path separator
+# requires updating importlib as well.
 if 'posix' in _names:
     name = 'posix'
     linesep = '\n'

Lib/pydoc.py

@@ -299,9 +299,8 @@ def safeimport(path, forceload=0, cache={}):
         elif exc is SyntaxError:
             # A SyntaxError occurred before we could execute the module.
             raise ErrorDuringImport(value.filename, info)
-        elif exc is ImportError and extract_tb(tb)[-1][2]=='safeimport':
+        elif exc is ImportError and value.name == path:
-            # The import error occurred directly in this function,
+            # No such module in the path.
-            # which means there is no such module in the path.
             return None
         else:
             # Some other error occurred during the importing process.

Lib/site.py

@@ -81,7 +81,8 @@ def makepath(*paths):
 def abs_paths():
     """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in set(sys.modules.values()):
-        if hasattr(m, '__loader__'):
+        if (getattr(getattr(m, '__loader__', None), '__module__', None) !=
+                '_frozen_importlib'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
             m.__file__ = os.path.abspath(m.__file__)

Lib/test/test_frozen.py

@@ -5,6 +5,12 @@ import unittest
 import sys
 
 class FrozenTests(unittest.TestCase):
+
+    module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
+                              '__file__', '__loader__', '__name__',
+                              '__package__'])
+    package_attrs = frozenset(list(module_attrs) + ['__path__'])
+
     def test_frozen(self):
         with captured_stdout() as stdout:
             try:
@@ -12,7 +18,9 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __hello__ failed:" + str(x))
             self.assertEqual(__hello__.initialized, True)
-            self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
+            expect = set(self.module_attrs)
+            expect.add('initialized')
+            self.assertEqual(set(dir(__hello__)), expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         with captured_stdout() as stdout:
@@ -21,10 +29,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__ failed:" + str(x))
             self.assertEqual(__phello__.initialized, True)
+            expect = set(self.package_attrs)
+            expect.add('initialized')
             if not "__phello__.spam" in sys.modules:
-                self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+                self.assertEqual(set(dir(__phello__)), expect)
             else:
-                self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
+                expect.add('spam')
+                self.assertEqual(set(dir(__phello__)), expect)
             self.assertEqual(__phello__.__path__, [__phello__.__name__])
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
@@ -34,8 +45,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__.spam failed:" + str(x))
             self.assertEqual(__phello__.spam.initialized, True)
-            self.assertEqual(len(dir(__phello__.spam)), 7)
+            spam_expect = set(self.module_attrs)
-            self.assertEqual(len(dir(__phello__)), 9)
+            spam_expect.add('initialized')
+            self.assertEqual(set(dir(__phello__.spam)), spam_expect)
+            phello_expect = set(self.package_attrs)
+            phello_expect.add('initialized')
+            phello_expect.add('spam')
+            self.assertEqual(set(dir(__phello__)), phello_expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         try:

Lib/test/test_import.py

@@ -73,6 +73,7 @@ class ImportTests(unittest.TestCase):
 
             if TESTFN in sys.modules:
                 del sys.modules[TESTFN]
+            importlib.invalidate_caches()
             try:
                 try:
                     mod = __import__(TESTFN)
@@ -402,6 +403,7 @@ func_filename = func.__code__.co_filename
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
         pyc_file = make_legacy_pyc(self.file_name)
+        importlib.invalidate_caches()
         mod = self.import_module()
         self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
@@ -509,7 +511,7 @@ class RelativeImportTests(unittest.TestCase):
 
         # Check relative import fails with package set to a non-string
         ns = dict(__package__=object())
-        self.assertRaises(ValueError, check_relative)
+        self.assertRaises(TypeError, check_relative)
 
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
@@ -644,6 +646,7 @@ class PycacheTests(unittest.TestCase):
             pass
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
@@ -666,9 +669,11 @@ class PycacheTests(unittest.TestCase):
             pass
         with open(os.path.join('pep3147', 'foo.py'), 'w'):
             pass
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))

Lib/test/test_pkg.py

@@ -196,14 +196,15 @@ class TestPkg(unittest.TestCase):
 
         import t5
         self.assertEqual(fixdir(dir(t5)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__loader__',
-                          '__package__', '__path__', 'foo', 'string', 't5'])
+                          '__name__', '__package__', '__path__', 'foo',
+                          'string', 't5'])
         self.assertEqual(fixdir(dir(t5.foo)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__loader__',
-                          '__package__', 'string'])
+                          '__name__', '__package__', 'string'])
         self.assertEqual(fixdir(dir(t5.string)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__loader__',
-                          '__package__', 'spam'])
+                          '__name__', '__package__', 'spam'])
 
     def test_6(self):
         hier = [
@@ -219,14 +220,14 @@ class TestPkg(unittest.TestCase):
         import t6
         self.assertEqual(fixdir(dir(t6)),
                          ['__all__', '__cached__', '__doc__', '__file__',
-                          '__name__', '__package__', '__path__'])
+                          '__loader__', '__name__', '__package__', '__path__'])
         s = """
             import t6
             from t6 import *
             self.assertEqual(fixdir(dir(t6)),
                              ['__all__', '__cached__', '__doc__', '__file__',
-                              '__name__', '__package__', '__path__',
+                              '__loader__', '__name__', '__package__',
-                              'eggs', 'ham', 'spam'])
+                              '__path__', 'eggs', 'ham', 'spam'])
             self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
             """
         self.run_code(s)
@@ -252,19 +253,19 @@ class TestPkg(unittest.TestCase):
         t7, sub, subsub = None, None, None
         import t7 as tas
         self.assertEqual(fixdir(dir(tas)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__loader__',
-                          '__package__', '__path__'])
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         from t7 import sub as subpar
         self.assertEqual(fixdir(dir(subpar)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__loader__',
-                          '__package__', '__path__'])
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         from t7.sub import subsub as subsubsub
         self.assertEqual(fixdir(dir(subsubsub)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
+                         ['__cached__', '__doc__', '__file__', '__loader__',
-                         '__package__', '__path__', 'spam'])
+                          '__name__', '__package__', '__path__', 'spam'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         self.assertFalse(subsub)

Lib/test/test_pydoc.py

@@ -383,11 +383,10 @@ class PydocImportTest(unittest.TestCase):
         modname = 'testmod_xyzzy'
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
-            ('test.i_am_not_here_either', 'i_am_not_here_either'),
+            ('test.i_am_not_here_either', 'test.i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
+            ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'),
-            ('i_am_not_here.{}'.format(modname),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
-             'i_am_not_here.{}'.format(modname)),
+            ('test.{}'.format(modname), 'test.{}'.format(modname)),
-            ('test.{}'.format(modname), modname),
             )
 
         sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py"

Lib/test/test_runpy.py

@@ -5,6 +5,7 @@ import os.path
 import sys
 import re
 import tempfile
+import importlib
 import py_compile
 from test.support import (
     forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -172,11 +173,13 @@ class RunModuleTest(unittest.TestCase):
             self.assertIn("x", d1)
             self.assertEqual(d1["x"], 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name) # Read from bytecode
             self.assertIn("x", d2)
             self.assertEqual(d2["x"], 1)
@@ -196,11 +199,13 @@ class RunModuleTest(unittest.TestCase):
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", pkg_name)
+            importlib.invalidate_caches()
             d2 = run_module(pkg_name) # Read from bytecode
             self.assertIn("x", d2)
             self.assertTrue(d2["x"] == 1)
@@ -250,11 +255,13 @@ from ..uncle.cousin import nephew
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case the loader caches paths
             if verbose: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
             self.assertIn("__package__", d2)
             self.assertTrue(d2["__package__"] == pkg_name)

Lib/test/test_support.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import importlib
 import sys
 import os
 import unittest
@@ -59,6 +60,7 @@ class TestSupport(unittest.TestCase):
         with open(mod_filename, 'w') as f:
             print('foo = 1', file=f)
         sys.path.insert(0, os.curdir)
+        importlib.invalidate_caches()
         try:
             mod = __import__(TESTFN)
             self.assertIn(TESTFN, sys.modules)

Lib/test/test_trace.py

@@ -322,7 +322,7 @@ class TestCoverage(unittest.TestCase):
             self._coverage(tracer)
         if os.path.exists(TESTFN):
             files = os.listdir(TESTFN)
-            self.assertEqual(files, [])
+            self.assertEqual(files, ['_importlib.cover'])  # Ignore __import__
 
     def test_issue9936(self):
         tracer = trace.Trace(trace=0, count=1)

Makefile.pre.in

@@ -573,7 +573,12 @@ Modules/Setup: $(srcdir)/Modules/Setup.dist
 
 Modules/_testembed: Modules/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
 	$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Modules/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
+############################################################################
+# Importlib
 
+Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/freeze_importlib.py
+	./$(BUILDPYTHON) $(srcdir)/Python/freeze_importlib.py \
+	    $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
 ############################################################################
 # Special rules for object files
 
@@ -787,6 +792,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/unicodeobject.h \
 		$(srcdir)/Include/warnings.h \
 		$(srcdir)/Include/weakrefobject.h \
+		$(srcdir)/Python/importlib.h \
 		pyconfig.h \
 		$(PARSER_HEADERS)
 

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.3.0 Alpha 3?
 Core and Builtins
 -----------------
 
+- Issue #2377: Make importlib the implementation of __import__().
+
 - Issue #1559549: ImportError now has 'name' and 'path' attributes that are set
   using keyword arguments to its constructor. They are currently not set by
   import as they are meant for use by importlib.

Objects/abstract.c

@@ -2376,6 +2376,35 @@ PyObject_CallMethodObjArgs(PyObject *callable, PyObject *name, ...)
     return tmp;
 }
 
+PyObject *
+_PyObject_CallMethodObjIdArgs(PyObject *callable,
+        struct _Py_Identifier *name, ...)
+{
+    PyObject *args, *tmp;
+    va_list vargs;
+
+    if (callable == NULL || name == NULL)
+        return null_error();
+
+    callable = _PyObject_GetAttrId(callable, name);
+    if (callable == NULL)
+        return NULL;
+
+    /* count the args */
+    va_start(vargs, name);
+    args = objargs_mktuple(vargs);
+    va_end(vargs);
+    if (args == NULL) {
+        Py_DECREF(callable);
+        return NULL;
+    }
+    tmp = PyObject_Call(callable, args, NULL);
+    Py_DECREF(args);
+    Py_DECREF(callable);
+
+    return tmp;
+}
+
 PyObject *
 PyObject_CallFunctionObjArgs(PyObject *callable, ...)
 {

Objects/dictobject.c

@@ -788,6 +788,16 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key)
     return ep->me_value;
 }
 
+PyObject *
+_PyDict_GetItemIdWithError(PyObject *dp, struct _Py_Identifier *key)
+{
+    PyObject *kv;
+    kv = _PyUnicode_FromId(key); /* borrowed */
+    if (kv == NULL)
+        return NULL;
+    return PyDict_GetItemWithError(dp, kv);
+}
+
 static int
 dict_set_item_by_hash_or_entry(register PyObject *op, PyObject *key,
                                Py_hash_t hash, PyDictEntry *ep, PyObject *value)

Objects/exceptions.c

@@ -2344,9 +2344,9 @@ PyObject *PyExc_RecursionErrorInst = NULL;
 #endif /* MS_WINDOWS */
 
 void
-_PyExc_Init(void)
+_PyExc_Init(PyObject *bltinmod)
 {
-    PyObject *bltinmod, *bdict;
+    PyObject *bdict;
 
     PRE_INIT(BaseException)
     PRE_INIT(Exception)
@@ -2414,9 +2414,6 @@ _PyExc_Init(void)
     PRE_INIT(ProcessLookupError);
     PRE_INIT(TimeoutError);
 
-    bltinmod = PyImport_ImportModule("builtins");
-    if (bltinmod == NULL)
-        Py_FatalError("exceptions bootstrapping error.");
     bdict = PyModule_GetDict(bltinmod);
     if (bdict == NULL)
         Py_FatalError("exceptions bootstrapping error.");
@@ -2546,7 +2543,6 @@ _PyExc_Init(void)
             Py_DECREF(args_tuple);
         }
     }
-    Py_DECREF(bltinmod);
 }
 
 void

Python/bltinmodule.c

@@ -187,7 +187,7 @@ builtin___import__(PyObject *self, PyObject *args, PyObject *kwds)
     static char *kwlist[] = {"name", "globals", "locals", "fromlist",
                              "level", 0};
     PyObject *name, *globals = NULL, *locals = NULL, *fromlist = NULL;
-    int level = -1;
+    int level = 0;
 
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|OOOi:__import__",
                     kwlist, &name, &globals, &locals, &fromlist, &level))

Python/dynload_shlib.c

@@ -115,10 +115,6 @@ dl_funcptr _PyImport_GetDynLoadFunc(const char *shortname,
     dlopenflags = PyThreadState_GET()->interp->dlopenflags;
 #endif
 
-    if (Py_VerboseFlag)
-        PySys_WriteStderr("dlopen(\"%s\", %x);\n", pathname,
-                          dlopenflags);
-
 #ifdef __VMS
     /* VMS currently don't allow a pathname, use a logical name instead */
     /* Concatenate 'python_module_' and shortname */

Python/freeze_importlib.py

@@ -0,0 +1,37 @@
+#! /usr/bin/env python
+"""Freeze importlib for use as the implementation of import."""
+import marshal
+
+
+header = """/* Auto-generated by Python/freeze_importlib.py */"""
+
+
+def main(input_path, output_path):
+    with open(input_path, 'r', encoding='utf-8') as input_file:
+        source = input_file.read()
+
+    code = compile(source, '<frozen importlib._bootstrap>', 'exec')
+
+    lines = [header]
+    lines.append('unsigned char _Py_M__importlib[] = {')
+    data = marshal.dumps(code)
+    # Code from Tools/freeze/makefreeze.py:writecode()
+    for i in range(0, len(data), 16):
+        line = ['    ']
+        for c in data[i:i+16]:
+            line.append('%d,' % c)
+        lines.append(''.join(line))
+    lines.append('};\n')
+    with open(output_path, 'w') as output_file:
+        output_file.write('\n'.join(lines))
+
+
+if __name__ == '__main__':
+    import sys
+
+    args = sys.argv[1:]
+    if len(args) != 2:
+        print('Need to specify input and output file paths', file=sys.stderr)
+        sys.exit(1)
+
+    main(*args)

Python/frozen.c

@@ -2,6 +2,7 @@
 /* Dummy frozen modules initializer */
 
 #include "Python.h"
+#include "importlib.h"
 
 /* In order to test the support for frozen modules, by default we
    define a single frozen module, __hello__.  Loading it will print
@@ -28,6 +29,8 @@ static unsigned char M___hello__[] = {
 #define SIZE (int)sizeof(M___hello__)
 
 static struct _frozen _PyImport_FrozenModules[] = {
+    /* importlib */
+    {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)},
     /* Test module */
     {"__hello__", M___hello__, SIZE},
     /* Test package (negative size indicates package-ness) */

Python/import.c

@@ -203,17 +203,13 @@ _PyImport_Init(void)
 void
 _PyImportHooks_Init(void)
 {
-    PyObject *v, *path_hooks = NULL, *zimpimport;
+    PyObject *v, *path_hooks = NULL;
     int err = 0;
 
-    /* adding sys.path_hooks and sys.path_importer_cache, setting up
-       zipimport */
     if (PyType_Ready(&PyNullImporter_Type) < 0)
         goto error;
 
-    if (Py_VerboseFlag)
+    /* adding sys.path_hooks and sys.path_importer_cache */
-        PySys_WriteStderr("# installing zipimport hook\n");
-
     v = PyList_New(0);
     if (v == NULL)
         goto error;
@@ -239,6 +235,21 @@ _PyImportHooks_Init(void)
                   "path_importer_cache, or NullImporter failed"
                   );
     }
+    Py_DECREF(path_hooks);
+}
+
+void
+_PyImportZip_Init(void)
+{
+    PyObject *path_hooks, *zimpimport;
+    int err = 0;
+
+    path_hooks = PySys_GetObject("path_hooks");
+    if (path_hooks == NULL)
+        goto error;
+
+    if (Py_VerboseFlag)
+        PySys_WriteStderr("# installing zipimport hook\n");
 
     zimpimport = PyImport_ImportModule("zipimport");
     if (zimpimport == NULL) {
@@ -261,14 +272,20 @@ _PyImportHooks_Init(void)
             /* sys.path_hooks.append(zipimporter) */
             err = PyList_Append(path_hooks, zipimporter);
             Py_DECREF(zipimporter);
-            if (err)
+            if (err < 0) {
                 goto error;
+            }
             if (Py_VerboseFlag)
                 PySys_WriteStderr(
                     "# installed zipimport hook\n");
         }
     }
-    Py_DECREF(path_hooks);
+
+    return;
+
+  error:
+    PyErr_Print();
+    Py_FatalError("initializing zipimport or NullImporter failed");
 }
 
 /* Locking primitives to prevent parallel imports of the same module
@@ -2542,8 +2559,6 @@ init_builtin(PyObject *name)
                     name);
                 return -1;
             }
-            if (Py_VerboseFlag)
-                PySys_FormatStderr("import %U # builtin\n", name);
             mod = (*p->initfunc)();
             if (mod == 0)
                 return -1;
@@ -2654,9 +2669,6 @@ PyImport_ImportFrozenModuleObject(PyObject *name)
     ispackage = (size < 0);
     if (ispackage)
         size = -size;
-    if (Py_VerboseFlag)
-        PySys_FormatStderr("import %U # frozen%s\n",
-            name, ispackage ? " package" : "");
     co = PyMarshal_ReadObjectFromString((char *)p->code, size);
     if (co == NULL)
         return -1;
@@ -2786,130 +2798,264 @@ PyImport_ImportModuleNoBlock(const char *name)
     return result;
 }
 
-/* Forward declarations for helper routines */
-static PyObject *get_parent(PyObject *globals,
-                            PyObject **p_name,
-                            int level);
-static PyObject *load_next(PyObject *mod, PyObject *altmod,
-                           PyObject *inputname, PyObject **p_outputname,
-                           PyObject **p_prefix);
-static int mark_miss(PyObject *name);
-static int ensure_fromlist(PyObject *mod, PyObject *fromlist,
-                           PyObject *buf, int recursive);
-static PyObject * import_submodule(PyObject *mod, PyObject *name,
-                                   PyObject *fullname);
-
-/* The Magnum Opus of dotted-name import :-) */
-
-static PyObject *
-import_module_level(PyObject *name, PyObject *globals, PyObject *locals,
-                    PyObject *fromlist, int level)
-{
-    PyObject *parent, *next, *inputname, *outputname;
-    PyObject *head = NULL;
-    PyObject *tail = NULL;
-    PyObject *prefix = NULL;
-    PyObject *result = NULL;
-    Py_ssize_t sep, altsep;
-
-    if (PyUnicode_READY(name))
-        return NULL;
-
-    sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1);
-    if (sep == -2)
-        return NULL;
-#ifdef ALTSEP
-    altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1);
-    if (altsep == -2)
-        return NULL;
-#else
-    altsep = -1;
-#endif
-    if (sep != -1 || altsep != -1)
-    {
-        PyErr_SetString(PyExc_ImportError,
-                        "Import by filename is not supported.");
-        return NULL;
-    }
-
-    parent = get_parent(globals, &prefix, level);
-    if (parent == NULL) {
-        return NULL;
-    }
-
-    if (PyUnicode_READY(prefix))
-        return NULL;
-
-    head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname,
-                     &prefix);
-    if (head == NULL)
-        goto out;
-
-    tail = head;
-    Py_INCREF(tail);
-
-    if (outputname != NULL) {
-        while (1) {
-            inputname = outputname;
-            next = load_next(tail, tail, inputname, &outputname,
-                             &prefix);
-            Py_CLEAR(tail);
-            Py_CLEAR(inputname);
-            if (next == NULL)
-                goto out;
-            tail = next;
-
-            if (outputname == NULL) {
-                break;
-            }
-        }
-    }
-    if (tail == Py_None) {
-        /* If tail is Py_None, both get_parent and load_next found
-           an empty module name: someone called __import__("") or
-           doctored faulty bytecode */
-        PyErr_SetString(PyExc_ValueError, "Empty module name");
-        goto out;
-    }
-
-    if (fromlist != NULL) {
-        if (fromlist == Py_None || !PyObject_IsTrue(fromlist))
-            fromlist = NULL;
-    }
-
-    if (fromlist == NULL) {
-        result = head;
-        Py_INCREF(result);
-        goto out;
-    }
-
-    if (!ensure_fromlist(tail, fromlist, prefix, 0))
-        goto out;
-
-    result = tail;
-    Py_INCREF(result);
-  out:
-    Py_XDECREF(head);
-    Py_XDECREF(tail);
-    Py_XDECREF(prefix);
-    return result;
-}
 
 PyObject *
-PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
+PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals,
-                                 PyObject *locals, PyObject *fromlist,
+                                 PyObject *locals, PyObject *given_fromlist,
                                  int level)
 {
-    PyObject *mod;
+    _Py_IDENTIFIER(__import__);
-    _PyImport_AcquireLock();
+    _Py_IDENTIFIER(__package__);
-    mod = import_module_level(name, globals, locals, fromlist, level);
+    _Py_IDENTIFIER(__path__);
-    if (_PyImport_ReleaseLock() < 0) {
+    _Py_IDENTIFIER(__name__);
-        Py_XDECREF(mod);
+    _Py_IDENTIFIER(_find_and_load);
-        PyErr_SetString(PyExc_RuntimeError,
+    _Py_IDENTIFIER(_handle_fromlist);
-                        "not holding the import lock");
+    _Py_static_string(single_dot, ".");
-        return NULL;
+    PyObject *abs_name = NULL;
+    PyObject *builtins_import = NULL;
+    PyObject *final_mod = NULL;
+    PyObject *mod = NULL;
+    PyObject *package = NULL;
+    PyObject *globals = NULL;
+    PyObject *fromlist = NULL;
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
+
+    /* Make sure to use default values so as to not have
+       PyObject_CallMethodObjArgs() truncate the parameter list because of a
+       NULL argument. */
+    if (given_globals == NULL) {
+        globals = PyDict_New();
+        if (globals == NULL) {
+            goto error;
         }
-    return mod;
+    }
+    else {
+        /* Only have to care what given_globals is if it will be used
+           fortsomething. */
+        if (level > 0 && !PyDict_Check(given_globals)) {
+            PyErr_SetString(PyExc_TypeError, "globals must be a dict");
+            goto error;
+        }
+        globals = given_globals;
+        Py_INCREF(globals);
+    }
+
+    if (given_fromlist == NULL) {
+        fromlist = PyList_New(0);
+        if (fromlist == NULL) {
+            goto error;
+        }
+    }
+    else {
+        fromlist = given_fromlist;
+        Py_INCREF(fromlist);
+    }
+    if (name == NULL) {
+        PyErr_SetString(PyExc_ValueError, "Empty module name");
+        goto error;
+    }
+
+    /* The below code is importlib.__import__() & _gcd_import(), ported to C
+       for added performance. */
+
+    if (!PyUnicode_Check(name)) {
+        PyErr_SetString(PyExc_TypeError, "module name must be a string");
+        goto error;
+    }
+    else if (PyUnicode_READY(name) < 0) {
+        goto error;
+    }
+    if (level < 0) {
+        PyErr_SetString(PyExc_ValueError, "level must be >= 0");
+        goto error;
+    }
+    else if (level > 0) {
+        package = _PyDict_GetItemId(globals, &PyId___package__);
+        if (package != NULL && package != Py_None) {
+            Py_INCREF(package);
+            if (!PyUnicode_Check(package)) {
+                PyErr_SetString(PyExc_TypeError, "package must be a string");
+                goto error;
+            }
+        }
+        else {
+            package = _PyDict_GetItemIdWithError(globals, &PyId___name__);
+            if (package == NULL) {
+                goto error;
+            }
+            else if (!PyUnicode_Check(package)) {
+                PyErr_SetString(PyExc_TypeError, "__name__ must be a string");
+            }
+            Py_INCREF(package);
+
+            if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) {
+                PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot);
+                if (borrowed_dot == NULL) {
+                    goto error;
+                }
+                PyObject *partition = PyUnicode_RPartition(package,
+                                                           borrowed_dot);
+                Py_DECREF(package);
+                if (partition == NULL) {
+                    goto error;
+                }
+                package = PyTuple_GET_ITEM(partition, 0);
+                Py_INCREF(package);
+                Py_DECREF(partition);
+            }
+        }
+
+        if (PyDict_GetItem(interp->modules, package) == NULL) {
+            PyErr_Format(PyExc_SystemError,
+                    "Parent module %R not loaded, cannot perform relative "
+                    "import", package);
+            goto error;
+        }
+    }
+    else {  /* level == 0 */
+        if (PyUnicode_GET_LENGTH(name) == 0) {
+            PyErr_SetString(PyExc_ValueError, "Empty module name");
+            goto error;
+        }
+        package = Py_None;
+        Py_INCREF(package);
+    }
+
+    if (level > 0) {
+        Py_ssize_t last_dot = PyUnicode_GET_LENGTH(package);
+        PyObject *base = NULL;
+        int level_up = 1;
+
+        for (level_up = 1; level_up < level; level_up += 1) {
+            last_dot = PyUnicode_FindChar(package, '.', 0, last_dot, -1);
+            if (last_dot == -2) {
+                goto error;
+            }
+            else if (last_dot == -1) {
+                PyErr_SetString(PyExc_ValueError,
+                                "attempted relative import beyond top-level "
+                                "package");
+                goto error;
+            }
+        }
+        base = PyUnicode_Substring(package, 0, last_dot);
+        if (PyUnicode_GET_LENGTH(name) > 0) {
+            PyObject *borrowed_dot = NULL;
+            PyObject *seq = PyTuple_Pack(2, base, name);
+
+            borrowed_dot = _PyUnicode_FromId(&single_dot);
+            if (borrowed_dot == NULL || seq == NULL) {
+                goto error;
+            }
+
+            abs_name = PyUnicode_Join(borrowed_dot, seq);
+            Py_DECREF(seq);
+            if (abs_name == NULL) {
+                goto error;
+            }
+        }
+        else {
+            abs_name = base;
+        }
+    }
+    else {
+        abs_name = name;
+        Py_INCREF(abs_name);
+    }
+
+#if WITH_THREAD
+    _PyImport_AcquireLock();
+#endif
+   /* From this point forward, goto error_with_unlock! */
+    if (PyDict_Check(globals)) {
+        builtins_import = _PyDict_GetItemId(globals, &PyId___import__);
+    }
+    if (builtins_import == NULL) {
+        builtins_import = _PyDict_GetItemId(interp->builtins, &PyId___import__);
+        if (builtins_import == NULL) {
+            Py_FatalError("__import__ missing");
+        }
+    }
+    Py_INCREF(builtins_import);
+
+    mod = PyDict_GetItem(interp->modules, abs_name);
+    if (mod == Py_None) {
+        PyErr_Format(PyExc_ImportError,
+                     "import of %R halted; None in sys.modules", abs_name);
+        goto error_with_unlock;
+    }
+    else if (mod != NULL) {
+        Py_INCREF(mod);
+    }
+    else {
+        mod = _PyObject_CallMethodObjIdArgs(interp->importlib,
+                                            &PyId__find_and_load, abs_name,
+                                            builtins_import, NULL);
+        if (mod == NULL) {
+            goto error_with_unlock;
+        }
+    }
+
+    if (PyObject_Not(fromlist)) {
+        if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) {
+            PyObject *front = NULL;
+            PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot);
+
+            if (borrowed_dot == NULL) {
+                goto error_with_unlock;
+            }
+
+            PyObject *partition = PyUnicode_Partition(name, borrowed_dot);
+            if (partition == NULL) {
+                goto error_with_unlock;
+            }
+
+            front = PyTuple_GET_ITEM(partition, 0);
+            Py_INCREF(front);
+            Py_DECREF(partition);
+
+            if (level == 0) {
+                final_mod = PyDict_GetItemWithError(interp->modules, front);
+                Py_DECREF(front);
+                Py_XINCREF(final_mod);
+            }
+            else {
+                Py_ssize_t cut_off = PyUnicode_GetLength(name) -
+                                        PyUnicode_GetLength(front);
+                Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name);
+                PyObject *to_return = PyUnicode_Substring(name, 0,
+                                                        abs_name_len - cut_off);
+
+                final_mod = PyDict_GetItem(interp->modules, to_return);
+                Py_INCREF(final_mod);
+                Py_DECREF(to_return);
+            }
+        }
+        else {
+            final_mod = mod;
+            Py_INCREF(mod);
+        }
+    }
+    else {
+        final_mod = _PyObject_CallMethodObjIdArgs(interp->importlib,
+                                                  &PyId__handle_fromlist, mod,
+                                                  fromlist, builtins_import,
+                                                  NULL);
+    }
+  error_with_unlock:
+#if WITH_THREAD
+    if (_PyImport_ReleaseLock() < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "not holding the import lock");
+    }
+#endif
+  error:
+    Py_XDECREF(abs_name);
+    Py_XDECREF(builtins_import);
+    Py_XDECREF(mod);
+    Py_XDECREF(package);
+    Py_XDECREF(globals);
+    Py_XDECREF(fromlist);
+    return final_mod;
 }
 
 PyObject *
@@ -2927,430 +3073,6 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals
 }
 
 
-/* Return the package that an import is being performed in.  If globals comes
-   from the module foo.bar.bat (not itself a package), this returns the
-   sys.modules entry for foo.bar.  If globals is from a package's __init__.py,
-   the package's entry in sys.modules is returned, as a borrowed reference.
-
-   The name of the returned package is returned in *p_name.
-
-   If globals doesn't come from a package or a module in a package, or a
-   corresponding entry is not found in sys.modules, Py_None is returned.
-*/
-static PyObject *
-get_parent(PyObject *globals, PyObject **p_name, int level)
-{
-    PyObject *nameobj;
-
-    static PyObject *namestr = NULL;
-    static PyObject *pathstr = NULL;
-    static PyObject *pkgstr = NULL;
-    PyObject *pkgname, *modname, *modpath, *modules, *parent;
-    int orig_level = level;
-
-    if (globals == NULL || !PyDict_Check(globals) || !level)
-        goto return_none;
-
-    if (namestr == NULL) {
-        namestr = PyUnicode_InternFromString("__name__");
-        if (namestr == NULL)
-            return NULL;
-    }
-    if (pathstr == NULL) {
-        pathstr = PyUnicode_InternFromString("__path__");
-        if (pathstr == NULL)
-            return NULL;
-    }
-    if (pkgstr == NULL) {
-        pkgstr = PyUnicode_InternFromString("__package__");
-        if (pkgstr == NULL)
-            return NULL;
-    }
-
-    pkgname = PyDict_GetItem(globals, pkgstr);
-
-    if ((pkgname != NULL) && (pkgname != Py_None)) {
-        /* __package__ is set, so use it */
-        if (!PyUnicode_Check(pkgname)) {
-            PyErr_SetString(PyExc_ValueError,
-                            "__package__ set to non-string");
-            return NULL;
-        }
-        if (PyUnicode_GET_LENGTH(pkgname) == 0) {
-            if (level > 0) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Attempted relative import in non-package");
-                return NULL;
-            }
-            goto return_none;
-        }
-        Py_INCREF(pkgname);
-        nameobj = pkgname;
-    } else {
-        /* __package__ not set, so figure it out and set it */
-        modname = PyDict_GetItem(globals, namestr);
-        if (modname == NULL || !PyUnicode_Check(modname))
-            goto return_none;
-
-        modpath = PyDict_GetItem(globals, pathstr);
-        if (modpath != NULL) {
-            /* __path__ is set, so modname is already the package name */
-            int error;
-
-            error = PyDict_SetItem(globals, pkgstr, modname);
-            if (error) {
-                PyErr_SetString(PyExc_ValueError,
-                                "Could not set __package__");
-                return NULL;
-            }
-            Py_INCREF(modname);
-            nameobj = modname;
-        } else {
-            /* Normal module, so work out the package name if any */
-            Py_ssize_t len;
-            len = PyUnicode_FindChar(modname, '.',
-                                     0, PyUnicode_GET_LENGTH(modname), -1);
-            if (len == -2)
-                return NULL;
-            if (len < 0) {
-                if (level > 0) {
-                    PyErr_SetString(PyExc_ValueError,
-                        "Attempted relative import in non-package");
-                    return NULL;
-                }
-                if (PyDict_SetItem(globals, pkgstr, Py_None)) {
-                    PyErr_SetString(PyExc_ValueError,
-                        "Could not set __package__");
-                    return NULL;
-                }
-                goto return_none;
-            }
-            pkgname = PyUnicode_Substring(modname, 0, len);
-            if (pkgname == NULL)
-                return NULL;
-            if (PyDict_SetItem(globals, pkgstr, pkgname)) {
-                Py_DECREF(pkgname);
-                PyErr_SetString(PyExc_ValueError,
-                                "Could not set __package__");
-                return NULL;
-            }
-            nameobj = pkgname;
-        }
-    }
-    if (level > 1) {
-        Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj);
-        PyObject *newname;
-        while (--level > 0) {
-            dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1);
-            if (dot == -2) {
-                Py_DECREF(nameobj);
-                return NULL;
-            }
-            if (dot < 0) {
-                Py_DECREF(nameobj);
-                PyErr_SetString(PyExc_ValueError,
-                    "Attempted relative import beyond "
-                    "toplevel package");
-                return NULL;
-            }
-            end = dot;
-        }
-        newname = PyUnicode_Substring(nameobj, 0, end);
-        Py_DECREF(nameobj);
-        if (newname == NULL)
-            return NULL;
-        nameobj = newname;
-    }
-
-    modules = PyImport_GetModuleDict();
-    parent = PyDict_GetItem(modules, nameobj);
-    if (parent == NULL) {
-        int err;
-
-        if (orig_level >= 1) {
-            PyErr_Format(PyExc_SystemError,
-                "Parent module %R not loaded, "
-                "cannot perform relative import", nameobj);
-            Py_DECREF(nameobj);
-            return NULL;
-        }
-
-        err = PyErr_WarnFormat(
-            PyExc_RuntimeWarning, 1,
-            "Parent module %R not found while handling absolute import",
-            nameobj);
-        Py_DECREF(nameobj);
-        if (err)
-            return NULL;
-
-        goto return_none;
-    }
-    *p_name = nameobj;
-    return parent;
-    /* We expect, but can't guarantee, if parent != None, that:
-       - parent.__name__ == name
-       - parent.__dict__ is globals
-       If this is violated...  Who cares? */
-
-return_none:
-    nameobj = PyUnicode_New(0, 0);
-    if (nameobj == NULL)
-        return NULL;
-    *p_name = nameobj;
-    return Py_None;
-}
-
-/* altmod is either None or same as mod */
-static PyObject *
-load_next(PyObject *mod, PyObject *altmod,
-          PyObject *inputname, PyObject **p_outputname,
-          PyObject **p_prefix)
-{
-    Py_ssize_t dot;
-    Py_ssize_t len;
-    PyObject *fullname, *name = NULL, *result;
-
-    *p_outputname = NULL;
-
-    len = PyUnicode_GET_LENGTH(inputname);
-    if (len == 0) {
-        /* completely empty module name should only happen in
-           'from . import' (or '__import__("")')*/
-        Py_INCREF(mod);
-        return mod;
-    }
-
-
-    dot = PyUnicode_FindChar(inputname, '.', 0, len, 1);
-    if (dot >= 0) {
-        len = dot;
-        if (len == 0) {
-            PyErr_SetString(PyExc_ValueError,
-                            "Empty module name");
-            goto error;
-        }
-    }
-
-    /* name = inputname[:len] */
-    name = PyUnicode_Substring(inputname, 0, len);
-    if (name == NULL)
-        goto error;
-
-    if (PyUnicode_GET_LENGTH(*p_prefix)) {
-        /* fullname = prefix + "." + name */
-        fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name);
-        if (fullname == NULL)
-            goto error;
-    }
-    else {
-        fullname = name;
-        Py_INCREF(fullname);
-    }
-
-    result = import_submodule(mod, name, fullname);
-    Py_DECREF(*p_prefix);
-    /* Transfer reference. */
-    *p_prefix = fullname;
-    if (result == Py_None && altmod != mod) {
-        Py_DECREF(result);
-        /* Here, altmod must be None and mod must not be None */
-        result = import_submodule(altmod, name, name);
-        if (result != NULL && result != Py_None) {
-            if (mark_miss(*p_prefix) != 0) {
-                Py_DECREF(result);
-                goto error;
-            }
-            Py_DECREF(*p_prefix);
-            *p_prefix = name;
-            Py_INCREF(*p_prefix);
-        }
-    }
-    if (result == NULL)
-        goto error;
-
-    if (result == Py_None) {
-        Py_DECREF(result);
-        PyErr_Format(PyExc_ImportError,
-                     "No module named %R", inputname);
-        goto error;
-    }
-
-    if (dot >= 0) {
-        *p_outputname = PyUnicode_Substring(inputname, dot+1,
-                                            PyUnicode_GET_LENGTH(inputname));
-        if (*p_outputname == NULL) {
-            Py_DECREF(result);
-            goto error;
-        }
-    }
-
-    Py_DECREF(name);
-    return result;
-
-error:
-    Py_XDECREF(name);
-    return NULL;
-}
-
-static int
-mark_miss(PyObject *name)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    return PyDict_SetItem(modules, name, Py_None);
-}
-
-static int
-ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name,
-                int recursive)
-{
-    int i;
-    PyObject *fullname;
-    Py_ssize_t fromlist_len;
-
-    if (!_PyObject_HasAttrId(mod, &PyId___path__))
-        return 1;
-
-    fromlist_len = PySequence_Size(fromlist);
-
-    for (i = 0; i < fromlist_len; i++) {
-        PyObject *item = PySequence_GetItem(fromlist, i);
-        int hasit;
-        if (item == NULL)
-            return 0;
-        if (!PyUnicode_Check(item)) {
-            PyErr_SetString(PyExc_TypeError,
-                            "Item in ``from list'' not a string");
-            Py_DECREF(item);
-            return 0;
-        }
-        if (PyUnicode_READ_CHAR(item, 0) == '*') {
-            PyObject *all;
-            _Py_IDENTIFIER(__all__);
-            Py_DECREF(item);
-            /* See if the package defines __all__ */
-            if (recursive)
-                continue; /* Avoid endless recursion */
-            all = _PyObject_GetAttrId(mod, &PyId___all__);
-            if (all == NULL)
-                PyErr_Clear();
-            else {
-                int ret = ensure_fromlist(mod, all, name, 1);
-                Py_DECREF(all);
-                if (!ret)
-                    return 0;
-            }
-            continue;
-        }
-        hasit = PyObject_HasAttr(mod, item);
-        if (!hasit) {
-            PyObject *submod;
-            fullname = PyUnicode_FromFormat("%U.%U", name, item);
-            if (fullname != NULL) {
-                submod = import_submodule(mod, item, fullname);
-                Py_DECREF(fullname);
-            }
-            else
-                submod = NULL;
-            Py_XDECREF(submod);
-            if (submod == NULL) {
-                Py_DECREF(item);
-                return 0;
-            }
-        }
-        Py_DECREF(item);
-    }
-
-    return 1;
-}
-
-static int
-add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname,
-              PyObject *subname, PyObject *modules)
-{
-    if (mod == Py_None)
-        return 1;
-    /* Irrespective of the success of this load, make a
-       reference to it in the parent package module.  A copy gets
-       saved in the modules dictionary under the full name, so get a
-       reference from there, if need be.  (The exception is when the
-       load failed with a SyntaxError -- then there's no trace in
-       sys.modules.  In that case, of course, do nothing extra.) */
-    if (submod == NULL) {
-        submod = PyDict_GetItem(modules, fullname);
-        if (submod == NULL)
-            return 1;
-    }
-    if (PyModule_Check(mod)) {
-        /* We can't use setattr here since it can give a
-         * spurious warning if the submodule name shadows a
-         * builtin name */
-        PyObject *dict = PyModule_GetDict(mod);
-        if (!dict)
-            return 0;
-        if (PyDict_SetItem(dict, subname, submod) < 0)
-            return 0;
-    }
-    else {
-        if (PyObject_SetAttr(mod, subname, submod) < 0)
-            return 0;
-    }
-    return 1;
-}
-
-static PyObject *
-import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    PyObject *m = NULL, *bufobj, *path_list, *loader;
-    struct filedescr *fdp;
-    FILE *fp;
-
-    /* Require:
-       if mod == None: subname == fullname
-       else: mod.__name__ + "." + subname == fullname
-    */
-
-    if ((m = PyDict_GetItem(modules, fullname)) != NULL) {
-        Py_INCREF(m);
-        return m;
-    }
-
-    if (mod == Py_None)
-        path_list = NULL;
-    else {
-        path_list = _PyObject_GetAttrId(mod, &PyId___path__);
-        if (path_list == NULL) {
-            PyErr_Clear();
-            Py_INCREF(Py_None);
-            return Py_None;
-        }
-    }
-
-    fdp = find_module(fullname, subname, path_list,
-                      &bufobj, &fp, &loader);
-    Py_XDECREF(path_list);
-    if (fdp == NULL) {
-        if (!PyErr_ExceptionMatches(PyExc_ImportError))
-            return NULL;
-        PyErr_Clear();
-        Py_INCREF(Py_None);
-        return Py_None;
-    }
-    m = load_module(fullname, fp, bufobj, fdp->type, loader);
-    Py_XDECREF(bufobj);
-    Py_XDECREF(loader);
-    if (fp)
-        fclose(fp);
-    if (m == NULL)
-        return NULL;
-    if (!add_submodule(mod, m, fullname, subname, modules)) {
-        Py_XDECREF(m);
-        return NULL;
-    }
-    return m;
-}
-
-
 /* Re-import a module of any kind and return its module object, WITH
    INCREMENTED REFERENCE COUNT */
 

Python/importdl.c

@@ -106,10 +106,6 @@ _PyImport_LoadDynamicModule(PyObject *name, PyObject *path, FILE *fp)
 
     if (_PyImport_FixupExtensionObject(m, name, path) < 0)
         goto error;
-    if (Py_VerboseFlag)
-        PySys_FormatStderr(
-            "import %U # dynamically loaded from %R\n",
-            name, path);
     Py_DECREF(nameascii);
     return m;
 

Python/pystate.c

@@ -79,6 +79,7 @@ PyInterpreterState_New(void)
         interp->codec_error_registry = NULL;
         interp->codecs_initialized = 0;
         interp->fscodec_initialized = 0;
+        interp->importlib = NULL;
 #ifdef HAVE_DLOPEN
 #ifdef RTLD_NOW
         interp->dlopenflags = RTLD_NOW;
@@ -116,6 +117,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
     Py_CLEAR(interp->modules_reloading);
     Py_CLEAR(interp->sysdict);
     Py_CLEAR(interp->builtins);
+    Py_CLEAR(interp->importlib);
 }
 
 

Python/pythonrun.c

@@ -190,6 +190,58 @@ get_locale_encoding(void)
 #endif
 }
 
+static void
+import_init(PyInterpreterState *interp, PyObject *sysmod)
+{
+    PyObject *importlib;
+    PyObject *impmod;
+    PyObject *sys_modules;
+    PyObject *value;
+
+    /* Import _importlib through its frozen version, _frozen_importlib. */
+    /* XXX(bcannon): The file path for _frozen_importlib is completely off
+     */
+    if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) {
+        Py_FatalError("Py_Initialize: can't import _frozen_importlib");
+    }
+    else if (Py_VerboseFlag) {
+        PySys_FormatStderr("import _frozen_importlib # frozen\n");
+    }
+    importlib = PyImport_AddModule("_frozen_importlib");
+    if (importlib == NULL) {
+        Py_FatalError("Py_Initialize: couldn't get _frozen_importlib from "
+                      "sys.modules");
+    }
+    interp->importlib = importlib;
+    Py_INCREF(interp->importlib);
+
+    /* Install _importlib as __import__ */
+    impmod = PyInit_imp();
+    if (impmod == NULL) {
+        Py_FatalError("Py_Initialize: can't import imp");
+    }
+    else if (Py_VerboseFlag) {
+        PySys_FormatStderr("import imp # builtin\n");
+    }
+    sys_modules = PyImport_GetModuleDict();
+    if (Py_VerboseFlag) {
+        PySys_FormatStderr("import sys # builtin\n");
+    }
+    if (PyDict_SetItemString(sys_modules, "imp", impmod) < 0) {
+        Py_FatalError("Py_Initialize: can't save imp to sys.modules");
+    }
+
+    value = PyObject_CallMethod(importlib, "_setup", "OO", sysmod, impmod);
+    if (value == NULL) {
+        PyErr_Print();
+        Py_FatalError("Py_Initialize: importlib install failed");
+    }
+    Py_DECREF(value);
+
+    _PyImportZip_Init();
+}
+
+
 void
 Py_InitializeEx(int install_sigs)
 {
@@ -281,7 +333,7 @@ Py_InitializeEx(int install_sigs)
     Py_INCREF(interp->builtins);
 
     /* initialize builtin exceptions */
-    _PyExc_Init();
+    _PyExc_Init(bimod);
 
     sysmod = _PySys_Init();
     if (sysmod == NULL)
@@ -315,6 +367,8 @@ Py_InitializeEx(int install_sigs)
     /* Initialize _warnings. */
     _PyWarnings_Init();
 
+    import_init(interp, sysmod);
+
     _PyTime_Init();
 
     if (initfsencoding(interp) < 0)
@@ -638,11 +692,12 @@ Py_NewInterpreter(void)
     }
 
     /* initialize builtin exceptions */
-    _PyExc_Init();
+    _PyExc_Init(bimod);
 
     sysmod = _PyImport_FindBuiltin("sys");
     if (bimod != NULL && sysmod != NULL) {
         PyObject *pstderr;
+
         interp->sysdict = PyModule_GetDict(sysmod);
         if (interp->sysdict == NULL)
             goto handle_error;
@@ -661,6 +716,8 @@ Py_NewInterpreter(void)
 
         _PyImportHooks_Init();
 
+        import_init(interp, sysmod);
+
         if (initfsencoding(interp) < 0)
             goto handle_error;