[3.7] bpo-33169: Remove values of None from sys.path_importer_cache when invalidating caches (GH-6402) (GH-6403)

An entry of None in sys.path_importer_cache represents a negative/missing finder for a path, so clearing it out makes sense.
(cherry picked from commit 9e2be60634)

Doc/library/importlib.rst

@@ -1081,7 +1081,12 @@ find and load modules.
    .. classmethod:: invalidate_caches()
 
       Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all
-      finders stored in :attr:`sys.path_importer_cache`.
+      finders stored in :data:`sys.path_importer_cache` that define the method.
+      Otherwise entries in :data:`sys.path_importer_cache` set to ``None`` are
+      deleted.
+
+      .. versionchanged:: 3.7
+         Entries of ``None`` in :data:`sys.path_importer_cache` are deleted.
 
    .. versionchanged:: 3.4
       Calls objects in :data:`sys.path_hooks` with the current working

Doc/whatsnew/3.7.rst

@@ -561,7 +561,7 @@ importlib
 ---------
 
 The :class:`importlib.abc.ResourceReader` ABC was introduced to
-support the loading of resource from packages.
+support the loading of resources from packages.
 
 locale
 ------
@@ -1274,6 +1274,11 @@ Changes in the Python API
   previous behaviour, or use
   :attr:`STARTUPINFO.lpAttributeList <subprocess.STARTUPINFO.lpAttributeList>`.
 
+* :meth:`importlib.machinery.PathFinder.invalidate_caches` -- which implicitly
+  affects :func:`importlib.invalidate_caches` -- now deletes entries
+  in :data:`sys.path_importer_cache` which are set to ``None``.
+  (Contributed by Brett Cannon in :issue:`33169`.)
+
 
 Changes in the C API
 --------------------

Lib/importlib/_bootstrap_external.py

@@ -1179,8 +1179,10 @@ class PathFinder:
     def invalidate_caches(cls):
         """Call the invalidate_caches() method on all path entry finders
         stored in sys.path_importer_caches (where implemented)."""
-        for finder in sys.path_importer_cache.values():
+        for name, finder in list(sys.path_importer_cache.items()):
-            if hasattr(finder, 'invalidate_caches'):
+            if finder is None:
+                del sys.path_importer_cache[name]
+            elif hasattr(finder, 'invalidate_caches'):
                 finder.invalidate_caches()
 
     @classmethod

Lib/test/test_importlib/import_/test_path.py

@@ -184,6 +184,27 @@ class FinderTests:
             # Do not want FileNotFoundError raised.
             self.assertIsNone(self.machinery.PathFinder.find_spec('whatever'))
 
+    def test_invalidate_caches_finders(self):
+        # Finders with an invalidate_caches() method have it called.
+        class FakeFinder:
+            def __init__(self):
+                self.called = False
+
+            def invalidate_caches(self):
+                self.called = True
+
+        cache = {'leave_alone': object(), 'finder_to_invalidate': FakeFinder()}
+        with util.import_state(path_importer_cache=cache):
+            self.machinery.PathFinder.invalidate_caches()
+        self.assertTrue(cache['finder_to_invalidate'].called)
+
+    def test_invalidate_caches_clear_out_None(self):
+        # Clear out None in sys.path_importer_cache() when invalidating caches.
+        cache = {'clear_out': None}
+        with util.import_state(path_importer_cache=cache):
+            self.machinery.PathFinder.invalidate_caches()
+        self.assertEqual(len(cache), 0)
+
 
 class FindModuleTests(FinderTests):
     def find(self, *args, **kwargs):

Lib/test/test_importlib/test_api.py

@@ -406,7 +406,7 @@ class InvalidateCacheTests:
         # There should be no issues if the method is not defined.
         key = 'gobbledeegook'
         sys.path_importer_cache[key] = None
-        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
+        self.addCleanup(lambda: sys.path_importer_cache.pop(key, None))
         self.init.invalidate_caches()  # Shouldn't trigger an exception.
 
 

Misc/NEWS.d/next/Library/2018-04-06-14-56-26.bpo-33169.ByhDqb.rst

@@ -0,0 +1,2 @@
+Delete entries of ``None`` in :data:`sys.path_importer_cache` when
+:meth:`importlib.machinery.invalidate_caches` is called.