bpo-43428: Sync with importlib_metadata 3.7. (GH-24782)

* bpo-43428: Sync with importlib_metadata 3.7.2 (67234b6)

* Add blurb

* Reformat blurb to create separate paragraphs for each change included.

Doc/library/importlib.metadata.rst

@@ -74,18 +74,20 @@ This package provides the following functionality via its public API.
 Entry points
 ------------
 
-The ``entry_points()`` function returns a dictionary of all entry points,
+The ``entry_points()`` function returns a collection of entry points.
-keyed by group.  Entry points are represented by ``EntryPoint`` instances;
+Entry points are represented by ``EntryPoint`` instances;
 each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and
 a ``.load()`` method to resolve the value.  There are also ``.module``,
 ``.attr``, and ``.extras`` attributes for getting the components of the
 ``.value`` attribute::
 
     >>> eps = entry_points()  # doctest: +SKIP
-    >>> list(eps)  # doctest: +SKIP
+    >>> sorted(eps.groups)  # doctest: +SKIP
     ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation']
-    >>> scripts = eps['console_scripts']  # doctest: +SKIP
+    >>> scripts = eps.select(group='console_scripts')  # doctest: +SKIP
-    >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0]  # doctest: +SKIP
+    >>> 'wheel' in scripts.names  # doctest: +SKIP
+    True
+    >>> wheel = scripts['wheel']  # doctest: +SKIP
     >>> wheel  # doctest: +SKIP
     EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')
     >>> wheel.module  # doctest: +SKIP
@@ -187,6 +189,17 @@ function::
     ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"]
 
 
+Package distributions
+---------------------
+
+A convience method to resolve the distribution or
+distributions (in the case of a namespace package) for top-level
+Python packages or modules::
+
+    >>> packages_distributions()
+    {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...}
+
+
 Distributions
 =============
 

Lib/importlib/_itertools.py

@@ -0,0 +1,19 @@
+from itertools import filterfalse
+
+
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element

Lib/importlib/metadata.py

@@ -4,20 +4,24 @@ import abc
 import csv
 import sys
 import email
+import inspect
 import pathlib
 import zipfile
 import operator
+import warnings
 import functools
 import itertools
 import posixpath
-import collections
+import collections.abc
+
+from ._itertools import unique_everseen
 
 from configparser import ConfigParser
 from contextlib import suppress
 from importlib import import_module
 from importlib.abc import MetaPathFinder
 from itertools import starmap
-from typing import Any, List, Optional, Protocol, TypeVar, Union
+from typing import Any, List, Mapping, Optional, Protocol, TypeVar, Union
 
 
 __all__ = [
@@ -120,18 +124,19 @@ class EntryPoint(
         config.read_string(text)
         return cls._from_config(config)
 
-    @classmethod
-    def _from_text_for(cls, text, dist):
-        return (ep._for(dist) for ep in cls._from_text(text))
-
     def _for(self, dist):
         self.dist = dist
         return self
 
     def __iter__(self):
         """
-        Supply iter so one may construct dicts of EntryPoints easily.
+        Supply iter so one may construct dicts of EntryPoints by name.
         """
+        msg = (
+            "Construction of dict of EntryPoints is deprecated in "
+            "favor of EntryPoints."
+        )
+        warnings.warn(msg, DeprecationWarning)
         return iter((self.name, self))
 
     def __reduce__(self):
@@ -140,6 +145,143 @@ class EntryPoint(
             (self.name, self.value, self.group),
         )
 
+    def matches(self, **params):
+        attrs = (getattr(self, param) for param in params)
+        return all(map(operator.eq, params.values(), attrs))
+
+
+class EntryPoints(tuple):
+    """
+    An immutable collection of selectable EntryPoint objects.
+    """
+
+    __slots__ = ()
+
+    def __getitem__(self, name):  # -> EntryPoint:
+        try:
+            return next(iter(self.select(name=name)))
+        except StopIteration:
+            raise KeyError(name)
+
+    def select(self, **params):
+        return EntryPoints(ep for ep in self if ep.matches(**params))
+
+    @property
+    def names(self):
+        return set(ep.name for ep in self)
+
+    @property
+    def groups(self):
+        """
+        For coverage while SelectableGroups is present.
+        >>> EntryPoints().groups
+        set()
+        """
+        return set(ep.group for ep in self)
+
+    @classmethod
+    def _from_text_for(cls, text, dist):
+        return cls(ep._for(dist) for ep in EntryPoint._from_text(text))
+
+
+def flake8_bypass(func):
+    is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5])
+    return func if not is_flake8 else lambda: None
+
+
+class Deprecated:
+    """
+    Compatibility add-in for mapping to indicate that
+    mapping behavior is deprecated.
+
+    >>> recwarn = getfixture('recwarn')
+    >>> class DeprecatedDict(Deprecated, dict): pass
+    >>> dd = DeprecatedDict(foo='bar')
+    >>> dd.get('baz', None)
+    >>> dd['foo']
+    'bar'
+    >>> list(dd)
+    ['foo']
+    >>> list(dd.keys())
+    ['foo']
+    >>> 'foo' in dd
+    True
+    >>> list(dd.values())
+    ['bar']
+    >>> len(recwarn)
+    1
+    """
+
+    _warn = functools.partial(
+        warnings.warn,
+        "SelectableGroups dict interface is deprecated. Use select.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    def __getitem__(self, name):
+        self._warn()
+        return super().__getitem__(name)
+
+    def get(self, name, default=None):
+        flake8_bypass(self._warn)()
+        return super().get(name, default)
+
+    def __iter__(self):
+        self._warn()
+        return super().__iter__()
+
+    def __contains__(self, *args):
+        self._warn()
+        return super().__contains__(*args)
+
+    def keys(self):
+        self._warn()
+        return super().keys()
+
+    def values(self):
+        self._warn()
+        return super().values()
+
+
+class SelectableGroups(dict):
+    """
+    A backward- and forward-compatible result from
+    entry_points that fully implements the dict interface.
+    """
+
+    @classmethod
+    def load(cls, eps):
+        by_group = operator.attrgetter('group')
+        ordered = sorted(eps, key=by_group)
+        grouped = itertools.groupby(ordered, by_group)
+        return cls((group, EntryPoints(eps)) for group, eps in grouped)
+
+    @property
+    def _all(self):
+        """
+        Reconstruct a list of all entrypoints from the groups.
+        """
+        return EntryPoints(itertools.chain.from_iterable(self.values()))
+
+    @property
+    def groups(self):
+        return self._all.groups
+
+    @property
+    def names(self):
+        """
+        for coverage:
+        >>> SelectableGroups().names
+        set()
+        """
+        return self._all.names
+
+    def select(self, **params):
+        if not params:
+            return self
+        return self._all.select(**params)
+
 
 class PackagePath(pathlib.PurePosixPath):
     """A reference to a path in a package"""
@@ -296,7 +438,7 @@ class Distribution:
 
     @property
     def entry_points(self):
-        return list(EntryPoint._from_text_for(self.read_text('entry_points.txt'), self))
+        return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self)
 
     @property
     def files(self):
@@ -485,15 +627,22 @@ class Prepared:
     """
 
     normalized = None
-    suffixes = '.dist-info', '.egg-info'
+    suffixes = 'dist-info', 'egg-info'
     exact_matches = [''][:0]
+    egg_prefix = ''
+    versionless_egg_name = ''
 
     def __init__(self, name):
         self.name = name
         if name is None:
             return
         self.normalized = self.normalize(name)
-        self.exact_matches = [self.normalized + suffix for suffix in self.suffixes]
+        self.exact_matches = [
+            self.normalized + '.' + suffix for suffix in self.suffixes
+        ]
+        legacy_normalized = self.legacy_normalize(self.name)
+        self.egg_prefix = legacy_normalized + '-'
+        self.versionless_egg_name = legacy_normalized + '.egg'
 
     @staticmethod
     def normalize(name):
@@ -512,8 +661,9 @@ class Prepared:
 
     def matches(self, cand, base):
         low = cand.lower()
-        pre, ext = os.path.splitext(low)
+        # rpartition is faster than splitext and suitable for this purpose.
-        name, sep, rest = pre.partition('-')
+        pre, _, ext = low.rpartition('.')
+        name, _, rest = pre.partition('-')
         return (
             low in self.exact_matches
             or ext in self.suffixes
@@ -524,12 +674,9 @@ class Prepared:
         )
 
     def is_egg(self, base):
-        normalized = self.legacy_normalize(self.name or '')
-        prefix = normalized + '-' if normalized else ''
-        versionless_egg_name = normalized + '.egg' if self.name else ''
         return (
-            base == versionless_egg_name
+            base == self.versionless_egg_name
-            or base.startswith(prefix)
+            or base.startswith(self.egg_prefix)
             and base.endswith('.egg')
         )
 
@@ -551,8 +698,9 @@ class MetadataPathFinder(DistributionFinder):
     @classmethod
     def _search_paths(cls, name, paths):
         """Find metadata directories in paths heuristically."""
+        prepared = Prepared(name)
         return itertools.chain.from_iterable(
-            path.search(Prepared(name)) for path in map(FastPath, paths)
+            path.search(prepared) for path in map(FastPath, paths)
         )
 
 
@@ -617,16 +765,28 @@ def version(distribution_name):
     return distribution(distribution_name).version
 
 
-def entry_points():
+def entry_points(**params) -> Union[EntryPoints, SelectableGroups]:
     """Return EntryPoint objects for all installed packages.
 
-    :return: EntryPoint objects for all installed packages.
+    Pass selection parameters (group or name) to filter the
+    result to entry points matching those properties (see
+    EntryPoints.select()).
+
+    For compatibility, returns ``SelectableGroups`` object unless
+    selection parameters are supplied. In the future, this function
+    will return ``EntryPoints`` instead of ``SelectableGroups``
+    even when no selection parameters are supplied.
+
+    For maximum future compatibility, pass selection parameters
+    or invoke ``.select`` with parameters on the result.
+
+    :return: EntryPoints or SelectableGroups for all installed packages.
     """
-    eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions())
+    unique = functools.partial(unique_everseen, key=operator.attrgetter('name'))
-    by_group = operator.attrgetter('group')
+    eps = itertools.chain.from_iterable(
-    ordered = sorted(eps, key=by_group)
+        dist.entry_points for dist in unique(distributions())
-    grouped = itertools.groupby(ordered, by_group)
+    )
-    return {group: tuple(eps) for group, eps in grouped}
+    return SelectableGroups.load(eps).select(**params)
 
 
 def files(distribution_name):
@@ -646,3 +806,19 @@ def requires(distribution_name):
     packaging.requirement.Requirement.
     """
     return distribution(distribution_name).requires
+
+
+def packages_distributions() -> Mapping[str, List[str]]:
+    """
+    Return a mapping of top-level packages to their
+    distributions.
+
+    >>> pkgs = packages_distributions()
+    >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values())
+    True
+    """
+    pkg_to_dist = collections.defaultdict(list)
+    for dist in distributions():
+        for pkg in (dist.read_text('top_level.txt') or '').split():
+            pkg_to_dist[pkg].append(dist.metadata['Name'])
+    return dict(pkg_to_dist)

Lib/test/test_importlib/fixtures.py

@@ -5,7 +5,6 @@ import pathlib
 import tempfile
 import textwrap
 import contextlib
-import unittest
 
 from test.support.os_helper import FS_NONASCII
 from typing import Dict, Union
@@ -221,7 +220,6 @@ class LocalPackage:
         build_files(self.files)
 
 
-
 def build_files(file_defs, prefix=pathlib.Path()):
     """Build a set of files/directories, as described by the
 
@@ -260,9 +258,6 @@ class FileBuilder:
     def unicode_filename(self):
         return FS_NONASCII or self.skip("File system does not support non-ascii.")
 
-    def skip(self, reason):
-        raise unittest.SkipTest(reason)
-
 
 def DALS(str):
     "Dedent and left-strip"

Lib/test/test_importlib/test_main.py

@@ -3,6 +3,7 @@ import json
 import pickle
 import textwrap
 import unittest
+import warnings
 import importlib.metadata
 
 try:
@@ -58,13 +59,11 @@ class ImportTests(fixtures.DistInfoPkg, unittest.TestCase):
             importlib.import_module('does_not_exist')
 
     def test_resolve(self):
-        entries = dict(entry_points()['entries'])
+        ep = entry_points(group='entries')['main']
-        ep = entries['main']
         self.assertEqual(ep.load().__name__, "main")
 
     def test_entrypoint_with_colon_in_name(self):
-        entries = dict(entry_points()['entries'])
+        ep = entry_points(group='entries')['ns:sub']
-        ep = entries['ns:sub']
         self.assertEqual(ep.value, 'mod:main')
 
     def test_resolve_without_attr(self):
@@ -250,7 +249,8 @@ class TestEntryPoints(unittest.TestCase):
         json should not expect to be able to dump an EntryPoint
         """
         with self.assertRaises(Exception):
-            json.dumps(self.ep)
+            with warnings.catch_warnings(record=True):
+                json.dumps(self.ep)
 
     def test_module(self):
         assert self.ep.module == 'value'

Lib/test/test_importlib/test_metadata_api.py

@@ -1,6 +1,7 @@
 import re
 import textwrap
 import unittest
+import warnings
 
 from . import fixtures
 from importlib.metadata import (
@@ -64,18 +65,97 @@ class APITests(
         self.assertEqual(top_level.read_text(), 'mod\n')
 
     def test_entry_points(self):
-        entries = dict(entry_points()['entries'])
+        eps = entry_points()
+        assert 'entries' in eps.groups
+        entries = eps.select(group='entries')
+        assert 'main' in entries.names
         ep = entries['main']
         self.assertEqual(ep.value, 'mod:main')
         self.assertEqual(ep.extras, [])
 
     def test_entry_points_distribution(self):
-        entries = dict(entry_points()['entries'])
+        entries = entry_points(group='entries')
         for entry in ("main", "ns:sub"):
             ep = entries[entry]
             self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg'))
             self.assertEqual(ep.dist.version, "1.0.0")
 
+    def test_entry_points_unique_packages(self):
+        """
+        Entry points should only be exposed for the first package
+        on sys.path with a given name.
+        """
+        alt_site_dir = self.fixtures.enter_context(fixtures.tempdir())
+        self.fixtures.enter_context(self.add_sys_path(alt_site_dir))
+        alt_pkg = {
+            "distinfo_pkg-1.1.0.dist-info": {
+                "METADATA": """
+                Name: distinfo-pkg
+                Version: 1.1.0
+                """,
+                "entry_points.txt": """
+                [entries]
+                main = mod:altmain
+            """,
+            },
+        }
+        fixtures.build_files(alt_pkg, alt_site_dir)
+        entries = entry_points(group='entries')
+        assert not any(
+            ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0'
+            for ep in entries
+        )
+        # ns:sub doesn't exist in alt_pkg
+        assert 'ns:sub' not in entries
+
+    def test_entry_points_missing_name(self):
+        with self.assertRaises(KeyError):
+            entry_points(group='entries')['missing']
+
+    def test_entry_points_missing_group(self):
+        assert entry_points(group='missing') == ()
+
+    def test_entry_points_dict_construction(self):
+        """
+        Prior versions of entry_points() returned simple lists and
+        allowed casting those lists into maps by name using ``dict()``.
+        Capture this now deprecated use-case.
+        """
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.filterwarnings("default", category=DeprecationWarning)
+            eps = dict(entry_points(group='entries'))
+
+        assert 'main' in eps
+        assert eps['main'] == entry_points(group='entries')['main']
+
+        # check warning
+        expected = next(iter(caught))
+        assert expected.category is DeprecationWarning
+        assert "Construction of dict of EntryPoints is deprecated" in str(expected)
+
+    def test_entry_points_groups_getitem(self):
+        """
+        Prior versions of entry_points() returned a dict. Ensure
+        that callers using '.__getitem__()' are supported but warned to
+        migrate.
+        """
+        with warnings.catch_warnings(record=True):
+            entry_points()['entries'] == entry_points(group='entries')
+
+            with self.assertRaises(KeyError):
+                entry_points()['missing']
+
+    def test_entry_points_groups_get(self):
+        """
+        Prior versions of entry_points() returned a dict. Ensure
+        that callers using '.get()' are supported but warned to
+        migrate.
+        """
+        with warnings.catch_warnings(record=True):
+            entry_points().get('missing', 'default') == 'default'
+            entry_points().get('entries', 'default') == entry_points()['entries']
+            entry_points().get('missing', ()) == ()
+
     def test_metadata_for_this_package(self):
         md = metadata('egginfo-pkg')
         assert md['author'] == 'Steven Ma'

Lib/test/test_importlib/test_zip.py

@@ -41,7 +41,7 @@ class TestZip(unittest.TestCase):
             version('definitely-not-installed')
 
     def test_zip_entry_points(self):
-        scripts = dict(entry_points()['console_scripts'])
+        scripts = entry_points(group='console_scripts')
         entry_point = scripts['example']
         self.assertEqual(entry_point.value, 'example:main')
         entry_point = scripts['Example']

Misc/NEWS.d/next/Library/2021-03-07-18-54-39.bpo-43428.br0XmX.rst

@@ -0,0 +1,17 @@
+Include changes from `importlib_metadata 3.7
+<https://importlib-metadata.readthedocs.io/en/latest/history.html#v3-7-0>`_:
+
+Performance enhancements to distribution discovery.
+
+``entry_points`` only returns unique distributions.
+
+Introduces new ``EntryPoints`` object
+for containing a set of entry points with convenience methods for selecting
+entry points by group or name.  ``entry_points`` now returns this object if
+selection parameters are supplied but continues to return a dict object for
+compatibility. Users are encouraged to rely on the selection interface. The
+dict object result is likely to be deprecated in the future.
+
+Added
+packages_distributions function to return a mapping of packages to the
+distributions that provide them.