From a1d873b7d66a66c76ee24dfae5626a0274233039 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 25 Mar 2019 22:23:55 -0400 Subject: [PATCH] Commit importlib.metadata as found at https://gitlab.com/python-devs/importlib_metadata/commit/6f8bd81d --- Doc/library/importlib_metadata.rst | 256 +++++++++++++ Lib/importlib/metadata/__init__.py | 17 + Lib/importlib/metadata/_hooks.py | 138 +++++++ Lib/importlib/metadata/abc.py | 21 ++ Lib/importlib/metadata/api.py | 353 ++++++++++++++++++ Lib/importlib/metadata/zipp.py | 110 ++++++ Lib/test/test_importlib/data/__init__.py | 0 .../data/example-21.12-py3-none-any.whl | Bin 0 -> 1453 bytes Lib/test/test_importlib/fixtures.py | 175 +++++++++ Lib/test/test_importlib/test_main.py | 157 ++++++++ Lib/test/test_importlib/test_metadata_api.py | 134 +++++++ Lib/test/test_importlib/test_zip.py | 48 +++ 12 files changed, 1409 insertions(+) create mode 100644 Doc/library/importlib_metadata.rst create mode 100644 Lib/importlib/metadata/__init__.py create mode 100644 Lib/importlib/metadata/_hooks.py create mode 100644 Lib/importlib/metadata/abc.py create mode 100644 Lib/importlib/metadata/api.py create mode 100644 Lib/importlib/metadata/zipp.py create mode 100644 Lib/test/test_importlib/data/__init__.py create mode 100644 Lib/test/test_importlib/data/example-21.12-py3-none-any.whl create mode 100644 Lib/test/test_importlib/fixtures.py create mode 100644 Lib/test/test_importlib/test_main.py create mode 100644 Lib/test/test_importlib/test_metadata_api.py create mode 100644 Lib/test/test_importlib/test_zip.py diff --git a/Doc/library/importlib_metadata.rst b/Doc/library/importlib_metadata.rst new file mode 100644 index 00000000000..93dad3df1f1 --- /dev/null +++ b/Doc/library/importlib_metadata.rst @@ -0,0 +1,256 @@ +.. _using: + +========================== + Using importlib_metadata +========================== + +``importlib_metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +``importlib.resources`` in `Python 3.7 +and newer`_ (backported as `importlib_resources`_ for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by `PEP 566`_ or its older specifications. +By default, package metadata can live on the file system or in wheels on +``sys.path``. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it:: + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install importlib_metadata + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following:: + + (example) $ python + >>> from importlib_metadata import version + >>> version('wheel') + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint ` objects. + +You can get the :ref:`metadata for a distribution `:: + + >>> list(metadata('wheel')) + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number `, list its +:ref:`constituent files `_, and get a list of the distribution's +:ref:`requirements`_. + + +Distributions +============= + +.. CAUTION:: The ``Distribution`` class described here may or may not end up + in the final stable public API. Consider this class `provisional + `_ until the 1.0 + release. + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib_metadata import distribution + >>> dist = distribution('wheel') + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> d.metadata['Requires-Python'] + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> d.metadata['License'] + 'MIT' + +The full set of available metadata is not described here. See `PEP 566 +`_ for additional details. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points:: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. + + >>> eps = entry_points() + >>> list(eps) + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] + >>> wheel + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> main = wheel.load() + >>> main + + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +`_ +for more information on entrypoints, their definition, and usage. + + +.. _metadata:: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') + +The keys of the returned data structure [#f1]_ name the metadata keywords, and +their values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version:: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') + '0.32.3' + + +.. _files:: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a `pathlib.Path`_ derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] + >>> util + PackagePath('wheel/util.py') + >>> util.size + 859 + >>> util.dist + + >>> util.hash + + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + + +.. _requirements:: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function. Note that this returns an iterator:: + + >>> list(requires('wheel')) + ["pytest (>=3.0.0) ; extra == 'test'"] + + + +Extending the search algorithm +============================== + +Because package metadata is not available through ``sys.path`` searches, or +package loaders directly, the metadata for a package is found through import +system `finders`_. To find a distribution package's metadata, +``importlib_metadata`` queries the list of `meta path finders`_ on +`sys.meta_path`_. + +By default ``importlib_metadata`` installs a finder for distribution packages +found on the file system. This finder doesn't actually find any *packages*, +but it can find the packages' metadata. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib_metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +``sys.meta_path``. If the finder has this method, it must return +an iterator over instances of the ``Distribution`` abstract class. This +method must have the signature:: + + def find_distributions(name=None, path=sys.path): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the name + (or all names if not supplied) along the paths in the list + of directories ``path`` (defaults to sys.path). + """ + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, you should derive from +``Distribution`` and implement the ``load_metadata()`` method. This takes a +single argument which is the name of the package whose metadata is being +found. This instance of the ``Distribution`` base abstract class is what your +finder's ``find_distributions()`` method should return. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`Python 3.7 and newer`: https://docs.python.org/3/library/importlib.html#module-importlib.resources +.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html +.. _`PEP 566`: https://www.python.org/dev/peps/pep-0566/ +.. _`finders`: https://docs.python.org/3/reference/import.html#finders-and-loaders +.. _`meta path finders`: https://docs.python.org/3/glossary.html#term-meta-path-finder +.. _`sys.meta_path`: https://docs.python.org/3/library/sys.html#sys.meta_path +.. _`pathlib.Path`: https://docs.python.org/3/library/pathlib.html#pathlib.Path + + +.. rubric:: Footnotes + +.. [#f1] Technically, the returned distribution metadata object is an + `email.message.Message + `_ + instance, but this is an implementation detail, and not part of the + stable API. You should only use dictionary-like methods and syntax + to access the metadata contents. diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py new file mode 100644 index 00000000000..314ece65afe --- /dev/null +++ b/Lib/importlib/metadata/__init__.py @@ -0,0 +1,17 @@ +from .api import distribution, Distribution, PackageNotFoundError # noqa: F401 +from .api import ( + metadata, entry_points, version, files, requires, distributions, + ) + +# Import for installation side-effects. +from . import _hooks # noqa: F401 + + +__all__ = [ + 'entry_points', + 'files', + 'metadata', + 'requires', + 'version', + 'distributions', + ] diff --git a/Lib/importlib/metadata/_hooks.py b/Lib/importlib/metadata/_hooks.py new file mode 100644 index 00000000000..5f9bdb54a2e --- /dev/null +++ b/Lib/importlib/metadata/_hooks.py @@ -0,0 +1,138 @@ +import re +import sys +import itertools + +from . import zipp +from .api import Distribution +from .abc import DistributionFinder +from contextlib import suppress +from pathlib import Path + + +def install(cls): + """Class decorator for installation on sys.meta_path.""" + sys.meta_path.append(cls()) + return cls + + +class NullFinder(DistributionFinder): + """ + A "Finder" (aka "MetaClassFinder") that never finds any modules, + but may find distributions. + """ + @staticmethod + def find_spec(*args, **kwargs): + return None + + +@install +class MetadataPathFinder(NullFinder): + """A degenerate finder for distribution packages on the file system. + + This finder supplies only a find_distributions() method for versions + of Python that do not have a PathFinder find_distributions(). + """ + search_template = r'{pattern}(-.*)?\.(dist|egg)-info' + + def find_distributions(self, name=None, path=None): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the name + (or all names if not supplied) along the paths in the list + of directories ``path`` (defaults to sys.path). + """ + if path is None: + path = sys.path + pattern = '.*' if name is None else re.escape(name) + found = self._search_paths(pattern, path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, pattern, paths): + """ + Find metadata directories in paths heuristically. + """ + return itertools.chain.from_iterable( + cls._search_path(path, pattern) + for path in map(Path, paths) + ) + + @classmethod + def _search_path(cls, root, pattern): + if not root.is_dir(): + return () + normalized = pattern.replace('-', '_') + return ( + item + for item in root.iterdir() + if item.is_dir() + and re.match( + cls.search_template.format(pattern=normalized), + str(item.name), + flags=re.IGNORECASE, + ) + ) + + +class PathDistribution(Distribution): + def __init__(self, path): + """Construct a distribution from a path to the metadata directory.""" + self._path = path + + def read_text(self, filename): + with suppress(FileNotFoundError): + with self._path.joinpath(filename).open(encoding='utf-8') as fp: + return fp.read() + return None + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + +@install +class WheelMetadataFinder(NullFinder): + """A degenerate finder for distribution packages in wheels. + + This finder supplies only a find_distributions() method for versions + of Python that do not have a PathFinder find_distributions(). + """ + search_template = r'{pattern}(-.*)?\.whl' + + def find_distributions(self, name=None, path=None): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the name + (or all names if not supplied) along the paths in the list + of directories ``path`` (defaults to sys.path). + """ + if path is None: + path = sys.path + pattern = '.*' if name is None else re.escape(name) + found = self._search_paths(pattern, path) + return map(WheelDistribution, found) + + @classmethod + def _search_paths(cls, pattern, paths): + return ( + path + for path in map(Path, paths) + if re.match( + cls.search_template.format(pattern=pattern), + str(path.name), + flags=re.IGNORECASE, + ) + ) + + +class WheelDistribution(Distribution): + def __init__(self, archive): + self._archive = zipp.Path(archive) + name, version = archive.name.split('-')[0:2] + self._dist_info = '{}-{}.dist-info'.format(name, version) + + def read_text(self, filename): + target = self._archive / self._dist_info / filename + return target.read_text() if target.exists() else None + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._archive / path diff --git a/Lib/importlib/metadata/abc.py b/Lib/importlib/metadata/abc.py new file mode 100644 index 00000000000..1785cf3c1c2 --- /dev/null +++ b/Lib/importlib/metadata/abc.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import + + +import abc + +from importlib.abc import MetaPathFinder + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + @abc.abstractmethod + def find_distributions(self, name=None, path=None): + """ + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the name + (or all names if not supplied) along the paths in the list + of directories ``path`` (defaults to sys.path). + """ diff --git a/Lib/importlib/metadata/api.py b/Lib/importlib/metadata/api.py new file mode 100644 index 00000000000..df19aff438b --- /dev/null +++ b/Lib/importlib/metadata/api.py @@ -0,0 +1,353 @@ +from __future__ import absolute_import + +import re +import abc +import csv +import sys +import email +import operator +import functools +import itertools +import collections + +from importlib import import_module +from itertools import starmap + +import pathlib +from configparser import ConfigParser + +__metaclass__ = type + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + +class EntryPoint(collections.namedtuple('EntryPointBase', 'name value group')): + """An entry point as defined by Python packaging conventions.""" + + pattern = re.compile( + r'(?P[\w.]+)\s*' + r'(:\s*(?P[\w.]+))?\s*' + r'(?P\[.*\])?\s*$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def extras(self): + match = self.pattern.match(self.value) + return list(re.finditer(r'\w+', match.group('extras') or '')) + + @classmethod + def _from_config(cls, config): + return [ + cls(name, value, group) + for group in config.sections() + for name, value in config.items(group) + ] + + @classmethod + def _from_text(cls, text): + config = ConfigParser() + config.read_string(text) + return EntryPoint._from_config(config) + + def __iter__(self): + """ + Supply iter so one may construct dicts of EntryPoints easily. + """ + return iter((self.name, self)) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return ''.format(self.mode, self.value) + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename): + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + dists = resolver(name) + dist = next(dists, None) + if dist is not None: + return dist + else: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls): + """Return an iterable of Distribution objects for all packages. + + :return: Iterable of Distribution objects for all packages. + """ + return itertools.chain.from_iterable( + resolver() + for resolver in cls._discover_resolvers() + ) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) + for finder in sys.meta_path + ) + return filter(None, declared) + + @classmethod + def find_local(cls): + dists = itertools.chain.from_iterable( + resolver(path=['.']) + for resolver in cls._discover_resolvers() + ) + dist, = dists + return dist + + @property + def metadata(self): + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + text = self.read_text('METADATA') or self.read_text('PKG-INFO') + return email.message_from_string(text) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoint._from_text(self.read_text('entry_points.txt')) + + @property + def files(self): + file_lines = self._read_files_distinfo() or self._read_files_egginfo() + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + return file_lines and starmap(make_file, csv.reader(file_lines)) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo(self): + """ + SOURCES.txt might contain literal commas, so wrap each line + in quotes. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + return self._read_dist_info_reqs() or self._read_egg_info_reqs() + + def _read_dist_info_reqs(self): + spec = self.metadata['Requires-Dist'] + return spec and filter(None, spec.splitlines()) + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return self._deps_from_requires_text(source) + + @classmethod + def _deps_from_requires_text(cls, source): + section_pairs = cls._read_sections(source.splitlines()) + sections = { + section: list(map(operator.itemgetter('line'), results)) + for section, results in + itertools.groupby(section_pairs, operator.itemgetter('section')) + } + return cls._convert_egg_info_reqs_to_simple_reqs(sections) + + @staticmethod + def _read_sections(lines): + section = None + for line in filter(None, lines): + section_match = re.match(r'\[(.*)\]$', line) + if section_match: + section = section_match.group(1) + continue + yield locals() + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + def make_condition(name): + return name and 'extra == "{name}"'.format(name=name) + + def parse_condition(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = '({markers})'.format(markers=markers) + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + for section, deps in sections.items(): + for dep in deps: + yield dep + parse_condition(section) + + +def distribution(package): + """Get the ``Distribution`` instance for the given package. + + :param package: The name of the package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(package) + + +def distributions(): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover() + + +def local_distribution(): + """Get the ``Distribution`` instance for the package in CWD. + + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.find_local() + + +def metadata(package): + """Get the metadata for the package. + + :param package: The name of the distribution package to query. + :return: An email.Message containing the parsed metadata. + """ + return Distribution.from_name(package).metadata + + +def version(package): + """Get the version string for the named package. + + :param package: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(package).version + + +def entry_points(): + """Return EntryPoint objects for all installed packages. + + :return: EntryPoint objects for all installed packages. + """ + eps = itertools.chain.from_iterable( + dist.entry_points for dist in distributions()) + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return { + group: tuple(eps) + for group, eps in grouped + } + + +def files(package): + return distribution(package).files + + +def requires(package): + """ + Return a list of requirements for the indicated distribution. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(package).requires diff --git a/Lib/importlib/metadata/zipp.py b/Lib/importlib/metadata/zipp.py new file mode 100644 index 00000000000..ffd129f63b0 --- /dev/null +++ b/Lib/importlib/metadata/zipp.py @@ -0,0 +1,110 @@ +""" +>>> root = Path(getfixture('zipfile_abcde')) +>>> a, b = root.iterdir() +>>> a +Path('abcde.zip', 'a.txt') +>>> b +Path('abcde.zip', 'b/') +>>> b.name +'b' +>>> c = b / 'c.txt' +>>> c +Path('abcde.zip', 'b/c.txt') +>>> c.name +'c.txt' +>>> c.read_text() +'content of c' +>>> c.exists() +True +>>> (b / 'missing.txt').exists() +False +>>> str(c) +'abcde.zip/b/c.txt' +""" + +from __future__ import division + +import io +import sys +import posixpath +import zipfile +import operator +import functools + +__metaclass__ = type + + +class Path: + __repr = '{self.__class__.__name__}({self.root.filename!r}, {self.at!r})' + + def __init__(self, root, at=''): + self.root = root if isinstance(root, zipfile.ZipFile) \ + else zipfile.ZipFile(self._pathlib_compat(root)) + self.at = at + + @staticmethod + def _pathlib_compat(path): + """ + For path-like objects, convert to a filename for compatibility + on Python 3.6.1 and earlier. + """ + try: + return path.__fspath__() + except AttributeError: + return str(path) + + @property + def open(self): + return functools.partial(self.root.open, self.at) + + @property + def name(self): + return posixpath.basename(self.at.rstrip('/')) + + def read_text(self, *args, **kwargs): + with self.open() as strm: + return io.TextIOWrapper(strm, *args, **kwargs).read() + + def read_bytes(self): + with self.open() as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip('/')) == self.at.rstrip('/') + + def _next(self, at): + return Path(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith('/') + + def is_file(self): + return not self.is_dir() + + def exists(self): + return self.at in self.root.namelist() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + names = map(operator.attrgetter('filename'), self.root.infolist()) + subs = map(self._next, names) + return filter(self._is_child, subs) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def __truediv__(self, add): + add = self._pathlib_compat(add) + next = posixpath.join(self.at, add) + next_dir = posixpath.join(self.at, add, '') + names = self.root.namelist() + return self._next( + next_dir if next not in names and next_dir in names else next + ) + + if sys.version_info < (3,): + __div__ = __truediv__ \ No newline at end of file diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/test_importlib/data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..f92f7716e3e613a2a9703be785135fd8b35cfb1c GIT binary patch literal 1453 zcmWIWW@Zs#U|`^2$X%l2E2Vj`LJ`OVVPPOntw_u*$Vt_YkI&4@EQycTE2#AL^bJ1Y zd*;mL3tJuqF*Gf@GU?JH8`iH^x{lmwniEp0#}EKF@#|6@-~Vw}E~^1e(gI=)go(OF zhI)oZdMTO3CAyh;Y5Dr8c_l@a@df#rc_qbqB^4#ze&^0>pF8i_tM8|GN=HMp@2S^X zk2AU_JVQ5xHf&l`By9Y7#||{R?xt`CaRKe%0Af`#eJG?#%hkK?YZh9~AkY_15*$IjO%X$iwTT zj$Wre`^vxz1{aLYt{7i=!gcDr{>864*LXE_z0RKW*%YLqspb2W%hP9jkj4s=YiCcN z_rB_TX7!Ut=+0vKml077bk1%dR>0#dU)K;v7sn9C*zS#7hYUnqzke6~*(h?!FxuR) z*JA$j=D}6nJuk63>$g|^aHQ)0StmYywxSwA=est9~!zT(kqV|fK2BE1^mxK0q zD(qh)b?4sPb$w-l5$bn#F3Qs`KJ!0*^MQ6R{<p(1r$KgS)&i+9zwP$x1H90UiT)ZdwKjq+uV@j;~jI4*9 zFSbRdFl*>>tZs;(uk_eQ>hgU{@!k)&u4PPICc4gJ)}^;Bzl1h5{m``K-uhy9=9lTe z-NnTxRmGRCybveUZnWu;{lQCXJ}s6};eD{ z#=6?Q%UABG&=n2Q`J~J@GtS}8)q{&~{}TM{1aZ}u05ukm*h zl5ERPr=_}oV=S9Bfwe1=@2=aOxF-kSE;PEja34gdfE literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py new file mode 100644 index 00000000000..8a8b4add31c --- /dev/null +++ b/Lib/test/test_importlib/fixtures.py @@ -0,0 +1,175 @@ +from __future__ import unicode_literals + +import os +import sys +import shutil +import tempfile +import textwrap +import contextlib + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + +try: + import pathlib +except ImportError: + import pathlib2 as pathlib + + +__metaclass__ = type + + +@contextlib.contextmanager +def tempdir(): + tmpdir = tempfile.mkdtemp() + sys.path[:0] = [tmpdir] + try: + yield pathlib.Path(tmpdir) + finally: + sys.path.remove(tmpdir) + shutil.rmtree(tmpdir) + + +@contextlib.contextmanager +def save_cwd(): + orig = os.getcwd() + try: + yield + finally: + os.chdir(orig) + + +@contextlib.contextmanager +def tempdir_as_cwd(): + with tempdir() as tmp: + with save_cwd(): + os.chdir(str(tmp)) + yield tmp + + +class SiteDir: + @staticmethod + @contextlib.contextmanager + def site_dir(): + with tempdir() as tmp: + sys.path[:0] = [str(tmp)] + yield tmp + + def setUp(self): + self.fixtures = ExitStack() + self.addCleanup(self.fixtures.close) + self.site_dir = self.fixtures.enter_context(self.site_dir()) + + +class DistInfoPkg(SiteDir): + files = { + "distinfo_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Author: Steven Ma + Version: 1.0.0 + Requires-Dist: wheel >= 1.0 + Requires-Dist: pytest; extra == 'test' + """, + "RECORD": "mod.py,sha256=abc,20\n", + "entry_points.txt": """ + [entries] + main = mod:main + """ + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def setUp(self): + super(DistInfoPkg, self).setUp() + build_files(DistInfoPkg.files, self.site_dir) + + +class EggInfoPkg(SiteDir): + files = { + "egginfo_pkg.egg-info": { + "PKG-INFO": """ + Name: egginfo-pkg + Author: Steven Ma + License: Unknown + Version: 1.0.0 + Classifier: Intended Audience :: Developers + Classifier: Topic :: Software Development :: Libraries + """, + "SOURCES.txt": """ + mod.py + egginfo_pkg.egg-info/top_level.txt + """, + "entry_points.txt": """ + [entries] + main = mod:main + """, + "requires.txt": """ + wheel >= 1.0; python_version >= "2.7" + [test] + pytest + """, + "top_level.txt": "mod\n" + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def setUp(self): + super(EggInfoPkg, self).setUp() + build_files(EggInfoPkg.files, prefix=self.site_dir) + + +class LocalPackage: + def setUp(self): + self.fixtures = ExitStack() + self.addCleanup(self.fixtures.close) + self.fixtures.enter_context(tempdir_as_cwd()) + build_files(EggInfoPkg.files) + + +def build_files(file_defs, prefix=pathlib.Path()): + """ + Build a set of files/directories, as described by the + file_defs dictionary. + Each key/value pair in the dictionary is interpreted as + a filename/contents + pair. If the contents value is a dictionary, a directory + is created, and the + dictionary interpreted as the files within it, recursively. + For example: + {"README.txt": "A README file", + "foo": { + "__init__.py": "", + "bar": { + "__init__.py": "", + }, + "baz.py": "# Some code", + } + } + """ + for name, contents in file_defs.items(): + full_name = prefix / name + if isinstance(contents, dict): + full_name.mkdir() + build_files(contents, prefix=full_name) + else: + if isinstance(contents, bytes): + with full_name.open('wb') as f: + f.write(contents) + else: + with full_name.open('w') as f: + f.write(DALS(contents)) + + +def DALS(str): + "Dedent and left-strip" + return textwrap.dedent(str).lstrip() + diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py new file mode 100644 index 00000000000..64270850a9f --- /dev/null +++ b/Lib/test/test_importlib/test_main.py @@ -0,0 +1,157 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re +import textwrap +import unittest +import importlib +import importlib.metadata + +from . import fixtures +from importlib.metadata import _hooks + + +class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + dist = importlib.metadata.Distribution.from_name('distinfo-pkg') + assert isinstance(dist.version, str) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(importlib.metadata.PackageNotFoundError): + importlib.metadata.Distribution.from_name('does-not-exist') + + def test_new_style_classes(self): + self.assertIsInstance(importlib.metadata.Distribution, type) + self.assertIsInstance(_hooks.MetadataPathFinder, type) + self.assertIsInstance(_hooks.WheelMetadataFinder, type) + self.assertIsInstance(_hooks.WheelDistribution, type) + + +class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): + def test_import_nonexistent_module(self): + # Ensure that the MetadataPathFinder does not crash an import of a + # non-existant module. + with self.assertRaises(ImportError): + importlib.import_module('does_not_exist') + + def test_resolve(self): + entries = dict(importlib.metadata.entry_points()['entries']) + ep = entries['main'] + self.assertEqual(ep.load().__name__, "main") + + def test_resolve_without_attr(self): + ep = importlib.metadata.api.EntryPoint( + name='ep', + value='importlib.metadata.api', + group='grp', + ) + assert ep.load() is importlib.metadata.api + + +class NameNormalizationTests(fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_dashes(site_dir): + """ + Create minimal metadata for a package with dashes + in the name (and thus underscores in the filename). + """ + metadata_dir = site_dir / 'my_pkg.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w') as strm: + strm.write('Version: 1.0\n') + return 'my-pkg' + + def test_dashes_in_dist_name_found_as_underscores(self): + """ + For a package with a dash in the name, the dist-info metadata + uses underscores in the name. Ensure the metadata loads. + """ + pkg_name = self.pkg_with_dashes(self.site_dir) + assert importlib.metadata.version(pkg_name) == '1.0' + + @staticmethod + def pkg_with_mixed_case(site_dir): + """ + Create minimal metadata for a package with mixed case + in the name. + """ + metadata_dir = site_dir / 'CherryPy.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w') as strm: + strm.write('Version: 1.0\n') + return 'CherryPy' + + def test_dist_name_found_as_any_case(self): + """ + Ensure the metadata loads when queried with any case. + """ + pkg_name = self.pkg_with_mixed_case(self.site_dir) + assert importlib.metadata.version(pkg_name) == '1.0' + assert importlib.metadata.version(pkg_name.lower()) == '1.0' + assert importlib.metadata.version(pkg_name.upper()) == '1.0' + + +class NonASCIITests(fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_non_ascii_description(site_dir): + """ + Create minimal metadata for a package with non-ASCII in + the description. + """ + metadata_dir = site_dir / 'portend.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as fp: + fp.write('Description: pôrˈtend\n') + return 'portend' + + @staticmethod + def pkg_with_non_ascii_description_egg_info(site_dir): + """ + Create minimal metadata for an egg-info package with + non-ASCII in the description. + """ + metadata_dir = site_dir / 'portend.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as fp: + fp.write(textwrap.dedent(""" + Name: portend + + pôrˈtend + """).lstrip()) + return 'portend' + + def test_metadata_loads(self): + pkg_name = self.pkg_with_non_ascii_description(self.site_dir) + meta = importlib.metadata.metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + def test_metadata_loads_egg_info(self): + pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) + meta = importlib.metadata.metadata(pkg_name) + assert meta.get_payload() == 'pôrˈtend\n' + + +class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, + unittest.TestCase): + + def test_package_discovery(self): + dists = list(importlib.metadata.api.distributions()) + assert all( + isinstance(dist, importlib.metadata.Distribution) + for dist in dists + ) + assert any( + dist.metadata['Name'] == 'egginfo-pkg' + for dist in dists + ) + assert any( + dist.metadata['Name'] == 'distinfo-pkg' + for dist in dists + ) diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py new file mode 100644 index 00000000000..a6aa395831d --- /dev/null +++ b/Lib/test/test_importlib/test_metadata_api.py @@ -0,0 +1,134 @@ +import re +import textwrap +import unittest +import importlib.metadata + +from collections.abc import Iterator + +from . import fixtures + + +class APITests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + version = importlib.metadata.version('egginfo-pkg') + assert isinstance(version, str) + assert re.match(self.version_pattern, version) + + def test_retrieves_version_of_pip(self): + version = importlib.metadata.version('distinfo-pkg') + assert isinstance(version, str) + assert re.match(self.version_pattern, version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(importlib.metadata.PackageNotFoundError): + importlib.metadata.distribution('does-not-exist') + + def test_for_top_level(self): + distribution = importlib.metadata.distribution('egginfo-pkg') + self.assertEqual( + distribution.read_text('top_level.txt').strip(), + 'mod') + + def test_read_text(self): + top_level = [ + path for path in importlib.metadata.files('egginfo-pkg') + if path.name == 'top_level.txt' + ][0] + self.assertEqual(top_level.read_text(), 'mod\n') + + def test_entry_points(self): + entires = importlib.metadata.entry_points()['entries'] + entries = dict(entires) + ep = entries['main'] + self.assertEqual(ep.value, 'mod:main') + self.assertEqual(ep.extras, []) + + def test_metadata_for_this_package(self): + md = importlib.metadata.metadata('egginfo-pkg') + assert md['author'] == 'Steven Ma' + assert md['LICENSE'] == 'Unknown' + assert md['Name'] == 'egginfo-pkg' + classifiers = md.get_all('Classifier') + assert 'Topic :: Software Development :: Libraries' in classifiers + + @staticmethod + def _test_files(files_iter): + assert isinstance(files_iter, Iterator), files_iter + files = list(files_iter) + root = files[0].root + for file in files: + assert file.root == root + assert not file.hash or file.hash.value + assert not file.hash or file.hash.mode == 'sha256' + assert not file.size or file.size >= 0 + assert file.locate().exists() + assert isinstance(file.read_binary(), bytes) + if file.name.endswith('.py'): + file.read_text() + + def test_file_hash_repr(self): + assertRegex = self.assertRegex + + util = [ + p for p in importlib.metadata.files('distinfo-pkg') + if p.name == 'mod.py' + ][0] + assertRegex( + repr(util.hash), + '') + + def test_files_dist_info(self): + self._test_files(importlib.metadata.files('distinfo-pkg')) + + def test_files_egg_info(self): + self._test_files(importlib.metadata.files('egginfo-pkg')) + + def test_requires(self): + deps = importlib.metadata.requires('egginfo-pkg') + assert any( + dep == 'wheel >= 1.0; python_version >= "2.7"' + for dep in deps + ) + + def test_requires_dist_info(self): + deps = list(importlib.metadata.requires('distinfo-pkg')) + assert deps and all(deps) + + def test_more_complex_deps_requires_text(self): + requires = textwrap.dedent(""" + dep1 + dep2 + + [:python_version < "3"] + dep3 + + [extra1] + dep4 + + [extra2:python_version < "3"] + dep5 + """) + deps = sorted( + importlib.metadata.api.Distribution._deps_from_requires_text( + requires) + ) + expected = [ + 'dep1', + 'dep2', + 'dep3; python_version < "3"', + 'dep4; extra == "extra1"', + 'dep5; (python_version < "3") and extra == "extra2"', + ] + # It's important that the environment marker expression be + # wrapped in parentheses to avoid the following 'and' binding more + # tightly than some other part of the environment expression. + + assert deps == expected + + +class LocalProjectTests(fixtures.LocalPackage, unittest.TestCase): + def test_find_local(self): + dist = importlib.metadata.api.local_distribution() + assert dist.metadata['Name'] == 'egginfo-pkg' diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py new file mode 100644 index 00000000000..4af027a12be --- /dev/null +++ b/Lib/test/test_importlib/test_zip.py @@ -0,0 +1,48 @@ +import sys +import unittest +import importlib.metadata + +from importlib.resources import path + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + + +class BespokeLoader: + archive = 'bespoke' + + +class TestZip(unittest.TestCase): + def setUp(self): + # Find the path to the example.*.whl so we can add it to the front of + # sys.path, where we'll then try to find the metadata thereof. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + wheel = self.resources.enter_context( + path('test.test_importlib.data', + 'example-21.12-py3-none-any.whl')) + sys.path.insert(0, str(wheel)) + self.resources.callback(sys.path.pop, 0) + + def test_zip_version(self): + self.assertEqual(importlib.metadata.version('example'), '21.12') + + def test_zip_entry_points(self): + scripts = dict(importlib.metadata.entry_points()['console_scripts']) + entry_point = scripts['example'] + self.assertEqual(entry_point.value, 'example:main') + + def test_missing_metadata(self): + distribution = importlib.metadata.distribution('example') + self.assertIsNone(distribution.read_text('does not exist')) + + def test_case_insensitive(self): + self.assertEqual(importlib.metadata.version('Example'), '21.12') + + def test_files(self): + files = importlib.metadata.files('example') + for file in files: + path = str(file.dist.locate_file(file)) + assert '.whl/' in path, path