Branch merge

Doc/library/collections.rst

@@ -83,7 +83,7 @@ The class can be used to simulate nested scopes and is useful in templating.
       creating subcontexts that can be updated without altering values in any
       of the parent mappings.
 
-   .. attribute:: parents()
+   .. method:: parents()
 
       Returns a new :class:`ChainMap` containing all of the maps in the current
       instance except the first one.  This is useful for skipping the first map

Doc/packaging/setupcfg.rst

@@ -176,15 +176,19 @@ compilers
       compilers =
           hotcompiler.SmartCCompiler
 
-setup_hook
+setup_hooks
-   defines a callable that will be called right after the
+   Defines a list of callables to be called right after the :file:`setup.cfg`
-   :file:`setup.cfg` file is read. The callable receives the configuration
+   file is read, before any other processing.  The callables are executed in the
-   in form of a mapping and can make some changes to it. *optional*
+   order they're found in the file; if one of them cannot be found, tools should
+   not stop, but for example produce a warning and continue with the next line.
+   Each callable receives the configuration as a dictionary (keys are
+   :file:`setup.cfg` sections, values are dictionaries of fields) and can make
+   any changes to it.  *optional*, *multi*
 
    Example::
 
       [global]
-      setup_hook = package.setup.customize_dist
+      setup_hooks = package.setup.customize_dist
 
 
 Metadata
@@ -285,6 +289,7 @@ One extra field not present in PEP 345 is supported:
 
 description-file
    Path to a text file that will be used to fill the ``description`` field.
+   Multiple values are accepted; they must be separated by whitespace.
    ``description-file`` and ``description`` are mutually exclusive.  *optional*
 
 

Lib/packaging/config.py

@@ -9,7 +9,8 @@ from configparser import RawConfigParser
 from packaging import logger
 from packaging.errors import PackagingOptionError
 from packaging.compiler.extension import Extension
-from packaging.util import check_environ, iglob, resolve_name, strtobool
+from packaging.util import (check_environ, iglob, resolve_name, strtobool,
+                            split_multiline)
 from packaging.compiler import set_compiler
 from packaging.command import set_command
 from packaging.markers import interpret
@@ -60,17 +61,15 @@ def get_resources_dests(resources_root, rules):
 
 
 class Config:
-    """Reads configuration files and work with the Distribution instance
+    """Class used to work with configuration files"""
-    """
     def __init__(self, dist):
         self.dist = dist
-        self.setup_hook = None
+        self.setup_hooks = []
 
-    def run_hook(self, config):
+    def run_hooks(self, config):
-        if self.setup_hook is None:
+        """Run setup hooks in the order defined in the spec."""
-            return
+        for hook in self.setup_hooks:
-        # the hook gets only the config
+            hook(config)
-        self.setup_hook(config)
 
     def find_config_files(self):
         """Find as many configuration files as should be processed for this
@@ -124,29 +123,26 @@ class Config:
         # XXX
         return value
 
-    def _multiline(self, value):
-        value = [v for v in
-                 [v.strip() for v in value.split('\n')]
-                 if v != '']
-        return value
-
     def _read_setup_cfg(self, parser, cfg_filename):
         cfg_directory = os.path.dirname(os.path.abspath(cfg_filename))
         content = {}
         for section in parser.sections():
             content[section] = dict(parser.items(section))
 
-        # global:setup_hook is called *first*
+        # global setup hooks are called first
         if 'global' in content:
-            if 'setup_hook' in content['global']:
+            if 'setup_hooks' in content['global']:
-                setup_hook = content['global']['setup_hook']
+                setup_hooks = split_multiline(content['global']['setup_hooks'])
-                try:
+
-                    self.setup_hook = resolve_name(setup_hook)
+                for line in setup_hooks:
-                except ImportError as e:
+                    try:
-                    logger.warning('could not import setup_hook: %s',
+                        hook = resolve_name(line)
-                            e.args[0])
+                    except ImportError as e:
-                else:
+                        logger.warning('cannot find setup hook: %s', e.args[0])
-                    self.run_hook(content)
+                    else:
+                        self.setup_hooks.append(hook)
+
+                self.run_hooks(content)
 
         metadata = self.dist.metadata
 
@@ -155,7 +151,7 @@ class Config:
             for key, value in content['metadata'].items():
                 key = key.replace('_', '-')
                 if metadata.is_multi_field(key):
-                    value = self._multiline(value)
+                    value = split_multiline(value)
 
                 if key == 'project-url':
                     value = [(label.strip(), url.strip())
@@ -168,21 +164,18 @@ class Config:
                                "mutually exclusive")
                         raise PackagingOptionError(msg)
 
-                    if isinstance(value, list):
+                    filenames = value.split()
-                        filenames = value
-                    else:
-                        filenames = value.split()
 
-                    # concatenate each files
+                    # concatenate all files
-                    value = ''
+                    value = []
                     for filename in filenames:
                         # will raise if file not found
                         with open(filename) as description_file:
-                            value += description_file.read().strip() + '\n'
+                            value.append(description_file.read().strip())
                         # add filename as a required file
                         if filename not in metadata.requires_files:
                             metadata.requires_files.append(filename)
-                    value = value.strip()
+                    value = '\n'.join(value).strip()
                     key = 'description'
 
                 if metadata.is_metadata_field(key):
@@ -192,7 +185,7 @@ class Config:
             files = content['files']
             self.dist.package_dir = files.pop('packages_root', None)
 
-            files = dict((key, self._multiline(value)) for key, value in
+            files = dict((key, split_multiline(value)) for key, value in
                          files.items())
 
             self.dist.packages = []
@@ -310,7 +303,7 @@ class Config:
                     opt = opt.replace('-', '_')
 
                     if opt == 'sub_commands':
-                        val = self._multiline(val)
+                        val = split_multiline(val)
                         if isinstance(val, str):
                             val = [val]
 
@@ -348,14 +341,14 @@ class Config:
                     raise PackagingOptionError(msg)
 
     def _load_compilers(self, compilers):
-        compilers = self._multiline(compilers)
+        compilers = split_multiline(compilers)
         if isinstance(compilers, str):
             compilers = [compilers]
         for compiler in compilers:
             set_compiler(compiler.strip())
 
     def _load_commands(self, commands):
-        commands = self._multiline(commands)
+        commands = split_multiline(commands)
         if isinstance(commands, str):
             commands = [commands]
         for command in commands:

Lib/packaging/tests/test_config.py

@@ -90,7 +90,7 @@ commands =
 compilers =
     packaging.tests.test_config.DCompiler
 
-setup_hook = %(setup-hook)s
+setup_hooks = %(setup-hooks)s
 
 
 
@@ -135,8 +135,16 @@ class DCompiler:
         pass
 
 
-def hook(content):
+def version_hook(config):
-    content['metadata']['version'] += '.dev1'
+    config['metadata']['version'] += '.dev1'
+
+
+def first_hook(config):
+    config['files']['modules'] += '\n first'
+
+
+def third_hook(config):
+    config['files']['modules'] += '\n third'
 
 
 class FooBarBazTest:
@@ -186,7 +194,7 @@ class ConfigTestCase(support.TempdirManager,
 
     def write_setup(self, kwargs=None):
         opts = {'description-file': 'README', 'extra-files': '',
-                'setup-hook': 'packaging.tests.test_config.hook'}
+                'setup-hooks': 'packaging.tests.test_config.version_hook'}
         if kwargs:
             opts.update(kwargs)
         self.write_file('setup.cfg', SETUP_CFG % opts, encoding='utf-8')
@@ -318,16 +326,30 @@ class ConfigTestCase(support.TempdirManager,
         self.assertEqual(ext.extra_compile_args, cargs)
         self.assertEqual(ext.language, 'cxx')
 
-    def test_missing_setuphook_warns(self):
+    def test_missing_setup_hook_warns(self):
-        self.write_setup({'setup-hook': 'this.does._not.exist'})
+        self.write_setup({'setup-hooks': 'this.does._not.exist'})
         self.write_file('README', 'yeah')
         dist = self.get_dist()
         logs = self.get_logs(logging.WARNING)
         self.assertEqual(1, len(logs))
-        self.assertIn('could not import setup_hook', logs[0])
+        self.assertIn('cannot find setup hook', logs[0])
+
+    def test_multiple_setup_hooks(self):
+        self.write_setup({
+            'setup-hooks': '\n  packaging.tests.test_config.first_hook'
+                           '\n  packaging.tests.test_config.missing_hook'
+                           '\n  packaging.tests.test_config.third_hook'
+        })
+        self.write_file('README', 'yeah')
+        dist = self.get_dist()
+
+        self.assertEqual(['haven', 'first', 'third'], dist.py_modules)
+        logs = self.get_logs(logging.WARNING)
+        self.assertEqual(1, len(logs))
+        self.assertIn('cannot find setup hook', logs[0])
 
     def test_metadata_requires_description_files_missing(self):
-        self.write_setup({'description-file': 'README\n  README2'})
+        self.write_setup({'description-file': 'README README2'})
         self.write_file('README', 'yeah')
         self.write_file('README2', 'yeah')
         os.mkdir('src')

Lib/packaging/tests/test_util.py

@@ -8,16 +8,18 @@ import subprocess
 from io import StringIO
 
 from packaging.tests import support, unittest
+from packaging.tests.test_config import SETUP_CFG
 from packaging.errors import (
     PackagingPlatformError, PackagingByteCompileError, PackagingFileError,
     PackagingExecError, InstallationException)
 from packaging import util
+from packaging.dist import Distribution
 from packaging.util import (
     convert_path, change_root, split_quoted, strtobool, rfc822_escape,
     get_compiler_versions, _MAC_OS_X_LD_VERSION, byte_compile, find_packages,
     spawn, get_pypirc_path, generate_pypirc, read_pypirc, resolve_name, iglob,
     RICH_GLOB, egginfo_to_distinfo, is_setuptools, is_distutils, is_packaging,
-    get_install_method)
+    get_install_method, cfg_to_args)
 
 
 PYPIRC = """\
@@ -88,13 +90,15 @@ class UtilTestCase(support.EnvironRestorer,
                    support.LoggingCatcher,
                    unittest.TestCase):
 
-    restore_environ = ['HOME']
+    restore_environ = ['HOME', 'PLAT']
 
     def setUp(self):
         super(UtilTestCase, self).setUp()
-        self.tmp_dir = self.mkdtemp()
+        self.addCleanup(os.chdir, os.getcwd())
-        self.rc = os.path.join(self.tmp_dir, '.pypirc')
+        tempdir = self.mkdtemp()
-        os.environ['HOME'] = self.tmp_dir
+        self.rc = os.path.join(tempdir, '.pypirc')
+        os.environ['HOME'] = tempdir
+        os.chdir(tempdir)
         # saving the environment
         self.name = os.name
         self.platform = sys.platform
@@ -103,7 +107,6 @@ class UtilTestCase(support.EnvironRestorer,
         self.join = os.path.join
         self.isabs = os.path.isabs
         self.splitdrive = os.path.splitdrive
-        #self._config_vars = copy(sysconfig._config_vars)
 
         # patching os.uname
         if hasattr(os, 'uname'):
@@ -137,7 +140,6 @@ class UtilTestCase(support.EnvironRestorer,
             os.uname = self.uname
         else:
             del os.uname
-        #sysconfig._config_vars = copy(self._config_vars)
         util.find_executable = self.old_find_executable
         subprocess.Popen = self.old_popen
         sys.old_stdout = self.old_stdout
@@ -491,6 +493,38 @@ class UtilTestCase(support.EnvironRestorer,
             content = f.read()
         self.assertEqual(content, WANTED)
 
+    def test_cfg_to_args(self):
+        opts = {'description-file': 'README', 'extra-files': '',
+                'setup-hooks': 'packaging.tests.test_config.version_hook'}
+        self.write_file('setup.cfg', SETUP_CFG % opts)
+        self.write_file('README', 'loooong description')
+
+        args = cfg_to_args()
+        # use Distribution to get the contents of the setup.cfg file
+        dist = Distribution()
+        dist.parse_config_files()
+        metadata = dist.metadata
+
+        self.assertEqual(args['name'], metadata['Name'])
+        # + .dev1 because the test SETUP_CFG also tests a hook function in
+        # test_config.py for appending to the version string
+        self.assertEqual(args['version'] + '.dev1', metadata['Version'])
+        self.assertEqual(args['author'], metadata['Author'])
+        self.assertEqual(args['author_email'], metadata['Author-Email'])
+        self.assertEqual(args['maintainer'], metadata['Maintainer'])
+        self.assertEqual(args['maintainer_email'],
+                         metadata['Maintainer-Email'])
+        self.assertEqual(args['description'], metadata['Summary'])
+        self.assertEqual(args['long_description'], metadata['Description'])
+        self.assertEqual(args['classifiers'], metadata['Classifier'])
+        self.assertEqual(args['requires'], metadata['Requires-Dist'])
+        self.assertEqual(args['provides'], metadata['Provides-Dist'])
+
+        self.assertEqual(args['package_dir'].get(''), dist.package_dir)
+        self.assertEqual(args['packages'], dist.packages)
+        self.assertEqual(args['scripts'], dist.scripts)
+        self.assertEqual(args['py_modules'], dist.py_modules)
+
 
 class GlobTestCaseBase(support.TempdirManager,
                        support.LoggingCatcher,

Lib/packaging/util.py

@@ -250,6 +250,14 @@ def split_quoted(s):
     return words
 
 
+def split_multiline(value):
+    """Split a multiline string into a list, excluding blank lines."""
+
+    return [element for element in
+            (line.strip() for line in value.split('\n'))
+            if element]
+
+
 def execute(func, args, msg=None, verbose=0, dry_run=False):
     """Perform some action that affects the outside world.
 
@@ -542,18 +550,15 @@ def write_file(filename, contents):
 
 
 def _is_package(path):
-    if not os.path.isdir(path):
+    return os.path.isdir(path) and os.path.isfile(
-        return False
+        os.path.join(path, '__init__.py'))
-    return os.path.isfile(os.path.join(path, '__init__.py'))
 
 
 # Code taken from the pip project
 def _is_archive_file(name):
     archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar')
     ext = splitext(name)[1].lower()
-    if ext in archives:
+    return ext in archives
-        return True
-    return False
 
 
 def _under(path, root):
@@ -772,12 +777,13 @@ def spawn(cmd, search_path=True, verbose=0, dry_run=False, env=None):
     Raise PackagingExecError if running the program fails in any way; just
     return on success.
     """
-    logger.info(' '.join(cmd))
+    logger.debug('spawn: running %r', cmd)
     if dry_run:
+        logging.debug('dry run, no process actually spawned')
         return
     exit_status = subprocess.call(cmd, env=env)
     if exit_status != 0:
-        msg = "command '%s' failed with exit status %d"
+        msg = "command %r failed with exit status %d"
         raise PackagingExecError(msg % (cmd, exit_status))
 
 
@@ -1010,16 +1016,20 @@ def cfg_to_args(path='setup.cfg'):
                         "requires": ("metadata", "requires_dist"),
                         "provides": ("metadata", "provides_dist"),  # **
                         "obsoletes": ("metadata", "obsoletes_dist"),  # **
+                        "package_dir": ("files", 'packages_root'),
                         "packages": ("files",),
                         "scripts": ("files",),
                         "py_modules": ("files", "modules"),  # **
                         }
 
     MULTI_FIELDS = ("classifiers",
-                    "requires",
                     "platforms",
+                    "requires",
+                    "provides",
+                    "obsoletes",
                     "packages",
-                    "scripts")
+                    "scripts",
+                    "py_modules")
 
     def has_get_option(config, section, option):
         if config.has_option(section, option):
@@ -1031,9 +1041,9 @@ def cfg_to_args(path='setup.cfg'):
 
     # The real code starts here
     config = RawConfigParser()
-    if not os.path.exists(file):
+    if not os.path.exists(path):
         raise PackagingFileError("file '%s' does not exist" %
-                                 os.path.abspath(file))
+                                 os.path.abspath(path))
     config.read(path)
 
     kwargs = {}
@@ -1050,17 +1060,24 @@ def cfg_to_args(path='setup.cfg'):
         in_cfg_value = has_get_option(config, section, option)
         if not in_cfg_value:
             # There is no such option in the setup.cfg
-            if arg == "long_description":
+            if arg == 'long_description':
-                filename = has_get_option(config, section, "description_file")
+                filenames = has_get_option(config, section, 'description-file')
-                if filename:
+                if filenames:
-                    with open(filename) as fp:
+                    filenames = split_multiline(filenames)
-                        in_cfg_value = fp.read()
+                    in_cfg_value = []
+                    for filename in filenames:
+                        with open(filename) as fp:
+                            in_cfg_value.append(fp.read())
+                    in_cfg_value = '\n\n'.join(in_cfg_value)
             else:
                 continue
 
+        if arg == 'package_dir' and in_cfg_value:
+            in_cfg_value = {'': in_cfg_value}
+
         if arg in MULTI_FIELDS:
             # support multiline options
-            in_cfg_value = in_cfg_value.strip().split('\n')
+            in_cfg_value = split_multiline(in_cfg_value)
 
         kwargs[arg] = in_cfg_value
 

Misc/ACKS

@@ -116,6 +116,7 @@ Monty Brandenberg
 Georg Brandl
 Christopher Brannon
 Terrence Brannon
+Erik Bray
 Brian Brazil
 Dave Brennan
 Tom Bridgman

Misc/NEWS

@@ -187,6 +187,13 @@ Core and Builtins
 Library
 -------
 
+- Issue #12240: Allow multiple setup hooks in packaging's setup.cfg files.
+  Original patch by Erik Bray.
+
+- Issue #11595: Fix assorted bugs in packaging.util.cfg_to_args, a
+  compatibility helper for the distutils-packaging transition.  Original patch
+  by Erik Bray.
+
 - Issue #12287: In ossaudiodev, check that the device isn't closed in several
   methods.