Issue #4073: Add 2to3 support to build_scripts, refactor that support

in build_py.

Demo/distutils/test2to3/setup.py

@@ -6,6 +6,11 @@ try:
 except ImportError:
     from distutils.command.build_py import build_py
 
+try:
+    from distutils.command.build_scripts import build_scripts_2to3 as build_scripts
+except ImportError:
+    from distutils.command.build_scripts import build_scripts
+
 setup(
     name = "test2to3",
     version = "1.0",
@@ -14,5 +19,8 @@ setup(
     author_email = "python-dev@python.org",
     license = "PSF license",
     packages = ["test2to3"],
-    cmdclass = {'build_py':build_py}
+    scripts = ["maintest.py"],
+    cmdclass = {'build_py':build_py,
+                'build_scripts':build_scripts,
+                }
 )

Lib/distutils/command/build_py.py

@@ -9,7 +9,7 @@ from glob import glob
 
 from distutils.core import Command
 from distutils.errors import *
-from distutils.util import convert_path
+from distutils.util import convert_path, Mixin2to3
 from distutils import log
 
 class build_py (Command):
@@ -384,19 +384,7 @@ class build_py (Command):
             byte_compile(files, optimize=self.optimize,
                          force=self.force, prefix=prefix, dry_run=self.dry_run)
 
-from lib2to3.refactor import RefactoringTool, get_fixers_from_package
-class DistutilsRefactoringTool(RefactoringTool):
-    def log_error(self, msg, *args, **kw):
-        # XXX ignores kw
-        log.error(msg, *args)
-
-    def log_message(self, msg, *args):
-        log.info(msg, *args)
-
-    def log_debug(self, msg, *args):
-        log.debug(msg, *args)
-
-class build_py_2to3(build_py):
+class build_py_2to3(build_py, Mixin2to3):
     def run(self):
         self.updated_files = []
 
@@ -408,12 +396,7 @@ class build_py_2to3(build_py):
             self.build_package_data()
 
         # 2to3
-        fixers = get_fixers_from_package('lib2to3.fixes')
-        options = dict(fix=[], list_fixes=[],
-                       print_function=False, verbose=False,
-                       write=True)
-        r = DistutilsRefactoringTool(fixers, options)
-        r.refactor(self.updated_files, write=True)
+        self.run_2to3(self.updated_files)
 
         # Remaining base class code
         self.byte_compile(self.get_outputs(include_bytecode=0))

Lib/distutils/command/build_scripts.py

@@ -9,7 +9,7 @@ from stat import ST_MODE
 from distutils import sysconfig
 from distutils.core import Command
 from distutils.dep_util import newer
-from distutils.util import convert_path
+from distutils.util import convert_path, Mixin2to3
 from distutils import log
 
 # check if Python is called on the first line with this expression
@@ -59,6 +59,7 @@ class build_scripts(Command):
         """
         self.mkpath(self.build_dir)
         outfiles = []
+        updated_files = []
         for script in self.scripts:
             adjust = False
             script = convert_path(script)
@@ -92,6 +93,7 @@ class build_scripts(Command):
             if adjust:
                 log.info("copying and adjusting %s -> %s", script,
                          self.build_dir)
+                updated_files.append(outfile)
                 if not self.dry_run:
                     outf = open(outfile, "w")
                     if not sysconfig.python_build:
@@ -112,6 +114,7 @@ class build_scripts(Command):
             else:
                 if f:
                     f.close()
+                updated_files.append(outfile)
                 self.copy_file(script, outfile)
 
         if os.name == 'posix':
@@ -125,3 +128,13 @@ class build_scripts(Command):
                         log.info("changing mode of %s from %o to %o",
                                  file, oldmode, newmode)
                         os.chmod(file, newmode)
+        # XXX should we modify self.outfiles?
+        return outfiles, updated_files
+
+class build_scripts_2to3(build_scripts, Mixin2to3):
+
+    def copy_scripts(self):
+        outfiles, updated_files = build_scripts.copy_scripts(self)
+        if not self.dry_run:
+            self.run_2to3(updated_files)
+        return outfiles, updated_files

Lib/distutils/util.py

@@ -531,3 +531,51 @@ def rfc822_escape (header):
     lines = [x.strip() for x in header.split('\n')]
     sep = '\n' + 8*' '
     return sep.join(lines)
+
+# 2to3 support
+
+def run_2to3(files, fixer_names=None, options=None, explicit=None):
+    """Invoke 2to3 on a list of Python files.
+    The files should all come from the build area, as the
+    modification is done in-place. To reduce the build time,
+    only files modified since the last invocation of this
+    function should be passed in the files argument."""
+
+    if not files:
+        return
+
+    # Make this class local, to delay import of 2to3
+    from lib2to3.refactor import RefactoringTool, get_fixers_from_package
+    class DistutilsRefactoringTool(RefactoringTool):
+        def log_error(self, msg, *args, **kw):
+            log.error(msg, *args)
+
+        def log_message(self, msg, *args):
+            log.info(msg, *args)
+
+        def log_debug(self, msg, *args):
+            log.debug(msg, *args)
+
+    if fixer_names is None:
+        fixer_names = get_fixers_from_package('lib2to3.fixes')
+    r = DistutilsRefactoringTool(fixer_names, options=options)
+    r.refactor(files, write=True)
+
+class Mixin2to3:
+    '''Mixin class for commands that run 2to3.
+    To configure 2to3, setup scripts may either change
+    the class variables, or inherit from individual commands
+    to override how 2to3 is invoked.'''
+
+    # provide list of fixers to run;
+    # defaults to all from lib2to3.fixers
+    fixer_names = None
+
+    # options dictionary
+    options = None
+
+    # list of fixers to invoke even though they are marked as explicit
+    explicit = None
+
+    def run_2to3(self, files):
+        return run_2to3(files, self.fixer_names, self.options, self.explicit)

Misc/NEWS

@@ -24,6 +24,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #4073: Add 2to3 support to build_scripts, refactor that support
+  in build_py.
+
 - IDLE would print a "Unhandled server exception!" message when internal
   debugging is enabled.