mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-02 03:01:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 15

Issue #16314: Added support for the LZMA compression in distutils.

Browse source
This commit is contained in:
Serhiy Storchaka 2015-05-16 22:13:27 +03:00
parent b8cd3e4e30
commit b9cec6a30f
10 changed files with 180 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

32
Doc/distutils/apiref.rst
View file

@ -868,23 +868,31 @@ tarballs or zipfiles.
Create an archive file (eg. ``zip`` or ``tar``). *base_name* is the name of
the file to create, minus any format-specific extension; *format* is the
archive format: one of ``zip``, ``tar``, ``ztar``, or ``gztar``. *root_dir* is
a directory that will be the root directory of the archive; ie. we typically
``chdir`` into *root_dir* before creating the archive. *base_dir* is the
directory where we start archiving from; ie. *base_dir* will be the common
prefix of all files and directories in the archive. *root_dir* and *base_dir*
both default to the current directory. Returns the name of the archive file.
archive format: one of ``zip``, ``tar``, ``gztar``, ``bztar``, ``xztar``, or
``ztar``. *root_dir* is a directory that will be the root directory of the
archive; ie. we typically ``chdir`` into *root_dir* before creating the
archive. *base_dir* is the directory where we start archiving from; ie.
*base_dir* will be the common prefix of all files and directories in the
archive. *root_dir* and *base_dir* both default to the current directory.
Returns the name of the archive file.
.. versionchanged: 3.5
Added support for the ``xztar`` format.
.. function:: make_tarball(base_name, base_dir[, compress='gzip', verbose=0, dry_run=0])
'Create an (optional compressed) archive as a tar file from all files in and
under *base_dir*. *compress* must be ``'gzip'`` (the default), ``'compress'``,
``'bzip2'``, or ``None``. Both :program:`tar` and the compression utility named
by *compress* must be on the default program search path, so this is probably
Unix-specific. The output tar file will be named :file:`base_dir.tar`,
possibly plus the appropriate compression extension (:file:`.gz`, :file:`.bz2`
or :file:`.Z`). Return the output filename.
under *base_dir*. *compress* must be ``'gzip'`` (the default),
``'bzip2'``, ``'xz'``, ``'compress'``, or ``None``. For the ``'compress'``
method the compression utility named by :program:`compress` must be on the
default program search path, so this is probably Unix-specific. The output
tar file will be named :file:`base_dir.tar`, possibly plus the appropriate
compression extension (``.gz``, ``.bz2``, ``.xz`` or ``.Z``). Return the
output filename.
.. versionchanged: 3.5
Added support for the ``xz`` compression.
.. function:: make_zipfile(base_name, base_dir[, verbose=0, dry_run=0])

45
Doc/distutils/builtdist.rst
View file

@ -72,13 +72,19 @@ The available formats for built distributions are:
+-------------+------------------------------+---------+
| Format | Description | Notes |
+=============+==============================+=========+
| ``gztar`` | gzipped tar file | (1),(3) |
| ``gztar`` | gzipped tar file | \(1) |
| | (:file:`.tar.gz`) | |
+-------------+------------------------------+---------+
| ``bztar`` | bzipped tar file | |
| | (:file:`.tar.bz2`) | |
+-------------+------------------------------+---------+
| ``xztar`` | xzipped tar file | |
| | (:file:`.tar.xz`) | |
+-------------+------------------------------+---------+
| ``ztar`` | compressed tar file | \(3) |
| | (:file:`.tar.Z`) | |
+-------------+------------------------------+---------+
| ``tar`` | tar file (:file:`.tar`) | \(3) |
| ``tar`` | tar file (:file:`.tar`) | |
+-------------+------------------------------+---------+
| ``zip`` | zip file (:file:`.zip`) | (2),(4) |
+-------------+------------------------------+---------+
@ -94,6 +100,9 @@ The available formats for built distributions are:
| ``msi`` | Microsoft Installer. | |
+-------------+------------------------------+---------+
.. versionchanged: 3.5
Added support for the ``xztar`` format.
Notes:
@ -104,8 +113,7 @@ Notes:
default on Windows
(3)
requires external utilities: :program:`tar` and possibly one of :program:`gzip`,
:program:`bzip2`, or :program:`compress`
requires external :program:`compress` utility.
(4)
requires either external :program:`zip` utility or :mod:`zipfile` module (part
@ -119,21 +127,22 @@ You don't have to use the :command:`bdist` command with the :option:`--formats`
option; you can also use the command that directly implements the format you're
interested in. Some of these :command:`bdist` "sub-commands" actually generate
several similar formats; for instance, the :command:`bdist_dumb` command
generates all the "dumb" archive formats (``tar``, ``ztar``, ``gztar``, and
``zip``), and :command:`bdist_rpm` generates both binary and source RPMs. The
:command:`bdist` sub-commands, and the formats generated by each, are:
generates all the "dumb" archive formats (``tar``, ``gztar``, ``bztar``,
``xztar``, ``ztar``, and ``zip``), and :command:`bdist_rpm` generates both
binary and source RPMs. The :command:`bdist` sub-commands, and the formats
generated by each, are:
+--------------------------+-----------------------+
| Command | Formats |
+==========================+=======================+
| :command:`bdist_dumb` | tar, ztar, gztar, zip |
+--------------------------+-----------------------+
| :command:`bdist_rpm` | rpm, srpm |
+--------------------------+-----------------------+
| :command:`bdist_wininst` | wininst |
+--------------------------+-----------------------+
| :command:`bdist_msi` | msi |
+--------------------------+-----------------------+
+--------------------------+-------------------------------------+
| Command | Formats |
+==========================+=====================================+
| :command:`bdist_dumb` | tar, gztar, bztar, xztar, ztar, zip |
+--------------------------+-------------------------------------+
| :command:`bdist_rpm` | rpm, srpm |
+--------------------------+-------------------------------------+
| :command:`bdist_wininst` | wininst |
+--------------------------+-------------------------------------+
| :command:`bdist_msi` | msi |
+--------------------------+-------------------------------------+
The following sections give details on the individual :command:`bdist_\*`
commands.

8
Doc/distutils/sourcedist.rst
View file

@ -32,12 +32,18 @@ to create a gzipped tarball and a zip file. The available formats are:
| ``bztar`` | bzip2'ed tar file | |
| | (:file:`.tar.bz2`) | |
+-----------+-------------------------+---------+
| ``xztar`` | xz'ed tar file | |
| | (:file:`.tar.xz`) | |
+-----------+-------------------------+---------+
| ``ztar`` | compressed tar file | \(4) |
| | (:file:`.tar.Z`) | |
+-----------+-------------------------+---------+
| ``tar`` | tar file (:file:`.tar`) | |
+-----------+-------------------------+---------+
.. versionchanged: 3.5
Added support for the ``xztar`` format.
Notes:
(1)
@ -54,7 +60,7 @@ Notes:
requires the :program:`compress` program. Notice that this format is now
pending for deprecation and will be removed in the future versions of Python.
When using any ``tar`` format (``gztar``, ``bztar``, ``ztar`` or
When using any ``tar`` format (``gztar``, ``bztar``, ``xztar``, ``ztar`` or
``tar``), under Unix you can specify the ``owner`` and ``group`` names
that will be set for each member of the archive.

3
Doc/whatsnew/3.5.rst
View file

@ -338,6 +338,9 @@ distutils
option to enable parallel building of extension modules.
(Contributed by Antoine Pitrou in :issue:`5309`.)
* Added support for the LZMA compression.
(Contributed by Serhiy Storchaka in :issue:`16314`.)
doctest
-------

21
Lib/distutils/archive_util.py
View file

@ -57,26 +57,28 @@ def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
"""Create a (possibly compressed) tar file from all the files under
'base_dir'.
'compress' must be "gzip" (the default), "compress", "bzip2", or None.
(compress will be deprecated in Python 3.2)
'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or
None. ("compress" will be deprecated in Python 3.2)
'owner' and 'group' can be used to define an owner and a group for the
archive that is being built. If not provided, the current owner and group
will be used.
The output tar file will be named 'base_dir' + ".tar", possibly plus
the appropriate compression extension (".gz", ".bz2" or ".Z").
the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z").
Returns the output filename.
"""
tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', None: '', 'compress': ''}
compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'compress': '.Z'}
tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '',
'compress': ''}
compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz',
'compress': '.Z'}
# flags for compression program, each element of list will be an argument
if compress is not None and compress not in compress_ext.keys():
raise ValueError(
"bad value for 'compress': must be None, 'gzip', 'bzip2' "
"or 'compress'")
"bad value for 'compress': must be None, 'gzip', 'bzip2', "
"'xz' or 'compress'")
archive_name = base_name + '.tar'
if compress != 'compress':
@ -177,6 +179,7 @@ def make_zipfile(base_name, base_dir, verbose=0, dry_run=0):
ARCHIVE_FORMATS = {
'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"),
'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"),
'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"),
'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"),
'tar': (make_tarball, [('compress', None)], "uncompressed tar file"),
'zip': (make_zipfile, [],"ZIP file")
@ -197,8 +200,8 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
"""Create an archive file (eg. zip or tar).
'base_name' is the name of the file to create, minus any format-specific
extension; 'format' is the archive format: one of "zip", "tar", "ztar",
or "gztar".
extension; 'format' is the archive format: one of "zip", "tar", "gztar",
"bztar", "xztar", or "ztar".
'root_dir' is a directory that will be the root directory of the
archive; ie. we typically chdir into 'root_dir' before creating the

3
Lib/distutils/command/bdist.py
View file

@ -61,13 +61,14 @@ class bdist(Command):
'nt': 'zip'}
# Establish the preferred order (for the --help-formats option).
format_commands = ['rpm', 'gztar', 'bztar', 'ztar', 'tar',
format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar',
'wininst', 'zip', 'msi']
# And the real information.
format_command = {'rpm': ('bdist_rpm', "RPM distribution"),
'gztar': ('bdist_dumb', "gzip'ed tar file"),
'bztar': ('bdist_dumb', "bzip2'ed tar file"),
'xztar': ('bdist_dumb', "xz'ed tar file"),
'ztar': ('bdist_dumb', "compressed tar file"),
'tar': ('bdist_dumb', "tar file"),
'wininst': ('bdist_wininst',

3
Lib/distutils/command/bdist_dumb.py
View file

@ -22,7 +22,8 @@ class bdist_dumb(Command):
"platform name to embed in generated filenames "
"(default: %s)" % get_platform()),
('format=', 'f',
"archive format to create (tar, ztar, gztar, zip)"),
"archive format to create (tar, gztar, bztar, xztar, "
"ztar, zip)"),
('keep-temp', 'k',
"keep the pseudo-installation tree around after " +
"creating the distribution archive"),

154
Lib/distutils/tests/test_archive_util.py
View file

@ -13,7 +13,7 @@ from distutils.archive_util import (check_archive_formats, make_tarball,
ARCHIVE_FORMATS)
from distutils.spawn import find_executable, spawn
from distutils.tests import support
from test.support import check_warnings, run_unittest, patch
from test.support import check_warnings, run_unittest, patch, change_cwd
try:
import grp
@ -34,6 +34,16 @@ try:
except ImportError:
ZLIB_SUPPORT = False
try:
import bz2
except ImportError:
bz2 = None
try:
import lzma
except ImportError:
lzma = None
def can_fs_encode(filename):
"""
Return True if the filename can be saved in the file system.
@ -52,19 +62,36 @@ class ArchiveUtilTestCase(support.TempdirManager,
unittest.TestCase):
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
def test_make_tarball(self):
self._make_tarball('archive')
def test_make_tarball(self, name='archive'):
# creating something to tar
tmpdir = self._create_files()
self._make_tarball(tmpdir, name, '.tar.gz')
# trying an uncompressed one
self._make_tarball(tmpdir, name, '.tar', compress=None)
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
def test_make_tarball_gzip(self):
tmpdir = self._create_files()
self._make_tarball(tmpdir, 'archive', '.tar.gz', compress='gzip')
@unittest.skipUnless(bz2, 'Need bz2 support to run')
def test_make_tarball_bzip2(self):
tmpdir = self._create_files()
self._make_tarball(tmpdir, 'archive', '.tar.bz2', compress='bzip2')
@unittest.skipUnless(lzma, 'Need lzma support to run')
def test_make_tarball_xz(self):
tmpdir = self._create_files()
self._make_tarball(tmpdir, 'archive', '.tar.xz', compress='xz')
@unittest.skipUnless(can_fs_encode('årchiv'),
'File system cannot handle this filename')
def test_make_tarball_latin1(self):
"""
Mirror test_make_tarball, except filename contains latin characters.
"""
self._make_tarball('årchiv') # note this isn't a real word
self.test_make_tarball('årchiv') # note this isn't a real word
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
@unittest.skipUnless(can_fs_encode('のアーカイブ'),
'File system cannot handle this filename')
def test_make_tarball_extended(self):
@ -72,16 +99,9 @@ class ArchiveUtilTestCase(support.TempdirManager,
Mirror test_make_tarball, except filename contains extended
characters outside the latin charset.
"""
self._make_tarball('のアーカイブ') # japanese for archive
def _make_tarball(self, target_name):
# creating something to tar
tmpdir = self.mkdtemp()
self.write_file([tmpdir, 'file1'], 'xxx')
self.write_file([tmpdir, 'file2'], 'xxx')
os.mkdir(os.path.join(tmpdir, 'sub'))
self.write_file([tmpdir, 'sub', 'file3'], 'xxx')
self.test_make_tarball('のアーカイブ') # japanese for archive
def _make_tarball(self, tmpdir, target_name, suffix, **kwargs):
tmpdir2 = self.mkdtemp()
unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0],
"source and target should be on same drive")
@ -89,27 +109,13 @@ class ArchiveUtilTestCase(support.TempdirManager,
base_name = os.path.join(tmpdir2, target_name)
# working with relative paths to avoid tar warnings
old_dir = os.getcwd()
os.chdir(tmpdir)
try:
make_tarball(splitdrive(base_name)[1], '.')
finally:
os.chdir(old_dir)
with change_cwd(tmpdir):
make_tarball(splitdrive(base_name)[1], 'dist', **kwargs)
# check if the compressed tarball was created
tarball = base_name + '.tar.gz'
self.assertTrue(os.path.exists(tarball))
# trying an uncompressed one
base_name = os.path.join(tmpdir2, target_name)
old_dir = os.getcwd()
os.chdir(tmpdir)
try:
make_tarball(splitdrive(base_name)[1], '.', compress=None)
finally:
os.chdir(old_dir)
tarball = base_name + '.tar'
tarball = base_name + suffix
self.assertTrue(os.path.exists(tarball))
self.assertEqual(self._tarinfo(tarball), self._created_files)
def _tarinfo(self, path):
tar = tarfile.open(path)
@ -120,6 +126,9 @@ class ArchiveUtilTestCase(support.TempdirManager,
finally:
tar.close()
_created_files = ('dist', 'dist/file1', 'dist/file2',
'dist/sub', 'dist/sub/file3', 'dist/sub2')
def _create_files(self):
# creating something to tar
tmpdir = self.mkdtemp()
@ -130,15 +139,15 @@ class ArchiveUtilTestCase(support.TempdirManager,
os.mkdir(os.path.join(dist, 'sub'))
self.write_file([dist, 'sub', 'file3'], 'xxx')
os.mkdir(os.path.join(dist, 'sub2'))
tmpdir2 = self.mkdtemp()
base_name = os.path.join(tmpdir2, 'archive')
return tmpdir, tmpdir2, base_name
return tmpdir
@unittest.skipUnless(find_executable('tar') and find_executable('gzip')
and ZLIB_SUPPORT,
'Need the tar, gzip and zlib command to run')
def test_tarfile_vs_tar(self):
tmpdir, tmpdir2, base_name = self._create_files()
tmpdir = self._create_files()
tmpdir2 = self.mkdtemp()
base_name = os.path.join(tmpdir2, 'archive')
old_dir = os.getcwd()
os.chdir(tmpdir)
try:
@ -164,7 +173,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
self.assertTrue(os.path.exists(tarball2))
# let's compare both tarballs
self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2))
self.assertEqual(self._tarinfo(tarball), self._created_files)
self.assertEqual(self._tarinfo(tarball2), self._created_files)
# trying an uncompressed one
base_name = os.path.join(tmpdir2, 'archive')
@ -191,7 +201,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
@unittest.skipUnless(find_executable('compress'),
'The compress program is required')
def test_compress_deprecated(self):
tmpdir, tmpdir2, base_name = self._create_files()
tmpdir = self._create_files()
base_name = os.path.join(self.mkdtemp(), 'archive')
# using compress and testing the PendingDeprecationWarning
old_dir = os.getcwd()
@ -224,17 +235,17 @@ class ArchiveUtilTestCase(support.TempdirManager,
'Need zip and zlib support to run')
def test_make_zipfile(self):
# creating something to tar
tmpdir = self.mkdtemp()
self.write_file([tmpdir, 'file1'], 'xxx')
self.write_file([tmpdir, 'file2'], 'xxx')
tmpdir2 = self.mkdtemp()
base_name = os.path.join(tmpdir2, 'archive')
make_zipfile(base_name, tmpdir)
tmpdir = self._create_files()
base_name = os.path.join(self.mkdtemp(), 'archive')
with change_cwd(tmpdir):
make_zipfile(base_name, 'dist')
# check if the compressed tarball was created
tarball = base_name + '.zip'
self.assertTrue(os.path.exists(tarball))
with zipfile.ZipFile(tarball) as zf:
self.assertEqual(sorted(zf.namelist()),
['dist/file1', 'dist/file2', 'dist/sub/file3'])
@unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run')
def test_make_zipfile_no_zlib(self):
@ -250,18 +261,24 @@ class ArchiveUtilTestCase(support.TempdirManager,
patch(self, archive_util.zipfile, 'ZipFile', fake_zipfile)
# create something to tar and compress
tmpdir, tmpdir2, base_name = self._create_files()
make_zipfile(base_name, tmpdir)
tmpdir = self._create_files()
base_name = os.path.join(self.mkdtemp(), 'archive')
with change_cwd(tmpdir):
make_zipfile(base_name, 'dist')
tarball = base_name + '.zip'
self.assertEqual(called,
[((tarball, "w"), {'compression': zipfile.ZIP_STORED})])
self.assertTrue(os.path.exists(tarball))
with zipfile.ZipFile(tarball) as zf:
self.assertEqual(sorted(zf.namelist()),
['dist/file1', 'dist/file2', 'dist/sub/file3'])
def test_check_archive_formats(self):
self.assertEqual(check_archive_formats(['gztar', 'xxx', 'zip']),
'xxx')
self.assertEqual(check_archive_formats(['gztar', 'zip']), None)
self.assertIsNone(check_archive_formats(['gztar', 'bztar', 'xztar',
'ztar', 'tar', 'zip']))
def test_make_archive(self):
tmpdir = self.mkdtemp()
@ -282,6 +299,41 @@ class ArchiveUtilTestCase(support.TempdirManager,
finally:
del ARCHIVE_FORMATS['xxx']
def test_make_archive_tar(self):
base_dir = self._create_files()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'tar', base_dir, 'dist')
self.assertTrue(os.path.exists(res))
self.assertEqual(os.path.basename(res), 'archive.tar')
self.assertEqual(self._tarinfo(res), self._created_files)
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
def test_make_archive_gztar(self):
base_dir = self._create_files()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'gztar', base_dir, 'dist')
self.assertTrue(os.path.exists(res))
self.assertEqual(os.path.basename(res), 'archive.tar.gz')
self.assertEqual(self._tarinfo(res), self._created_files)
@unittest.skipUnless(bz2, 'Need bz2 support to run')
def test_make_archive_bztar(self):
base_dir = self._create_files()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'bztar', base_dir, 'dist')
self.assertTrue(os.path.exists(res))
self.assertEqual(os.path.basename(res), 'archive.tar.bz2')
self.assertEqual(self._tarinfo(res), self._created_files)
@unittest.skipUnless(bz2, 'Need xz support to run')
def test_make_archive_xztar(self):
base_dir = self._create_files()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'xztar', base_dir, 'dist')
self.assertTrue(os.path.exists(res))
self.assertEqual(os.path.basename(res), 'archive.tar.xz')
self.assertEqual(self._tarinfo(res), self._created_files)
def test_make_archive_owner_group(self):
# testing make_archive with owner and group, with various combinations
# this works even if there's not gid/uid support
@ -291,7 +343,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
else:
group = owner = 'root'
base_dir, root_dir, base_name = self._create_files()
base_dir = self._create_files()
root_dir = self.mkdtemp()
base_name = os.path.join(self.mkdtemp() , 'archive')
res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner,
group=group)
@ -311,7 +364,8 @@ class ArchiveUtilTestCase(support.TempdirManager,
@unittest.skipUnless(ZLIB_SUPPORT, "Requires zlib")
@unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
def test_tarfile_root_owner(self):
tmpdir, tmpdir2, base_name = self._create_files()
tmpdir = self._create_files()
base_name = os.path.join(self.mkdtemp(), 'archive')
old_dir = os.getcwd()
os.chdir(tmpdir)
group = grp.getgrgid(0)[0]

2
Lib/distutils/tests/test_bdist.py
View file

@ -21,7 +21,7 @@ class BuildTestCase(support.TempdirManager,
# what formats does bdist offer?
formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar',
'wininst', 'zip', 'ztar']
'wininst', 'xztar', 'zip', 'ztar']
found = sorted(cmd.format_command)
self.assertEqual(found, formats)

2
Misc/NEWS
View file

@ -47,6 +47,8 @@ Core and Builtins
Library
-------
- Issue #16314: Added support for the LZMA compression in distutils.
- Issue #21804: poplib now supports RFC 6856 (UTF8).
- Issue #18682: Optimized pprint functions for builtin scalar types.