mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-07 19:35:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-132983: Add the compression.zstd pacakge and tests (#133365)

Browse source 
Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
Co-authored-by: Gregory P. Smith <greg@krypto.org>
Co-authored-by: Tomas R. <tomas.roun8@gmail.com>
Co-authored-by: Rogdham <contact@rogdham.net>
This commit is contained in:
Emma Smith 2025-05-05 17:38:08 -07:00 • committed by GitHub
parent 793402e217
commit c273f59fb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 3358 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

234
Lib/compression/zstd/__init__.py Normal file
View file

@ -0,0 +1,234 @@
"""Python bindings to the Zstandard (zstd) compression library (RFC-8878)."""
__all__ = (
# compression.zstd
"COMPRESSION_LEVEL_DEFAULT",
"compress",
"CompressionParameter",
"decompress",
"DecompressionParameter",
"finalize_dict",
"get_frame_info",
"Strategy",
"train_dict",
# compression.zstd._zstdfile
"open",
"ZstdFile",
# _zstd
"get_frame_size",
"zstd_version",
"zstd_version_info",
"ZstdCompressor",
"ZstdDecompressor",
"ZstdDict",
"ZstdError",
)
import _zstd
import enum
from _zstd import *
from compression.zstd._zstdfile import ZstdFile, open, _nbytes
COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0]
"""The default compression level for Zstandard, currently '3'."""
class FrameInfo:
"""Information about a Zstandard frame."""
__slots__ = 'decompressed_size', 'dictionary_id'
def __init__(self, decompressed_size, dictionary_id):
super().__setattr__('decompressed_size', decompressed_size)
super().__setattr__('dictionary_id', dictionary_id)
def __repr__(self):
return (f'FrameInfo(decompressed_size={self.decompressed_size}, '
f'dictionary_id={self.dictionary_id})')
def __setattr__(self, name, _):
raise AttributeError(f"can't set attribute {name!r}")
def get_frame_info(frame_buffer):
"""Get Zstandard frame information from a frame header.
*frame_buffer* is a bytes-like object. It should start from the beginning
of a frame, and needs to include at least the frame header (6 to 18 bytes).
The returned FrameInfo object has two attributes.
'decompressed_size' is the size in bytes of the data in the frame when
decompressed, or None when the decompressed size is unknown.
'dictionary_id' is an int in the range (0, 2**32). The special value 0
means that the dictionary ID was not recorded in the frame header,
the frame may or may not need a dictionary to be decoded,
and the ID of such a dictionary is not specified.
"""
return FrameInfo(*_zstd._get_frame_info(frame_buffer))
def train_dict(samples, dict_size):
"""Return a ZstdDict representing a trained Zstandard dictionary.
*samples* is an iterable of samples, where a sample is a bytes-like
object representing a file.
*dict_size* is the dictionary's maximum size, in bytes.
"""
if not isinstance(dict_size, int):
ds_cls = type(dict_size).__qualname__
raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.')
samples = tuple(samples)
chunks = b''.join(samples)
chunk_sizes = tuple(_nbytes(sample) for sample in samples)
if not chunks:
raise ValueError("samples contained no data; can't train dictionary.")
dict_content = _zstd._train_dict(chunks, chunk_sizes, dict_size)
return ZstdDict(dict_content)
def finalize_dict(zstd_dict, /, samples, dict_size, level):
"""Return a ZstdDict representing a finalized Zstandard dictionary.
Given a custom content as a basis for dictionary, and a set of samples,
finalize *zstd_dict* by adding headers and statistics according to the
Zstandard dictionary format.
You may compose an effective dictionary content by hand, which is used as
basis dictionary, and use some samples to finalize a dictionary. The basis
dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict.
*samples* is an iterable of samples, where a sample is a bytes-like object
representing a file.
*dict_size* is the dictionary's maximum size, in bytes.
*level* is the expected compression level. The statistics for each
compression level differ, so tuning the dictionary to the compression level
can provide improvements.
"""
if not isinstance(zstd_dict, ZstdDict):
raise TypeError('zstd_dict argument should be a ZstdDict object.')
if not isinstance(dict_size, int):
raise TypeError('dict_size argument should be an int object.')
if not isinstance(level, int):
raise TypeError('level argument should be an int object.')
samples = tuple(samples)
chunks = b''.join(samples)
chunk_sizes = tuple(_nbytes(sample) for sample in samples)
if not chunks:
raise ValueError("The samples are empty content, can't finalize the"
"dictionary.")
dict_content = _zstd._finalize_dict(zstd_dict.dict_content,
chunks, chunk_sizes,
dict_size, level)
return ZstdDict(dict_content)
def compress(data, level=None, options=None, zstd_dict=None):
"""Return Zstandard compressed *data* as bytes.
*level* is an int specifying the compression level to use, defaulting to
COMPRESSION_LEVEL_DEFAULT ('3').
*options* is a dict object that contains advanced compression
parameters. See CompressionParameter for more on options.
*zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See
the function train_dict for how to train a ZstdDict on sample data.
For incremental compression, use a ZstdCompressor instead.
"""
comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict)
return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME)
def decompress(data, zstd_dict=None, options=None):
"""Decompress one or more frames of Zstandard compressed *data*.
*zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See
the function train_dict for how to train a ZstdDict on sample data.
*options* is a dict object that contains advanced compression
parameters. See DecompressionParameter for more on options.
For incremental decompression, use a ZstdDecompressor instead.
"""
results = []
while True:
decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict)
results.append(decomp.decompress(data))
if not decomp.eof:
raise ZstdError("Compressed data ended before the "
"end-of-stream marker was reached")
data = decomp.unused_data
if not data:
break
return b"".join(results)
class CompressionParameter(enum.IntEnum):
"""Compression parameters."""
compression_level = _zstd._ZSTD_c_compressionLevel
window_log = _zstd._ZSTD_c_windowLog
hash_log = _zstd._ZSTD_c_hashLog
chain_log = _zstd._ZSTD_c_chainLog
search_log = _zstd._ZSTD_c_searchLog
min_match = _zstd._ZSTD_c_minMatch
target_length = _zstd._ZSTD_c_targetLength
strategy = _zstd._ZSTD_c_strategy
enable_long_distance_matching = _zstd._ZSTD_c_enableLongDistanceMatching
ldm_hash_log = _zstd._ZSTD_c_ldmHashLog
ldm_min_match = _zstd._ZSTD_c_ldmMinMatch
ldm_bucket_size_log = _zstd._ZSTD_c_ldmBucketSizeLog
ldm_hash_rate_log = _zstd._ZSTD_c_ldmHashRateLog
content_size_flag = _zstd._ZSTD_c_contentSizeFlag
checksum_flag = _zstd._ZSTD_c_checksumFlag
dict_id_flag = _zstd._ZSTD_c_dictIDFlag
nb_workers = _zstd._ZSTD_c_nbWorkers
job_size = _zstd._ZSTD_c_jobSize
overlap_log = _zstd._ZSTD_c_overlapLog
def bounds(self):
"""Return the (lower, upper) int bounds of a compression parameter.
Both the lower and upper bounds are inclusive.
"""
return _zstd._get_param_bounds(self.value, is_compress=True)
class DecompressionParameter(enum.IntEnum):
"""Decompression parameters."""
window_log_max = _zstd._ZSTD_d_windowLogMax
def bounds(self):
"""Return the (lower, upper) int bounds of a decompression parameter.
Both the lower and upper bounds are inclusive.
"""
return _zstd._get_param_bounds(self.value, is_compress=False)
class Strategy(enum.IntEnum):
"""Compression strategies, listed from fastest to strongest.
Note that new strategies might be added in the future.
Only the order (from fast to strong) is guaranteed,
the numeric value might change.
"""
fast = _zstd._ZSTD_fast
dfast = _zstd._ZSTD_dfast
greedy = _zstd._ZSTD_greedy
lazy = _zstd._ZSTD_lazy
lazy2 = _zstd._ZSTD_lazy2
btlazy2 = _zstd._ZSTD_btlazy2
btopt = _zstd._ZSTD_btopt
btultra = _zstd._ZSTD_btultra
btultra2 = _zstd._ZSTD_btultra2
# Check validity of the CompressionParameter & DecompressionParameter types
_zstd._set_parameter_types(CompressionParameter, DecompressionParameter)

349
Lib/compression/zstd/_zstdfile.py Normal file
View file

@ -0,0 +1,349 @@
import io
from os import PathLike
from _zstd import (ZstdCompressor, ZstdDecompressor, _ZSTD_DStreamSizes,
ZstdError)
from compression._common import _streams
__all__ = ("ZstdFile", "open")
_ZSTD_DStreamOutSize = _ZSTD_DStreamSizes[1]
_MODE_CLOSED = 0
_MODE_READ = 1
_MODE_WRITE = 2
def _nbytes(dat, /):
if isinstance(dat, (bytes, bytearray)):
return len(dat)
with memoryview(dat) as mv:
return mv.nbytes
class ZstdFile(_streams.BaseStream):
"""A file-like object providing transparent Zstandard (de)compression.
A ZstdFile can act as a wrapper for an existing file object, or refer
directly to a named file on disk.
ZstdFile provides a *binary* file interface. Data is read and returned as
bytes, and may only be written to objects that support the Buffer Protocol.
"""
FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK
FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME
def __init__(self, file, /, mode="r", *,
level=None, options=None, zstd_dict=None):
"""Open a Zstandard compressed file in binary mode.
*file* can be either an file-like object, or a file name to open.
*mode* can be "r" for reading (default), "w" for (over)writing, "x" for
creating exclusively, or "a" for appending. These can equivalently be
given as "rb", "wb", "xb" and "ab" respectively.
*level* is an optional int specifying the compression level to use,
or COMPRESSION_LEVEL_DEFAULT if not given.
*options* is an optional dict for advanced compression parameters.
See CompressionParameter and DecompressionParameter for the possible
options.
*zstd_dict* is an optional ZstdDict object, a pre-trained Zstandard
dictionary. See train_dict() to train ZstdDict on sample data.
"""
self._fp = None
self._close_fp = False
self._mode = _MODE_CLOSED
self._buffer = None
if not isinstance(mode, str):
raise ValueError("mode must be a str")
if options is not None and not isinstance(options, dict):
raise TypeError("options must be a dict or None")
mode = mode.removesuffix("b") # handle rb, wb, xb, ab
if mode == "r":
if level is not None:
raise TypeError("level is illegal in read mode")
self._mode = _MODE_READ
elif mode in {"w", "a", "x"}:
if level is not None and not isinstance(level, int):
raise TypeError("level must be int or None")
self._mode = _MODE_WRITE
self._compressor = ZstdCompressor(level=level, options=options,
zstd_dict=zstd_dict)
self._pos = 0
else:
raise ValueError(f"Invalid mode: {mode!r}")
if isinstance(file, (str, bytes, PathLike)):
self._fp = io.open(file, f'{mode}b')
self._close_fp = True
elif ((mode == 'r' and hasattr(file, "read"))
or (mode != 'r' and hasattr(file, "write"))):
self._fp = file
else:
raise TypeError("file must be a file-like object "
"or a str, bytes, or PathLike object")
if self._mode == _MODE_READ:
raw = _streams.DecompressReader(
self._fp,
ZstdDecompressor,
trailing_error=ZstdError,
zstd_dict=zstd_dict,
options=options,
)
self._buffer = io.BufferedReader(raw)
def close(self):
"""Flush and close the file.
May be called multiple times. Once the file has been closed,
any other operation on it will raise ValueError.
"""
if self._fp is None:
return
try:
if self._mode == _MODE_READ:
if getattr(self, '_buffer', None):
self._buffer.close()
self._buffer = None
elif self._mode == _MODE_WRITE:
self.flush(self.FLUSH_FRAME)
self._compressor = None
finally:
self._mode = _MODE_CLOSED
try:
if self._close_fp:
self._fp.close()
finally:
self._fp = None
self._close_fp = False
def write(self, data, /):
"""Write a bytes-like object *data* to the file.
Returns the number of uncompressed bytes written, which is
always the length of data in bytes. Note that due to buffering,
the file on disk may not reflect the data written until .flush()
or .close() is called.
"""
self._check_can_write()
length = _nbytes(data)
compressed = self._compressor.compress(data)
self._fp.write(compressed)
self._pos += length
return length
def flush(self, mode=FLUSH_BLOCK):
"""Flush remaining data to the underlying stream.
The mode argument can be FLUSH_BLOCK or FLUSH_FRAME. Abuse of this
method will reduce compression ratio, use it only when necessary.
If the program is interrupted afterwards, all data can be recovered.
To ensure saving to disk, also need to use os.fsync(fd).
This method does nothing in reading mode.
"""
if self._mode == _MODE_READ:
return
self._check_not_closed()
if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}:
raise ValueError("Invalid mode argument, expected either "
"ZstdFile.FLUSH_FRAME or "
"ZstdFile.FLUSH_BLOCK")
if self._compressor.last_mode == mode:
return
# Flush zstd block/frame, and write.
data = self._compressor.flush(mode)
self._fp.write(data)
if hasattr(self._fp, "flush"):
self._fp.flush()
def read(self, size=-1):
"""Read up to size uncompressed bytes from the file.
If size is negative or omitted, read until EOF is reached.
Returns b"" if the file is already at EOF.
"""
if size is None:
size = -1
self._check_can_read()
return self._buffer.read(size)
def read1(self, size=-1):
"""Read up to size uncompressed bytes, while trying to avoid
making multiple reads from the underlying stream. Reads up to a
buffer's worth of data if size is negative.
Returns b"" if the file is at EOF.
"""
self._check_can_read()
if size < 0:
# Note this should *not* be io.DEFAULT_BUFFER_SIZE.
# ZSTD_DStreamOutSize is the minimum amount to read guaranteeing
# a full block is read.
size = _ZSTD_DStreamOutSize
return self._buffer.read1(size)
def readinto(self, b):
"""Read bytes into b.
Returns the number of bytes read (0 for EOF).
"""
self._check_can_read()
return self._buffer.readinto(b)
def readinto1(self, b):
"""Read bytes into b, while trying to avoid making multiple reads
from the underlying stream.
Returns the number of bytes read (0 for EOF).
"""
self._check_can_read()
return self._buffer.readinto1(b)
def readline(self, size=-1):
"""Read a line of uncompressed bytes from the file.
The terminating newline (if present) is retained. If size is
non-negative, no more than size bytes will be read (in which
case the line may be incomplete). Returns b'' if already at EOF.
"""
self._check_can_read()
return self._buffer.readline(size)
def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
The new position is specified by offset, relative to the
position indicated by whence. Possible values for whence are:
0: start of stream (default): offset must not be negative
1: current stream position
2: end of stream; offset must not be positive
Returns the new file position.
Note that seeking is emulated, so depending on the arguments,
this operation may be extremely slow.
"""
self._check_can_read()
# BufferedReader.seek() checks seekable
return self._buffer.seek(offset, whence)
def peek(self, size=-1):
"""Return buffered data without advancing the file position.
Always returns at least one byte of data, unless at EOF.
The exact number of bytes returned is unspecified.
"""
# Relies on the undocumented fact that BufferedReader.peek() always
# returns at least one byte (except at EOF)
self._check_can_read()
return self._buffer.peek(size)
def __next__(self):
if ret := self._buffer.readline():
return ret
raise StopIteration
def tell(self):
"""Return the current file position."""
self._check_not_closed()
if self._mode == _MODE_READ:
return self._buffer.tell()
elif self._mode == _MODE_WRITE:
return self._pos
def fileno(self):
"""Return the file descriptor for the underlying file."""
self._check_not_closed()
return self._fp.fileno()
@property
def name(self):
self._check_not_closed()
return self._fp.name
@property
def mode(self):
return 'wb' if self._mode == _MODE_WRITE else 'rb'
@property
def closed(self):
"""True if this file is closed."""
return self._mode == _MODE_CLOSED
def seekable(self):
"""Return whether the file supports seeking."""
return self.readable() and self._buffer.seekable()
def readable(self):
"""Return whether the file was opened for reading."""
self._check_not_closed()
return self._mode == _MODE_READ
def writable(self):
"""Return whether the file was opened for writing."""
self._check_not_closed()
return self._mode == _MODE_WRITE
def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None,
encoding=None, errors=None, newline=None):
"""Open a Zstandard compressed file in binary or text mode.
file can be either a file name (given as a str, bytes, or PathLike object),
in which case the named file is opened, or it can be an existing file object
to read from or write to.
The mode parameter can be "r", "rb" (default), "w", "wb", "x", "xb", "a",
"ab" for binary mode, or "rt", "wt", "xt", "at" for text mode.
The level, options, and zstd_dict parameters specify the settings the same
as ZstdFile.
When using read mode (decompression), the options parameter is a dict
representing advanced decompression options. The level parameter is not
supported in this case. When using write mode (compression), only one of
level, an int representing the compression level, or options, a dict
representing advanced compression options, may be passed. In both modes,
zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary.
For binary mode, this function is equivalent to the ZstdFile constructor:
ZstdFile(filename, mode, ...). In this case, the encoding, errors and
newline parameters must not be provided.
For text mode, an ZstdFile object is created, and wrapped in an
io.TextIOWrapper instance with the specified encoding, error handling
behavior, and line ending(s).
"""
text_mode = "t" in mode
mode = mode.replace("t", "")
if text_mode:
if "b" in mode:
raise ValueError(f"Invalid mode: {mode!r}")
else:
if encoding is not None:
raise ValueError("Argument 'encoding' not supported in binary mode")
if errors is not None:
raise ValueError("Argument 'errors' not supported in binary mode")
if newline is not None:
raise ValueError("Argument 'newline' not supported in binary mode")
binary_file = ZstdFile(file, mode, level=level, options=options,
zstd_dict=zstd_dict)
if text_mode:
return io.TextIOWrapper(binary_file, encoding, errors, newline)
else:
return binary_file

19
Lib/shutil.py
View file

@ -32,6 +32,13 @@ try:
except ImportError:
_LZMA_SUPPORTED = False
try:
from compression import zstd
del zstd
_ZSTD_SUPPORTED = True
except ImportError:
_ZSTD_SUPPORTED = False
_WINDOWS = os.name == 'nt'
posix = nt = None
if os.name == 'posix':
@ -1006,6 +1013,8 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
tar_compression = 'bz2'
elif _LZMA_SUPPORTED and compress == 'xz':
tar_compression = 'xz'
elif _ZSTD_SUPPORTED and compress == 'zst':
tar_compression = 'zst'
else:
raise ValueError("bad value for 'compress', or compression format not "
"supported : {0}".format(compress))
@ -1134,6 +1143,10 @@ if _LZMA_SUPPORTED:
_ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')],
"xz'ed tar-file")
if _ZSTD_SUPPORTED:
_ARCHIVE_FORMATS['zstdtar'] = (_make_tarball, [('compress', 'zst')],
"zstd'ed tar-file")
def get_archive_formats():
"""Returns a list of supported formats for archiving and unarchiving.
@ -1174,7 +1187,7 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
'base_name' is the name of the file to create, minus any format-specific
extension; 'format' is the archive format: one of "zip", "tar", "gztar",
"bztar", or "xztar". Or any other registered format.
"bztar", "zstdtar", or "xztar". Or any other registered format.
'root_dir' is a directory that will be the root directory of the
archive; ie. we typically chdir into 'root_dir' before creating the
@ -1359,6 +1372,10 @@ if _LZMA_SUPPORTED:
_UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [],
"xz'ed tar-file")
if _ZSTD_SUPPORTED:
_UNPACK_FORMATS['zstdtar'] = (['.tar.zst', '.tzst'], _unpack_tarfile, [],
"zstd'ed tar-file")
def _find_unpack_format(filename):
for name, info in _UNPACK_FORMATS.items():
for extension in info[0]:

61
Lib/tarfile.py
View file

@ -399,7 +399,17 @@ class _Stream:
self.exception = lzma.LZMAError
else:
self.cmp = lzma.LZMACompressor(preset=preset)
elif comptype == "zst":
try:
from compression import zstd
except ImportError:
raise CompressionError("compression.zstd module is not available") from None
if mode == "r":
self.dbuf = b""
self.cmp = zstd.ZstdDecompressor()
self.exception = zstd.ZstdError
else:
self.cmp = zstd.ZstdCompressor()
elif comptype != "tar":
raise CompressionError("unknown compression type %r" % comptype)
@ -591,6 +601,8 @@ class _StreamProxy(object):
return "bz2"
elif self.buf.startswith((b"\x5d\x00\x00\x80", b"\xfd7zXZ")):
return "xz"
elif self.buf.startswith(b"\x28\xb5\x2f\xfd"):
return "zst"
else:
return "tar"
@ -1817,11 +1829,13 @@ class TarFile(object):
'r:gz' open for reading with gzip compression
'r:bz2' open for reading with bzip2 compression
'r:xz' open for reading with lzma compression
'r:zst' open for reading with zstd compression
'a' or 'a:' open for appending, creating the file if necessary
'w' or 'w:' open for writing without compression
'w:gz' open for writing with gzip compression
'w:bz2' open for writing with bzip2 compression
'w:xz' open for writing with lzma compression
'w:zst' open for writing with zstd compression
'x' or 'x:' create a tarfile exclusively without compression, raise
an exception if the file is already created
@ -1831,16 +1845,20 @@ class TarFile(object):
if the file is already created
'x:xz' create an lzma compressed tarfile, raise an exception
if the file is already created
'x:zst' create a zstd compressed tarfile, raise an exception
if the file is already created
'r|*' open a stream of tar blocks with transparent compression
'r|' open an uncompressed stream of tar blocks for reading
'r|gz' open a gzip compressed stream of tar blocks
'r|bz2' open a bzip2 compressed stream of tar blocks
'r|xz' open an lzma compressed stream of tar blocks
'r|zst' open a zstd compressed stream of tar blocks
'w|' open an uncompressed stream for writing
'w|gz' open a gzip compressed stream for writing
'w|bz2' open a bzip2 compressed stream for writing
'w|xz' open an lzma compressed stream for writing
'w|zst' open a zstd compressed stream for writing
"""
if not name and not fileobj:
@ -2006,12 +2024,48 @@ class TarFile(object):
t._extfileobj = False
return t
@classmethod
def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None,
zstd_dict=None, **kwargs):
"""Open zstd compressed tar archive name for reading or writing.
Appending is not allowed.
"""
if mode not in ("r", "w", "x"):
raise ValueError("mode must be 'r', 'w' or 'x'")
try:
from compression.zstd import ZstdFile, ZstdError
except ImportError:
raise CompressionError("compression.zstd module is not available") from None
fileobj = ZstdFile(
fileobj or name,
mode,
level=level,
options=options,
zstd_dict=zstd_dict
)
try:
t = cls.taropen(name, mode, fileobj, **kwargs)
except (ZstdError, EOFError) as e:
fileobj.close()
if mode == 'r':
raise ReadError("not a zstd file") from e
raise
except Exception:
fileobj.close()
raise
t._extfileobj = False
return t
# All *open() methods are registered here.
OPEN_METH = {
"tar": "taropen", # uncompressed tar
"gz": "gzopen", # gzip compressed tar
"bz2": "bz2open", # bzip2 compressed tar
"xz": "xzopen" # lzma compressed tar
"xz": "xzopen", # lzma compressed tar
"zst": "zstopen" # zstd compressed tar
}
#--------------------------------------------------------------------------
@ -2963,6 +3017,9 @@ def main():
'.tbz': 'bz2',
'.tbz2': 'bz2',
'.tb2': 'bz2',
# zstd
'.zst': 'zst',
'.tzst': 'zst',
}
tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w'
tar_files = args.create

9
Lib/test/support/__init__.py
View file

@ -33,7 +33,7 @@ __all__ = [
"is_resource_enabled", "requires", "requires_freebsd_version",
"requires_gil_enabled", "requires_linux_version", "requires_mac_ver",
"check_syntax_error",
"requires_gzip", "requires_bz2", "requires_lzma",
"requires_gzip", "requires_bz2", "requires_lzma", "requires_zstd",
"bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute",
"requires_IEEE_754", "requires_zlib",
"has_fork_support", "requires_fork",
@ -527,6 +527,13 @@ def requires_lzma(reason='requires lzma'):
lzma = None
return unittest.skipUnless(lzma, reason)
def requires_zstd(reason='requires zstd'):
try:
from compression import zstd
except ImportError:
zstd = None
return unittest.skipUnless(zstd, reason)
def has_no_debug_ranges():
try:
import _testcapi

4
Lib/test/test_shutil.py
View file

@ -2153,6 +2153,10 @@ class TestArchives(BaseTest, unittest.TestCase):
def test_unpack_archive_bztar(self):
self.check_unpack_tarball('bztar')
@support.requires_zstd()
def test_unpack_archive_zstdtar(self):
self.check_unpack_tarball('zstdtar')
@support.requires_lzma()
@unittest.skipIf(AIX and not _maxdataOK(), "AIX MAXDATA must be 0x20000000 or larger")
def test_unpack_archive_xztar(self):

44
Lib/test/test_tarfile.py
View file

@ -38,6 +38,10 @@ try:
import lzma
except ImportError:
lzma = None
try:
from compression import zstd
except ImportError:
zstd = None
def sha256sum(data):
return sha256(data).hexdigest()
@ -48,6 +52,7 @@ tarname = support.findfile("testtar.tar", subdir="archivetestdata")
gzipname = os.path.join(TEMPDIR, "testtar.tar.gz")
bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2")
xzname = os.path.join(TEMPDIR, "testtar.tar.xz")
zstname = os.path.join(TEMPDIR, "testtar.tar.zst")
tmpname = os.path.join(TEMPDIR, "tmp.tar")
dotlessname = os.path.join(TEMPDIR, "testtar")
@ -90,6 +95,12 @@ class LzmaTest:
open = lzma.LZMAFile if lzma else None
taropen = tarfile.TarFile.xzopen
@support.requires_zstd()
class ZstdTest:
tarname = zstname
suffix = 'zst'
open = zstd.ZstdFile if zstd else None
taropen = tarfile.TarFile.zstopen
class ReadTest(TarTest):
@ -271,6 +282,8 @@ class Bz2UstarReadTest(Bz2Test, UstarReadTest):
class LzmaUstarReadTest(LzmaTest, UstarReadTest):
pass
class ZstdUstarReadTest(ZstdTest, UstarReadTest):
pass
class ListTest(ReadTest, unittest.TestCase):
@ -375,6 +388,8 @@ class Bz2ListTest(Bz2Test, ListTest):
class LzmaListTest(LzmaTest, ListTest):
pass
class ZstdListTest(ZstdTest, ListTest):
pass
class CommonReadTest(ReadTest):
@ -837,6 +852,8 @@ class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase):
class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase):
pass
class ZstdMiscReadTest(ZstdTest, MiscReadTestBase, unittest.TestCase):
pass
class StreamReadTest(CommonReadTest, unittest.TestCase):
@ -909,6 +926,9 @@ class Bz2StreamReadTest(Bz2Test, StreamReadTest):
class LzmaStreamReadTest(LzmaTest, StreamReadTest):
pass
class ZstdStreamReadTest(ZstdTest, StreamReadTest):
pass
class TarStreamModeReadTest(StreamModeTest, unittest.TestCase):
def test_stream_mode_no_cache(self):
@ -925,6 +945,9 @@ class Bz2StreamModeReadTest(Bz2Test, TarStreamModeReadTest):
class LzmaStreamModeReadTest(LzmaTest, TarStreamModeReadTest):
pass
class ZstdStreamModeReadTest(ZstdTest, TarStreamModeReadTest):
pass
class DetectReadTest(TarTest, unittest.TestCase):
def _testfunc_file(self, name, mode):
try:
@ -986,6 +1009,8 @@ class Bz2DetectReadTest(Bz2Test, DetectReadTest):
class LzmaDetectReadTest(LzmaTest, DetectReadTest):
pass
class ZstdDetectReadTest(ZstdTest, DetectReadTest):
pass
class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase):
"""
@ -1666,6 +1691,8 @@ class Bz2WriteTest(Bz2Test, WriteTest):
class LzmaWriteTest(LzmaTest, WriteTest):
pass
class ZstdWriteTest(ZstdTest, WriteTest):
pass
class StreamWriteTest(WriteTestBase, unittest.TestCase):
@ -1727,6 +1754,9 @@ class Bz2StreamWriteTest(Bz2Test, StreamWriteTest):
class LzmaStreamWriteTest(LzmaTest, StreamWriteTest):
decompressor = lzma.LZMADecompressor if lzma else None
class ZstdStreamWriteTest(ZstdTest, StreamWriteTest):
decompressor = zstd.ZstdDecompressor if zstd else None
class _CompressedWriteTest(TarTest):
# This is not actually a standalone test.
# It does not inherit WriteTest because it only makes sense with gz,bz2
@ -2042,6 +2072,14 @@ class LzmaCreateTest(LzmaTest, CreateTest):
tobj.add(self.file_path)
class ZstdCreateTest(ZstdTest, CreateTest):
# Unlike gz and bz2, zstd uses the level keyword instead of compresslevel.
# It does not allow for level to be specified when reading.
def test_create_with_level(self):
with tarfile.open(tmpname, self.mode, level=1) as tobj:
tobj.add(self.file_path)
class CreateWithXModeTest(CreateTest):
prefix = "x"
@ -2523,6 +2561,8 @@ class Bz2AppendTest(Bz2Test, AppendTestBase, unittest.TestCase):
class LzmaAppendTest(LzmaTest, AppendTestBase, unittest.TestCase):
pass
class ZstdAppendTest(ZstdTest, AppendTestBase, unittest.TestCase):
pass
class LimitsTest(unittest.TestCase):
@ -2835,7 +2875,7 @@ class CommandLineTest(unittest.TestCase):
support.findfile('tokenize_tests-no-coding-cookie-'
'and-utf8-bom-sig-only.txt',
subdir='tokenizedata')]
for filetype in (GzipTest, Bz2Test, LzmaTest):
for filetype in (GzipTest, Bz2Test, LzmaTest, ZstdTest):
if not filetype.open:
continue
try:
@ -4257,7 +4297,7 @@ def setUpModule():
data = fobj.read()
# Create compressed tarfiles.
for c in GzipTest, Bz2Test, LzmaTest:
for c in GzipTest, Bz2Test, LzmaTest, ZstdTest:
if c.open:
os_helper.unlink(c.tarname)
testtarnames.append(c.tarname)

28
Lib/test/test_zipfile/test_core.py
View file

@ -23,7 +23,7 @@ from test import archiver_tests
from test.support import script_helper, os_helper
from test.support import (
findfile, requires_zlib, requires_bz2, requires_lzma,
captured_stdout, captured_stderr, requires_subprocess,
requires_zstd, captured_stdout, captured_stderr, requires_subprocess,
cpython_only
)
from test.support.os_helper import (
@ -702,6 +702,10 @@ class LzmaTestsWithSourceFile(AbstractTestsWithSourceFile,
unittest.TestCase):
compression = zipfile.ZIP_LZMA
@requires_zstd()
class ZstdTestsWithSourceFile(AbstractTestsWithSourceFile,
unittest.TestCase):
compression = zipfile.ZIP_ZSTANDARD
class AbstractTestZip64InSmallFiles:
# These tests test the ZIP64 functionality without using large files,
@ -1279,6 +1283,10 @@ class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles,
unittest.TestCase):
compression = zipfile.ZIP_LZMA
@requires_zstd()
class ZstdTestZip64InSmallFiles(AbstractTestZip64InSmallFiles,
unittest.TestCase):
compression = zipfile.ZIP_ZSTANDARD
class AbstractWriterTests:
@ -1348,6 +1356,9 @@ class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase):
class LzmaWriterTests(AbstractWriterTests, unittest.TestCase):
compression = zipfile.ZIP_LZMA
@requires_zstd()
class ZstdWriterTests(AbstractWriterTests, unittest.TestCase):
compression = zipfile.ZIP_ZSTANDARD
class PyZipFileTests(unittest.TestCase):
def assertCompiledIn(self, name, namelist):
@ -2678,6 +2689,17 @@ class LzmaBadCrcTests(AbstractBadCrcTests, unittest.TestCase):
b'ePK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00'
b'\x00>\x00\x00\x00\x00\x00')
@requires_zstd()
class ZstdBadCrcTests(AbstractBadCrcTests, unittest.TestCase):
compression = zipfile.ZIP_ZSTANDARD
zip_with_bad_crc = (
b'PK\x03\x04?\x00\x00\x00]\x00\x00\x00!\x00V\xb1\x17J\x14\x00'
b'\x00\x00\x0b\x00\x00\x00\x05\x00\x00\x00afile(\xb5/\xfd\x00'
b'XY\x00\x00Hello WorldPK\x01\x02?\x03?\x00\x00\x00]\x00\x00\x00'
b'!\x00V\xb0\x17J\x14\x00\x00\x00\x0b\x00\x00\x00\x05\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00afilePK'
b'\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00\x007\x00\x00\x00'
b'\x00\x00')
class DecryptionTests(unittest.TestCase):
"""Check that ZIP decryption works. Since the library does not
@ -2905,6 +2927,10 @@ class LzmaTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles,
unittest.TestCase):
compression = zipfile.ZIP_LZMA
@requires_zstd()
class ZstdTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles,
unittest.TestCase):
compression = zipfile.ZIP_ZSTANDARD
# Provide the tell() method but not seek()
class Tellable:

2507
Lib/test/test_zstd.py Normal file
View file

File diff suppressed because it is too large Load diff

20
Lib/zipfile/__init__.py
View file

@ -31,6 +31,11 @@ try:
except ImportError:
lzma = None
try:
from compression import zstd # We may need its compression method
except ImportError:
zstd = None
__all__ = ["BadZipFile", "BadZipfile", "error",
"ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA",
"is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile",
@ -58,12 +63,14 @@ ZIP_STORED = 0
ZIP_DEFLATED = 8
ZIP_BZIP2 = 12
ZIP_LZMA = 14
ZIP_ZSTANDARD = 93
# Other ZIP compression methods not supported
DEFAULT_VERSION = 20
ZIP64_VERSION = 45
BZIP2_VERSION = 46
LZMA_VERSION = 63
ZSTANDARD_VERSION = 63
# we recognize (but not necessarily support) all features up to that version
MAX_EXTRACT_VERSION = 63
@ -505,6 +512,8 @@ class ZipInfo:
min_version = max(BZIP2_VERSION, min_version)
elif self.compress_type == ZIP_LZMA:
min_version = max(LZMA_VERSION, min_version)
elif self.compress_type == ZIP_ZSTANDARD:
min_version = max(ZSTANDARD_VERSION, min_version)
self.extract_version = max(min_version, self.extract_version)
self.create_version = max(min_version, self.create_version)
@ -766,6 +775,7 @@ compressor_names = {
14: 'lzma',
18: 'terse',
19: 'lz77',
93: 'zstd',
97: 'wavpack',
98: 'ppmd',
}
@ -785,6 +795,10 @@ def _check_compression(compression):
if not lzma:
raise RuntimeError(
"Compression requires the (missing) lzma module")
elif compression == ZIP_ZSTANDARD:
if not zstd:
raise RuntimeError(
"Compression requires the (missing) compression.zstd module")
else:
raise NotImplementedError("That compression method is not supported")
@ -798,9 +812,11 @@ def _get_compressor(compress_type, compresslevel=None):
if compresslevel is not None:
return bz2.BZ2Compressor(compresslevel)
return bz2.BZ2Compressor()
# compresslevel is ignored for ZIP_LZMA
# compresslevel is ignored for ZIP_LZMA and ZIP_ZSTANDARD
elif compress_type == ZIP_LZMA:
return LZMACompressor()
elif compress_type == ZIP_ZSTANDARD:
return zstd.ZstdCompressor()
else:
return None
@ -815,6 +831,8 @@ def _get_decompressor(compress_type):
return bz2.BZ2Decompressor()
elif compress_type == ZIP_LZMA:
return LZMADecompressor()
elif compress_type == ZIP_ZSTANDARD:
return zstd.ZstdDecompressor()
else:
descr = compressor_names.get(compress_type)
if descr:

2
Makefile.pre.in
View file

@ -2507,7 +2507,7 @@ maninstall: altmaninstall
XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax
LIBSUBDIRS= asyncio \
collections \
compression compression/bz2 compression/gzip \
compression compression/bz2 compression/gzip compression/zstd \
compression/lzma compression/zlib compression/_common \
concurrent concurrent/futures \
csv \

95
Modules/_zstd/_zstdmodule.c
View file

@ -74,33 +74,33 @@ typedef struct {
static const ParameterInfo cp_list[] =
{
{ZSTD_c_compressionLevel, "compressionLevel"},
{ZSTD_c_windowLog, "windowLog"},
{ZSTD_c_hashLog, "hashLog"},
{ZSTD_c_chainLog, "chainLog"},
{ZSTD_c_searchLog, "searchLog"},
{ZSTD_c_minMatch, "minMatch"},
{ZSTD_c_targetLength, "targetLength"},
{ZSTD_c_compressionLevel, "compression_level"},
{ZSTD_c_windowLog, "window_log"},
{ZSTD_c_hashLog, "hash_log"},
{ZSTD_c_chainLog, "chain_log"},
{ZSTD_c_searchLog, "search_log"},
{ZSTD_c_minMatch, "min_match"},
{ZSTD_c_targetLength, "target_length"},
{ZSTD_c_strategy, "strategy"},
{ZSTD_c_enableLongDistanceMatching, "enableLongDistanceMatching"},
{ZSTD_c_ldmHashLog, "ldmHashLog"},
{ZSTD_c_ldmMinMatch, "ldmMinMatch"},
{ZSTD_c_ldmBucketSizeLog, "ldmBucketSizeLog"},
{ZSTD_c_ldmHashRateLog, "ldmHashRateLog"},
{ZSTD_c_enableLongDistanceMatching, "enable_long_distance_matching"},
{ZSTD_c_ldmHashLog, "ldm_hash_log"},
{ZSTD_c_ldmMinMatch, "ldm_min_match"},
{ZSTD_c_ldmBucketSizeLog, "ldm_bucket_size_log"},
{ZSTD_c_ldmHashRateLog, "ldm_hash_rate_log"},
{ZSTD_c_contentSizeFlag, "contentSizeFlag"},
{ZSTD_c_checksumFlag, "checksumFlag"},
{ZSTD_c_dictIDFlag, "dictIDFlag"},
{ZSTD_c_contentSizeFlag, "content_size_flag"},
{ZSTD_c_checksumFlag, "checksum_flag"},
{ZSTD_c_dictIDFlag, "dict_id_flag"},
{ZSTD_c_nbWorkers, "nbWorkers"},
{ZSTD_c_jobSize, "jobSize"},
{ZSTD_c_overlapLog, "overlapLog"}
{ZSTD_c_nbWorkers, "nb_workers"},
{ZSTD_c_jobSize, "job_size"},
{ZSTD_c_overlapLog, "overlap_log"}
};
static const ParameterInfo dp_list[] =
{
{ZSTD_d_windowLogMax, "windowLogMax"}
{ZSTD_d_windowLogMax, "window_log_max"}
};
void
@ -180,8 +180,8 @@ _zstd._train_dict
samples_bytes: PyBytesObject
Concatenation of samples.
samples_size_list: object(subclass_of='&PyList_Type')
List of samples' sizes.
samples_sizes: object(subclass_of='&PyTuple_Type')
Tuple of samples' sizes.
dict_size: Py_ssize_t
The size of the dictionary.
/
@ -191,8 +191,8 @@ Internal function, train a zstd dictionary on sample data.
static PyObject *
_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
PyObject *samples_size_list, Py_ssize_t dict_size)
/*[clinic end generated code: output=ee53c34c8f77886b input=b21d092c695a3a81]*/
PyObject *samples_sizes, Py_ssize_t dict_size)
/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/
{
// TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
// are pretty similar. We should see if we can refactor them to share that code.
@ -209,7 +209,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
return NULL;
}
chunks_number = Py_SIZE(samples_size_list);
chunks_number = Py_SIZE(samples_sizes);
if ((size_t) chunks_number > UINT32_MAX) {
PyErr_Format(PyExc_ValueError,
"The number of samples should be <= %u.", UINT32_MAX);
@ -225,12 +225,11 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
sizes_sum = 0;
for (i = 0; i < chunks_number; i++) {
PyObject *size = PyList_GetItemRef(samples_size_list, i);
PyObject *size = PyTuple_GetItem(samples_sizes, i);
chunk_sizes[i] = PyLong_AsSize_t(size);
Py_DECREF(size);
if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) {
PyErr_Format(PyExc_ValueError,
"Items in samples_size_list should be an int "
"Items in samples_sizes should be an int "
"object, with a value between 0 and %u.", SIZE_MAX);
goto error;
}
@ -239,7 +238,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
if (sizes_sum != Py_SIZE(samples_bytes)) {
PyErr_SetString(PyExc_ValueError,
"The samples size list doesn't match the concatenation's size.");
"The samples size tuple doesn't match the concatenation's size.");
goto error;
}
@ -287,8 +286,8 @@ _zstd._finalize_dict
Custom dictionary content.
samples_bytes: PyBytesObject
Concatenation of samples.
samples_size_list: object(subclass_of='&PyList_Type')
List of samples' sizes.
samples_sizes: object(subclass_of='&PyTuple_Type')
Tuple of samples' sizes.
dict_size: Py_ssize_t
The size of the dictionary.
compression_level: int
@ -301,9 +300,9 @@ Internal function, finalize a zstd dictionary.
static PyObject *
_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
PyBytesObject *samples_bytes,
PyObject *samples_size_list, Py_ssize_t dict_size,
PyObject *samples_sizes, Py_ssize_t dict_size,
int compression_level)
/*[clinic end generated code: output=9c2a7d8c845cee93 input=08531a803d87c56f]*/
/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/
{
Py_ssize_t chunks_number;
size_t *chunk_sizes = NULL;
@ -319,7 +318,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
return NULL;
}
chunks_number = Py_SIZE(samples_size_list);
chunks_number = Py_SIZE(samples_sizes);
if ((size_t) chunks_number > UINT32_MAX) {
PyErr_Format(PyExc_ValueError,
"The number of samples should be <= %u.", UINT32_MAX);
@ -335,11 +334,11 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
sizes_sum = 0;
for (i = 0; i < chunks_number; i++) {
PyObject *size = PyList_GET_ITEM(samples_size_list, i);
PyObject *size = PyTuple_GetItem(samples_sizes, i);
chunk_sizes[i] = PyLong_AsSize_t(size);
if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) {
PyErr_Format(PyExc_ValueError,
"Items in samples_size_list should be an int "
"Items in samples_sizes should be an int "
"object, with a value between 0 and %u.", SIZE_MAX);
goto error;
}
@ -348,7 +347,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
if (sizes_sum != Py_SIZE(samples_bytes)) {
PyErr_SetString(PyExc_ValueError,
"The samples size list doesn't match the concatenation's size.");
"The samples size tuple doesn't match the concatenation's size.");
goto error;
}
@ -402,18 +401,18 @@ success:
/*[clinic input]
_zstd._get_param_bounds
is_compress: bool
True for CParameter, False for DParameter.
parameter: int
The parameter to get bounds.
is_compress: bool
True for CompressionParameter, False for DecompressionParameter.
Internal function, get CParameter/DParameter bounds.
Internal function, get CompressionParameter/DecompressionParameter bounds.
[clinic start generated code]*/
static PyObject *
_zstd__get_param_bounds_impl(PyObject *module, int is_compress,
int parameter)
/*[clinic end generated code: output=b751dc710f89ef55 input=fb21ff96aff65df1]*/
_zstd__get_param_bounds_impl(PyObject *module, int parameter,
int is_compress)
/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/
{
ZSTD_bounds bound;
if (is_compress) {
@ -515,24 +514,24 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
_zstd._set_parameter_types
c_parameter_type: object(subclass_of='&PyType_Type')
CParameter IntEnum type object
CompressionParameter IntEnum type object
d_parameter_type: object(subclass_of='&PyType_Type')
DParameter IntEnum type object
DecompressionParameter IntEnum type object
Internal function, set CParameter/DParameter types for validity check.
Internal function, set CompressionParameter/DecompressionParameter types for validity check.
[clinic start generated code]*/
static PyObject *
_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
PyObject *d_parameter_type)
/*[clinic end generated code: output=a13d4890ccbd2873 input=3e7d0d37c3a1045a]*/
/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/
{
_zstd_state* const mod_state = get_zstd_state(module);
if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) {
PyErr_SetString(PyExc_ValueError,
"The two arguments should be CParameter and "
"DParameter types.");
"The two arguments should be CompressionParameter and "
"DecompressionParameter types.");
return NULL;
}

76
Modules/_zstd/clinic/_zstdmodule.c.h generated
View file

@ -10,15 +10,15 @@ preserve
#include "pycore_modsupport.h" // _PyArg_CheckPositional()
PyDoc_STRVAR(_zstd__train_dict__doc__,
"_train_dict($module, samples_bytes, samples_size_list, dict_size, /)\n"
"_train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n"
"--\n"
"\n"
"Internal function, train a zstd dictionary on sample data.\n"
"\n"
" samples_bytes\n"
" Concatenation of samples.\n"
" samples_size_list\n"
" List of samples\' sizes.\n"
" samples_sizes\n"
" Tuple of samples\' sizes.\n"
" dict_size\n"
" The size of the dictionary.");
@ -27,14 +27,14 @@ PyDoc_STRVAR(_zstd__train_dict__doc__,
static PyObject *
_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
PyObject *samples_size_list, Py_ssize_t dict_size);
PyObject *samples_sizes, Py_ssize_t dict_size);
static PyObject *
_zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
PyBytesObject *samples_bytes;
PyObject *samples_size_list;
PyObject *samples_sizes;
Py_ssize_t dict_size;
if (!_PyArg_CheckPositional("_train_dict", nargs, 3, 3)) {
@ -45,11 +45,11 @@ _zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
goto exit;
}
samples_bytes = (PyBytesObject *)args[0];
if (!PyList_Check(args[1])) {
_PyArg_BadArgument("_train_dict", "argument 2", "list", args[1]);
if (!PyTuple_Check(args[1])) {
_PyArg_BadArgument("_train_dict", "argument 2", "tuple", args[1]);
goto exit;
}
samples_size_list = args[1];
samples_sizes = args[1];
{
Py_ssize_t ival = -1;
PyObject *iobj = _PyNumber_Index(args[2]);
@ -62,7 +62,7 @@ _zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
}
dict_size = ival;
}
return_value = _zstd__train_dict_impl(module, samples_bytes, samples_size_list, dict_size);
return_value = _zstd__train_dict_impl(module, samples_bytes, samples_sizes, dict_size);
exit:
return return_value;
@ -70,7 +70,7 @@ exit:
PyDoc_STRVAR(_zstd__finalize_dict__doc__,
"_finalize_dict($module, custom_dict_bytes, samples_bytes,\n"
" samples_size_list, dict_size, compression_level, /)\n"
" samples_sizes, dict_size, compression_level, /)\n"
"--\n"
"\n"
"Internal function, finalize a zstd dictionary.\n"
@ -79,8 +79,8 @@ PyDoc_STRVAR(_zstd__finalize_dict__doc__,
" Custom dictionary content.\n"
" samples_bytes\n"
" Concatenation of samples.\n"
" samples_size_list\n"
" List of samples\' sizes.\n"
" samples_sizes\n"
" Tuple of samples\' sizes.\n"
" dict_size\n"
" The size of the dictionary.\n"
" compression_level\n"
@ -92,7 +92,7 @@ PyDoc_STRVAR(_zstd__finalize_dict__doc__,
static PyObject *
_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
PyBytesObject *samples_bytes,
PyObject *samples_size_list, Py_ssize_t dict_size,
PyObject *samples_sizes, Py_ssize_t dict_size,
int compression_level);
static PyObject *
@ -101,7 +101,7 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
PyObject *return_value = NULL;
PyBytesObject *custom_dict_bytes;
PyBytesObject *samples_bytes;
PyObject *samples_size_list;
PyObject *samples_sizes;
Py_ssize_t dict_size;
int compression_level;
@ -118,11 +118,11 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
goto exit;
}
samples_bytes = (PyBytesObject *)args[1];
if (!PyList_Check(args[2])) {
_PyArg_BadArgument("_finalize_dict", "argument 3", "list", args[2]);
if (!PyTuple_Check(args[2])) {
_PyArg_BadArgument("_finalize_dict", "argument 3", "tuple", args[2]);
goto exit;
}
samples_size_list = args[2];
samples_sizes = args[2];
{
Py_ssize_t ival = -1;
PyObject *iobj = _PyNumber_Index(args[3]);
@ -139,29 +139,29 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
if (compression_level == -1 && PyErr_Occurred()) {
goto exit;
}
return_value = _zstd__finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_size_list, dict_size, compression_level);
return_value = _zstd__finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level);
exit:
return return_value;
}
PyDoc_STRVAR(_zstd__get_param_bounds__doc__,
"_get_param_bounds($module, /, is_compress, parameter)\n"
"_get_param_bounds($module, /, parameter, is_compress)\n"
"--\n"
"\n"
"Internal function, get CParameter/DParameter bounds.\n"
"Internal function, get CompressionParameter/DecompressionParameter bounds.\n"
"\n"
" is_compress\n"
" True for CParameter, False for DParameter.\n"
" parameter\n"
" The parameter to get bounds.");
" The parameter to get bounds.\n"
" is_compress\n"
" True for CompressionParameter, False for DecompressionParameter.");
#define _ZSTD__GET_PARAM_BOUNDS_METHODDEF \
{"_get_param_bounds", _PyCFunction_CAST(_zstd__get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd__get_param_bounds__doc__},
static PyObject *
_zstd__get_param_bounds_impl(PyObject *module, int is_compress,
int parameter);
_zstd__get_param_bounds_impl(PyObject *module, int parameter,
int is_compress);
static PyObject *
_zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
@ -178,7 +178,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1,
.ob_item = { &_Py_ID(is_compress), &_Py_ID(parameter), },
.ob_item = { &_Py_ID(parameter), &_Py_ID(is_compress), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -187,7 +187,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"is_compress", "parameter", NULL};
static const char * const _keywords[] = {"parameter", "is_compress", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "_get_param_bounds",
@ -195,23 +195,23 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg
};
#undef KWTUPLE
PyObject *argsbuf[2];
int is_compress;
int parameter;
int is_compress;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser,
/*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
if (!args) {
goto exit;
}
is_compress = PyObject_IsTrue(args[0]);
if (is_compress < 0) {
goto exit;
}
parameter = PyLong_AsInt(args[1]);
parameter = PyLong_AsInt(args[0]);
if (parameter == -1 && PyErr_Occurred()) {
goto exit;
}
return_value = _zstd__get_param_bounds_impl(module, is_compress, parameter);
is_compress = PyObject_IsTrue(args[1]);
if (is_compress < 0) {
goto exit;
}
return_value = _zstd__get_param_bounds_impl(module, parameter, is_compress);
exit:
return return_value;
@ -360,12 +360,12 @@ PyDoc_STRVAR(_zstd__set_parameter_types__doc__,
"_set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n"
"--\n"
"\n"
"Internal function, set CParameter/DParameter types for validity check.\n"
"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n"
"\n"
" c_parameter_type\n"
" CParameter IntEnum type object\n"
" CompressionParameter IntEnum type object\n"
" d_parameter_type\n"
" DParameter IntEnum type object");
" DecompressionParameter IntEnum type object");
#define _ZSTD__SET_PARAMETER_TYPES_METHODDEF \
{"_set_parameter_types", _PyCFunction_CAST(_zstd__set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd__set_parameter_types__doc__},
@ -429,4 +429,4 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n
exit:
return return_value;
}
/*[clinic end generated code: output=077c8ea2b11fb188 input=a9049054013a1b77]*/
/*[clinic end generated code: output=189c462236a7096c input=a9049054013a1b77]*/

4
Modules/_zstd/compressor.c
View file

@ -71,14 +71,14 @@ _PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
if (Py_TYPE(key) == mod_state->DParameter_type) {
PyErr_SetString(PyExc_TypeError,
"Key of compression option dict should "
"NOT be DParameter.");
"NOT be DecompressionParameter.");
return -1;
}
int key_v = PyLong_AsInt(key);
if (key_v == -1 && PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
"Key of options dict should be a CParameter attribute.");
"Key of options dict should be a CompressionParameter attribute.");
return -1;
}

4
Modules/_zstd/decompressor.c
View file

@ -84,7 +84,7 @@ _PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
if (Py_TYPE(key) == mod_state->CParameter_type) {
PyErr_SetString(PyExc_TypeError,
"Key of decompression options dict should "
"NOT be CParameter.");
"NOT be CompressionParameter.");
return -1;
}
@ -92,7 +92,7 @@ _PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
int key_v = PyLong_AsInt(key);
if (key_v == -1 && PyErr_Occurred()) {
PyErr_SetString(PyExc_ValueError,
"Key of options dict should be a DParameter attribute.");
"Key of options dict should be a DecompressionParameter attribute.");
return -1;
}