mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-25 09:50:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

[3.14] gh-132983: Clean-ups for `_zstd` (GH-133670) (#133756)

Browse source 
gh-132983: Clean-ups for ``_zstd`` (GH-133670)
(cherry picked from commit c2a5d4b383)

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
This commit is contained in:
Miss Islington (bot) 2025-05-09 16:58:58 +02:00 • committed by GitHub
parent bd6aad0ec4
commit 2df021d9dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 106 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

68
Lib/compression/zstd/__init__.py
View file

@ -28,10 +28,16 @@ __all__ = (
import _zstd import _zstd
import enum import enum
from _zstd import * from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError,
get_frame_size, zstd_version)
from compression.zstd._zstdfile import ZstdFile, open, _nbytes from compression.zstd._zstdfile import ZstdFile, open, _nbytes
COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0] # zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE)
zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100),
_zstd.zstd_version_number % 100)
"""Version number of the runtime zstd library as a tuple of integers."""
COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT
"""The default compression level for Zstandard, currently '3'.""" """The default compression level for Zstandard, currently '3'."""
@ -167,28 +173,28 @@ def decompress(data, zstd_dict=None, options=None):
class CompressionParameter(enum.IntEnum): class CompressionParameter(enum.IntEnum):
"""Compression parameters.""" """Compression parameters."""
compression_level = _zstd._ZSTD_c_compressionLevel compression_level = _zstd.ZSTD_c_compressionLevel
window_log = _zstd._ZSTD_c_windowLog window_log = _zstd.ZSTD_c_windowLog
hash_log = _zstd._ZSTD_c_hashLog hash_log = _zstd.ZSTD_c_hashLog
chain_log = _zstd._ZSTD_c_chainLog chain_log = _zstd.ZSTD_c_chainLog
search_log = _zstd._ZSTD_c_searchLog search_log = _zstd.ZSTD_c_searchLog
min_match = _zstd._ZSTD_c_minMatch min_match = _zstd.ZSTD_c_minMatch
target_length = _zstd._ZSTD_c_targetLength target_length = _zstd.ZSTD_c_targetLength
strategy = _zstd._ZSTD_c_strategy strategy = _zstd.ZSTD_c_strategy
enable_long_distance_matching = _zstd._ZSTD_c_enableLongDistanceMatching enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching
ldm_hash_log = _zstd._ZSTD_c_ldmHashLog ldm_hash_log = _zstd.ZSTD_c_ldmHashLog
ldm_min_match = _zstd._ZSTD_c_ldmMinMatch ldm_min_match = _zstd.ZSTD_c_ldmMinMatch
ldm_bucket_size_log = _zstd._ZSTD_c_ldmBucketSizeLog ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog
ldm_hash_rate_log = _zstd._ZSTD_c_ldmHashRateLog ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog
content_size_flag = _zstd._ZSTD_c_contentSizeFlag content_size_flag = _zstd.ZSTD_c_contentSizeFlag
checksum_flag = _zstd._ZSTD_c_checksumFlag checksum_flag = _zstd.ZSTD_c_checksumFlag
dict_id_flag = _zstd._ZSTD_c_dictIDFlag dict_id_flag = _zstd.ZSTD_c_dictIDFlag
nb_workers = _zstd._ZSTD_c_nbWorkers nb_workers = _zstd.ZSTD_c_nbWorkers
job_size = _zstd._ZSTD_c_jobSize job_size = _zstd.ZSTD_c_jobSize
overlap_log = _zstd._ZSTD_c_overlapLog overlap_log = _zstd.ZSTD_c_overlapLog
def bounds(self): def bounds(self):
"""Return the (lower, upper) int bounds of a compression parameter. """Return the (lower, upper) int bounds of a compression parameter.
@ -201,7 +207,7 @@ class CompressionParameter(enum.IntEnum):
class DecompressionParameter(enum.IntEnum): class DecompressionParameter(enum.IntEnum):
"""Decompression parameters.""" """Decompression parameters."""
window_log_max = _zstd._ZSTD_d_windowLogMax window_log_max = _zstd.ZSTD_d_windowLogMax
def bounds(self): def bounds(self):
"""Return the (lower, upper) int bounds of a decompression parameter. """Return the (lower, upper) int bounds of a decompression parameter.
@ -219,15 +225,15 @@ class Strategy(enum.IntEnum):
the numeric value might change. the numeric value might change.
""" """
fast = _zstd._ZSTD_fast fast = _zstd.ZSTD_fast
dfast = _zstd._ZSTD_dfast dfast = _zstd.ZSTD_dfast
greedy = _zstd._ZSTD_greedy greedy = _zstd.ZSTD_greedy
lazy = _zstd._ZSTD_lazy lazy = _zstd.ZSTD_lazy
lazy2 = _zstd._ZSTD_lazy2 lazy2 = _zstd.ZSTD_lazy2
btlazy2 = _zstd._ZSTD_btlazy2 btlazy2 = _zstd.ZSTD_btlazy2
btopt = _zstd._ZSTD_btopt btopt = _zstd.ZSTD_btopt
btultra = _zstd._ZSTD_btultra btultra = _zstd.ZSTD_btultra
btultra2 = _zstd._ZSTD_btultra2 btultra2 = _zstd.ZSTD_btultra2
# Check validity of the CompressionParameter & DecompressionParameter types # Check validity of the CompressionParameter & DecompressionParameter types

8
Lib/compression/zstd/_zstdfile.py
View file

@ -1,13 +1,11 @@
import io import io
from os import PathLike from os import PathLike
from _zstd import (ZstdCompressor, ZstdDecompressor, _ZSTD_DStreamSizes, from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdError,
ZstdError) ZSTD_DStreamOutSize)
from compression._common import _streams from compression._common import _streams
__all__ = ("ZstdFile", "open") __all__ = ("ZstdFile", "open")
_ZSTD_DStreamOutSize = _ZSTD_DStreamSizes[1]
_MODE_CLOSED = 0 _MODE_CLOSED = 0
_MODE_READ = 1 _MODE_READ = 1
_MODE_WRITE = 2 _MODE_WRITE = 2
@ -188,7 +186,7 @@ class ZstdFile(_streams.BaseStream):
# Note this should *not* be io.DEFAULT_BUFFER_SIZE. # Note this should *not* be io.DEFAULT_BUFFER_SIZE.
# ZSTD_DStreamOutSize is the minimum amount to read guaranteeing # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing
# a full block is read. # a full block is read.
size = _ZSTD_DStreamOutSize size = ZSTD_DStreamOutSize
return self._buffer.read1(size) return self._buffer.read1(size)
def readinto(self, b): def readinto(self, b):

4
Lib/test/test_zstd.py
View file

@ -281,7 +281,7 @@ class CompressorTestCase(unittest.TestCase):
with self.assertRaisesRegex(ZstdError, with self.assertRaisesRegex(ZstdError,
(r'Error when setting zstd compression parameter "window_log", ' (r'Error when setting zstd compression parameter "window_log", '
r'it should \d+ <= value <= \d+, provided value is 100\. ' r'it should \d+ <= value <= \d+, provided value is 100\. '
r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): r'\((?:32|64)-bit build\)')):
compress(b'', options=option) compress(b'', options=option)
def test_unknown_compression_parameter(self): def test_unknown_compression_parameter(self):
@ -413,7 +413,7 @@ class DecompressorTestCase(unittest.TestCase):
with self.assertRaisesRegex(ZstdError, with self.assertRaisesRegex(ZstdError,
(r'Error when setting zstd decompression parameter "window_log_max", ' (r'Error when setting zstd decompression parameter "window_log_max", '
r'it should \d+ <= value <= \d+, provided value is 100\. ' r'it should \d+ <= value <= \d+, provided value is 100\. '
r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): r'\((?:32|64)-bit build\)')):
decompress(b'', options=options) decompress(b'', options=options)
def test_unknown_decompression_parameter(self): def test_unknown_decompression_parameter(self):

198
Modules/_zstd/_zstdmodule.c
View file

@ -33,9 +33,6 @@ set_zstd_error(const _zstd_state* const state,
case ERR_COMPRESS: case ERR_COMPRESS:
msg = "Unable to compress zstd data: %s"; msg = "Unable to compress zstd data: %s";
break; break;
case ERR_SET_PLEDGED_INPUT_SIZE:
msg = "Unable to set pledged uncompressed content size: %s";
break;
case ERR_LOAD_D_DICT: case ERR_LOAD_D_DICT:
msg = "Unable to load zstd dictionary or prefix for decompression: %s"; msg = "Unable to load zstd dictionary or prefix for decompression: %s";
@ -151,8 +148,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
} }
if (ZSTD_isError(bounds.error)) { if (ZSTD_isError(bounds.error)) {
PyErr_Format(state->ZstdError, PyErr_Format(state->ZstdError,
"Zstd %s parameter \"%s\" is invalid. (zstd v%s)", "Zstd %s parameter \"%s\" is invalid.",
type, name, ZSTD_versionString()); type, name);
return; return;
} }
@ -160,10 +157,10 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
PyErr_Format(state->ZstdError, PyErr_Format(state->ZstdError,
"Error when setting zstd %s parameter \"%s\", it " "Error when setting zstd %s parameter \"%s\", it "
"should %d <= value <= %d, provided value is %d. " "should %d <= value <= %d, provided value is %d. "
"(zstd v%s, %d-bit build)", "(%d-bit build)",
type, name, type, name,
bounds.lowerBound, bounds.upperBound, value_v, bounds.lowerBound, bounds.upperBound, value_v,
ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t)); 8*(int)sizeof(Py_ssize_t));
} }
static inline _zstd_state* static inline _zstd_state*
@ -558,150 +555,81 @@ static PyMethodDef _zstd_methods[] = {
}; };
#define ADD_INT_PREFIX_MACRO(module, macro) \
do { \
if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \
return -1; \
} \
} while(0)
static int
add_parameters(PyObject *module)
{
/* If add new parameters, please also add to cp_list/dp_list above. */
/* Compression parameters */
ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog);
/* Decompression parameters */
ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax);
/* ZSTD_strategy enum */
ADD_INT_PREFIX_MACRO(module, ZSTD_fast);
ADD_INT_PREFIX_MACRO(module, ZSTD_dfast);
ADD_INT_PREFIX_MACRO(module, ZSTD_greedy);
ADD_INT_PREFIX_MACRO(module, ZSTD_lazy);
ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2);
ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2);
ADD_INT_PREFIX_MACRO(module, ZSTD_btopt);
ADD_INT_PREFIX_MACRO(module, ZSTD_btultra);
ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2);
return 0;
}
static inline PyObject *
get_zstd_version_info(void)
{
uint32_t ver = ZSTD_versionNumber();
uint32_t major, minor, release;
major = ver / 10000;
minor = (ver / 100) % 100;
release = ver % 100;
return Py_BuildValue("III", major, minor, release);
}
static inline int static inline int
add_vars_to_module(PyObject *module) add_vars_to_module(PyObject *m)
{ {
PyObject *obj; #define ADD_INT_MACRO(MACRO) \
if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \
return -1; \
}
/* zstd_version, a str. */ /* zstd_version_number, int */
if (PyModule_AddStringConstant(module, "zstd_version", if (PyModule_AddIntConstant(m, "zstd_version_number",
ZSTD_versionNumber()) < 0) {
return -1;
}
/* zstd_version, str */
if (PyModule_AddStringConstant(m, "zstd_version",
ZSTD_versionString()) < 0) { ZSTD_versionString()) < 0) {
return -1; return -1;
} }
/* zstd_version_info, a tuple. */ /* ZSTD_CLEVEL_DEFAULT, int */
obj = get_zstd_version_info(); #if ZSTD_VERSION_NUMBER >= 10500
if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) { if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT",
Py_XDECREF(obj); ZSTD_defaultCLevel()) < 0) {
return -1; return -1;
} }
Py_DECREF(obj);
/* Add zstd parameters */
if (add_parameters(module) < 0) {
return -1;
}
/* _compressionLevel_values: (default, min, max)
ZSTD_defaultCLevel() was added in zstd v1.5.0 */
obj = Py_BuildValue("iii",
#if ZSTD_VERSION_NUMBER < 10500
ZSTD_CLEVEL_DEFAULT,
#else #else
ZSTD_defaultCLevel(), ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT);
#endif #endif
ZSTD_minCLevel(),
ZSTD_maxCLevel());
if (PyModule_AddObjectRef(module,
"_compressionLevel_values",
obj) < 0) {
Py_XDECREF(obj);
return -1;
}
Py_DECREF(obj);
/* _ZSTD_CStreamSizes */ /* ZSTD_DStreamOutSize, int */
obj = Py_BuildValue("II", if (PyModule_Add(m, "ZSTD_DStreamOutSize",
(uint32_t)ZSTD_CStreamInSize(), PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) {
(uint32_t)ZSTD_CStreamOutSize());
if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) {
Py_XDECREF(obj);
return -1; return -1;
} }
Py_DECREF(obj);
/* _ZSTD_DStreamSizes */ /* Add zstd compression parameters. All should also be in cp_list. */
obj = Py_BuildValue("II", ADD_INT_MACRO(ZSTD_c_compressionLevel);
(uint32_t)ZSTD_DStreamInSize(), ADD_INT_MACRO(ZSTD_c_windowLog);
(uint32_t)ZSTD_DStreamOutSize()); ADD_INT_MACRO(ZSTD_c_hashLog);
if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) { ADD_INT_MACRO(ZSTD_c_chainLog);
Py_XDECREF(obj); ADD_INT_MACRO(ZSTD_c_searchLog);
return -1; ADD_INT_MACRO(ZSTD_c_minMatch);
} ADD_INT_MACRO(ZSTD_c_targetLength);
Py_DECREF(obj); ADD_INT_MACRO(ZSTD_c_strategy);
/* _ZSTD_CONFIG */ ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching);
obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c", ADD_INT_MACRO(ZSTD_c_ldmHashLog);
Py_False, ADD_INT_MACRO(ZSTD_c_ldmMinMatch);
Py_True, ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog);
/* User mremap output buffer */ ADD_INT_MACRO(ZSTD_c_ldmHashRateLog);
#if defined(HAVE_MREMAP)
Py_True ADD_INT_MACRO(ZSTD_c_contentSizeFlag);
#else ADD_INT_MACRO(ZSTD_c_checksumFlag);
Py_False ADD_INT_MACRO(ZSTD_c_dictIDFlag);
#endif
); ADD_INT_MACRO(ZSTD_c_nbWorkers);
if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) { ADD_INT_MACRO(ZSTD_c_jobSize);
Py_XDECREF(obj); ADD_INT_MACRO(ZSTD_c_overlapLog);
return -1;
} /* Add zstd decompression parameters. All should also be in dp_list. */
Py_DECREF(obj); ADD_INT_MACRO(ZSTD_d_windowLogMax);
/* ZSTD_strategy enum */
ADD_INT_MACRO(ZSTD_fast);
ADD_INT_MACRO(ZSTD_dfast);
ADD_INT_MACRO(ZSTD_greedy);
ADD_INT_MACRO(ZSTD_lazy);
ADD_INT_MACRO(ZSTD_lazy2);
ADD_INT_MACRO(ZSTD_btlazy2);
ADD_INT_MACRO(ZSTD_btopt);
ADD_INT_MACRO(ZSTD_btultra);
ADD_INT_MACRO(ZSTD_btultra2);
#undef ADD_INT_MACRO
return 0; return 0;
} }

3
Modules/_zstd/_zstdmodule.h
View file

@ -137,7 +137,6 @@ typedef enum {
typedef enum { typedef enum {
ERR_DECOMPRESS, ERR_DECOMPRESS,
ERR_COMPRESS, ERR_COMPRESS,
ERR_SET_PLEDGED_INPUT_SIZE,
ERR_LOAD_D_DICT, ERR_LOAD_D_DICT,
ERR_LOAD_C_DICT, ERR_LOAD_C_DICT,
@ -147,7 +146,7 @@ typedef enum {
ERR_SET_C_LEVEL, ERR_SET_C_LEVEL,
ERR_TRAIN_DICT, ERR_TRAIN_DICT,
ERR_FINALIZE_DICT ERR_FINALIZE_DICT,
} error_type; } error_type;
typedef enum { typedef enum {