mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-24 17:33:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 14

[3.14] gh-132983: Clean-ups for `_zstd` (GH-133670) (#133756)

Browse source 
gh-132983: Clean-ups for ``_zstd`` (GH-133670)
(cherry picked from commit c2a5d4b383)

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
This commit is contained in:
Miss Islington (bot) 2025-05-09 16:58:58 +02:00 • committed by GitHub
parent bd6aad0ec4
commit 2df021d9dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 106 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

68
Lib/compression/zstd/__init__.py
View file

@ -28,10 +28,16 @@ __all__ = (
import _zstd
import enum
from _zstd import *
from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError,
get_frame_size, zstd_version)
from compression.zstd._zstdfile import ZstdFile, open, _nbytes
COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0]
# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE)
zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100),
_zstd.zstd_version_number % 100)
"""Version number of the runtime zstd library as a tuple of integers."""
COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT
"""The default compression level for Zstandard, currently '3'."""
@ -167,28 +173,28 @@ def decompress(data, zstd_dict=None, options=None):
class CompressionParameter(enum.IntEnum):
"""Compression parameters."""
compression_level = _zstd._ZSTD_c_compressionLevel
window_log = _zstd._ZSTD_c_windowLog
hash_log = _zstd._ZSTD_c_hashLog
chain_log = _zstd._ZSTD_c_chainLog
search_log = _zstd._ZSTD_c_searchLog
min_match = _zstd._ZSTD_c_minMatch
target_length = _zstd._ZSTD_c_targetLength
strategy = _zstd._ZSTD_c_strategy
compression_level = _zstd.ZSTD_c_compressionLevel
window_log = _zstd.ZSTD_c_windowLog
hash_log = _zstd.ZSTD_c_hashLog
chain_log = _zstd.ZSTD_c_chainLog
search_log = _zstd.ZSTD_c_searchLog
min_match = _zstd.ZSTD_c_minMatch
target_length = _zstd.ZSTD_c_targetLength
strategy = _zstd.ZSTD_c_strategy
enable_long_distance_matching = _zstd._ZSTD_c_enableLongDistanceMatching
ldm_hash_log = _zstd._ZSTD_c_ldmHashLog
ldm_min_match = _zstd._ZSTD_c_ldmMinMatch
ldm_bucket_size_log = _zstd._ZSTD_c_ldmBucketSizeLog
ldm_hash_rate_log = _zstd._ZSTD_c_ldmHashRateLog
enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching
ldm_hash_log = _zstd.ZSTD_c_ldmHashLog
ldm_min_match = _zstd.ZSTD_c_ldmMinMatch
ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog
ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog
content_size_flag = _zstd._ZSTD_c_contentSizeFlag
checksum_flag = _zstd._ZSTD_c_checksumFlag
dict_id_flag = _zstd._ZSTD_c_dictIDFlag
content_size_flag = _zstd.ZSTD_c_contentSizeFlag
checksum_flag = _zstd.ZSTD_c_checksumFlag
dict_id_flag = _zstd.ZSTD_c_dictIDFlag
nb_workers = _zstd._ZSTD_c_nbWorkers
job_size = _zstd._ZSTD_c_jobSize
overlap_log = _zstd._ZSTD_c_overlapLog
nb_workers = _zstd.ZSTD_c_nbWorkers
job_size = _zstd.ZSTD_c_jobSize
overlap_log = _zstd.ZSTD_c_overlapLog
def bounds(self):
"""Return the (lower, upper) int bounds of a compression parameter.
@ -201,7 +207,7 @@ class CompressionParameter(enum.IntEnum):
class DecompressionParameter(enum.IntEnum):
"""Decompression parameters."""
window_log_max = _zstd._ZSTD_d_windowLogMax
window_log_max = _zstd.ZSTD_d_windowLogMax
def bounds(self):
"""Return the (lower, upper) int bounds of a decompression parameter.
@ -219,15 +225,15 @@ class Strategy(enum.IntEnum):
the numeric value might change.
"""
fast = _zstd._ZSTD_fast
dfast = _zstd._ZSTD_dfast
greedy = _zstd._ZSTD_greedy
lazy = _zstd._ZSTD_lazy
lazy2 = _zstd._ZSTD_lazy2
btlazy2 = _zstd._ZSTD_btlazy2
btopt = _zstd._ZSTD_btopt
btultra = _zstd._ZSTD_btultra
btultra2 = _zstd._ZSTD_btultra2
fast = _zstd.ZSTD_fast
dfast = _zstd.ZSTD_dfast
greedy = _zstd.ZSTD_greedy
lazy = _zstd.ZSTD_lazy
lazy2 = _zstd.ZSTD_lazy2
btlazy2 = _zstd.ZSTD_btlazy2
btopt = _zstd.ZSTD_btopt
btultra = _zstd.ZSTD_btultra
btultra2 = _zstd.ZSTD_btultra2
# Check validity of the CompressionParameter & DecompressionParameter types

8
Lib/compression/zstd/_zstdfile.py
View file

@ -1,13 +1,11 @@
import io
from os import PathLike
from _zstd import (ZstdCompressor, ZstdDecompressor, _ZSTD_DStreamSizes,
ZstdError)
from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdError,
ZSTD_DStreamOutSize)
from compression._common import _streams
__all__ = ("ZstdFile", "open")
_ZSTD_DStreamOutSize = _ZSTD_DStreamSizes[1]
_MODE_CLOSED = 0
_MODE_READ = 1
_MODE_WRITE = 2
@ -188,7 +186,7 @@ class ZstdFile(_streams.BaseStream):
# Note this should *not* be io.DEFAULT_BUFFER_SIZE.
# ZSTD_DStreamOutSize is the minimum amount to read guaranteeing
# a full block is read.
size = _ZSTD_DStreamOutSize
size = ZSTD_DStreamOutSize
return self._buffer.read1(size)
def readinto(self, b):

4
Lib/test/test_zstd.py
View file

@ -281,7 +281,7 @@ class CompressorTestCase(unittest.TestCase):
with self.assertRaisesRegex(ZstdError,
(r'Error when setting zstd compression parameter "window_log", '
r'it should \d+ <= value <= \d+, provided value is 100\. '
r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')):
r'\((?:32|64)-bit build\)')):
compress(b'', options=option)
def test_unknown_compression_parameter(self):
@ -413,7 +413,7 @@ class DecompressorTestCase(unittest.TestCase):
with self.assertRaisesRegex(ZstdError,
(r'Error when setting zstd decompression parameter "window_log_max", '
r'it should \d+ <= value <= \d+, provided value is 100\. '
r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')):
r'\((?:32|64)-bit build\)')):
decompress(b'', options=options)
def test_unknown_decompression_parameter(self):

198
Modules/_zstd/_zstdmodule.c
View file

@ -33,9 +33,6 @@ set_zstd_error(const _zstd_state* const state,
case ERR_COMPRESS:
msg = "Unable to compress zstd data: %s";
break;
case ERR_SET_PLEDGED_INPUT_SIZE:
msg = "Unable to set pledged uncompressed content size: %s";
break;
case ERR_LOAD_D_DICT:
msg = "Unable to load zstd dictionary or prefix for decompression: %s";
@ -151,8 +148,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
}
if (ZSTD_isError(bounds.error)) {
PyErr_Format(state->ZstdError,
"Zstd %s parameter \"%s\" is invalid. (zstd v%s)",
type, name, ZSTD_versionString());
"Zstd %s parameter \"%s\" is invalid.",
type, name);
return;
}
@ -160,10 +157,10 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
PyErr_Format(state->ZstdError,
"Error when setting zstd %s parameter \"%s\", it "
"should %d <= value <= %d, provided value is %d. "
"(zstd v%s, %d-bit build)",
"(%d-bit build)",
type, name,
bounds.lowerBound, bounds.upperBound, value_v,
ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t));
8*(int)sizeof(Py_ssize_t));
}
static inline _zstd_state*
@ -558,150 +555,81 @@ static PyMethodDef _zstd_methods[] = {
};
#define ADD_INT_PREFIX_MACRO(module, macro) \
do { \
if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \
return -1; \
} \
} while(0)
static int
add_parameters(PyObject *module)
{
/* If add new parameters, please also add to cp_list/dp_list above. */
/* Compression parameters */
ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize);
ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog);
/* Decompression parameters */
ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax);
/* ZSTD_strategy enum */
ADD_INT_PREFIX_MACRO(module, ZSTD_fast);
ADD_INT_PREFIX_MACRO(module, ZSTD_dfast);
ADD_INT_PREFIX_MACRO(module, ZSTD_greedy);
ADD_INT_PREFIX_MACRO(module, ZSTD_lazy);
ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2);
ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2);
ADD_INT_PREFIX_MACRO(module, ZSTD_btopt);
ADD_INT_PREFIX_MACRO(module, ZSTD_btultra);
ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2);
return 0;
}
static inline PyObject *
get_zstd_version_info(void)
{
uint32_t ver = ZSTD_versionNumber();
uint32_t major, minor, release;
major = ver / 10000;
minor = (ver / 100) % 100;
release = ver % 100;
return Py_BuildValue("III", major, minor, release);
}
static inline int
add_vars_to_module(PyObject *module)
add_vars_to_module(PyObject *m)
{
PyObject *obj;
#define ADD_INT_MACRO(MACRO) \
if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \
return -1; \
}
/* zstd_version, a str. */
if (PyModule_AddStringConstant(module, "zstd_version",
/* zstd_version_number, int */
if (PyModule_AddIntConstant(m, "zstd_version_number",
ZSTD_versionNumber()) < 0) {
return -1;
}
/* zstd_version, str */
if (PyModule_AddStringConstant(m, "zstd_version",
ZSTD_versionString()) < 0) {
return -1;
}
/* zstd_version_info, a tuple. */
obj = get_zstd_version_info();
if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) {
Py_XDECREF(obj);
/* ZSTD_CLEVEL_DEFAULT, int */
#if ZSTD_VERSION_NUMBER >= 10500
if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT",
ZSTD_defaultCLevel()) < 0) {
return -1;
}
Py_DECREF(obj);
/* Add zstd parameters */
if (add_parameters(module) < 0) {
return -1;
}
/* _compressionLevel_values: (default, min, max)
ZSTD_defaultCLevel() was added in zstd v1.5.0 */
obj = Py_BuildValue("iii",
#if ZSTD_VERSION_NUMBER < 10500
ZSTD_CLEVEL_DEFAULT,
#else
ZSTD_defaultCLevel(),
ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT);
#endif
ZSTD_minCLevel(),
ZSTD_maxCLevel());
if (PyModule_AddObjectRef(module,
"_compressionLevel_values",
obj) < 0) {
Py_XDECREF(obj);
return -1;
}
Py_DECREF(obj);
/* _ZSTD_CStreamSizes */
obj = Py_BuildValue("II",
(uint32_t)ZSTD_CStreamInSize(),
(uint32_t)ZSTD_CStreamOutSize());
if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) {
Py_XDECREF(obj);
/* ZSTD_DStreamOutSize, int */
if (PyModule_Add(m, "ZSTD_DStreamOutSize",
PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) {
return -1;
}
Py_DECREF(obj);
/* _ZSTD_DStreamSizes */
obj = Py_BuildValue("II",
(uint32_t)ZSTD_DStreamInSize(),
(uint32_t)ZSTD_DStreamOutSize());
if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) {
Py_XDECREF(obj);
return -1;
}
Py_DECREF(obj);
/* Add zstd compression parameters. All should also be in cp_list. */
ADD_INT_MACRO(ZSTD_c_compressionLevel);
ADD_INT_MACRO(ZSTD_c_windowLog);
ADD_INT_MACRO(ZSTD_c_hashLog);
ADD_INT_MACRO(ZSTD_c_chainLog);
ADD_INT_MACRO(ZSTD_c_searchLog);
ADD_INT_MACRO(ZSTD_c_minMatch);
ADD_INT_MACRO(ZSTD_c_targetLength);
ADD_INT_MACRO(ZSTD_c_strategy);
/* _ZSTD_CONFIG */
obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c",
Py_False,
Py_True,
/* User mremap output buffer */
#if defined(HAVE_MREMAP)
Py_True
#else
Py_False
#endif
);
if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) {
Py_XDECREF(obj);
return -1;
}
Py_DECREF(obj);
ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching);
ADD_INT_MACRO(ZSTD_c_ldmHashLog);
ADD_INT_MACRO(ZSTD_c_ldmMinMatch);
ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog);
ADD_INT_MACRO(ZSTD_c_ldmHashRateLog);
ADD_INT_MACRO(ZSTD_c_contentSizeFlag);
ADD_INT_MACRO(ZSTD_c_checksumFlag);
ADD_INT_MACRO(ZSTD_c_dictIDFlag);
ADD_INT_MACRO(ZSTD_c_nbWorkers);
ADD_INT_MACRO(ZSTD_c_jobSize);
ADD_INT_MACRO(ZSTD_c_overlapLog);
/* Add zstd decompression parameters. All should also be in dp_list. */
ADD_INT_MACRO(ZSTD_d_windowLogMax);
/* ZSTD_strategy enum */
ADD_INT_MACRO(ZSTD_fast);
ADD_INT_MACRO(ZSTD_dfast);
ADD_INT_MACRO(ZSTD_greedy);
ADD_INT_MACRO(ZSTD_lazy);
ADD_INT_MACRO(ZSTD_lazy2);
ADD_INT_MACRO(ZSTD_btlazy2);
ADD_INT_MACRO(ZSTD_btopt);
ADD_INT_MACRO(ZSTD_btultra);
ADD_INT_MACRO(ZSTD_btultra2);
#undef ADD_INT_MACRO
return 0;
}

3
Modules/_zstd/_zstdmodule.h
View file

@ -137,7 +137,6 @@ typedef enum {
typedef enum {
ERR_DECOMPRESS,
ERR_COMPRESS,
ERR_SET_PLEDGED_INPUT_SIZE,
ERR_LOAD_D_DICT,
ERR_LOAD_C_DICT,
@ -147,7 +146,7 @@ typedef enum {
ERR_SET_C_LEVEL,
ERR_TRAIN_DICT,
ERR_FINALIZE_DICT
ERR_FINALIZE_DICT,
} error_type;
typedef enum {