mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-23 19:25:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-132983: Convert zstd `__new__` methods to Argument Clinic (#133860)

Browse source
This commit is contained in:
Adam Turner 2025-05-12 09:51:53 +01:00 • committed by GitHub
parent 0eb448cae5
commit d29ddbd90c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 275 additions and 371 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Include/internal/pycore_global_objects_fini_generated.h generated
View file

@ -902,7 +902,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict_content));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest));

1
Include/internal/pycore_global_strings.h
View file

@ -393,7 +393,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(deterministic)
STRUCT_FOR_ID(device) STRUCT_FOR_ID(device)
STRUCT_FOR_ID(dict) STRUCT_FOR_ID(dict)
STRUCT_FOR_ID(dict_content)
STRUCT_FOR_ID(dictcomp) STRUCT_FOR_ID(dictcomp)
STRUCT_FOR_ID(difference_update) STRUCT_FOR_ID(difference_update)
STRUCT_FOR_ID(digest) STRUCT_FOR_ID(digest)

1
Include/internal/pycore_runtime_init_generated.h generated
View file

@ -900,7 +900,6 @@ extern "C" {
INIT_ID(deterministic), \ INIT_ID(deterministic), \
INIT_ID(device), \ INIT_ID(device), \
INIT_ID(dict), \ INIT_ID(dict), \
INIT_ID(dict_content), \
INIT_ID(dictcomp), \ INIT_ID(dictcomp), \
INIT_ID(difference_update), \ INIT_ID(difference_update), \
INIT_ID(digest), \ INIT_ID(digest), \

4
Include/internal/pycore_unicodeobject_generated.h generated
View file

@ -1360,10 +1360,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
_PyUnicode_InternStatic(interp, &string); _PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1)); assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1); assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(dict_content);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(dictcomp); string = &_Py_ID(dictcomp);
_PyUnicode_InternStatic(interp, &string); _PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1)); assert(_PyUnicode_CheckConsistency(string, 1));

24
Lib/test/test_zstd.py
View file

@ -288,8 +288,8 @@ class CompressorTestCase(unittest.TestCase):
KEY = 100001234 KEY = 100001234
option = {CompressionParameter.compression_level: 10, option = {CompressionParameter.compression_level: 10,
KEY: 200000000} KEY: 200000000}
pattern = r'Zstd compression parameter.*?"unknown parameter \(key %d\)"' \ pattern = (r'Invalid zstd compression parameter.*?'
% KEY fr'"unknown parameter \(key {KEY}\)"')
with self.assertRaisesRegex(ZstdError, pattern): with self.assertRaisesRegex(ZstdError, pattern):
ZstdCompressor(options=option) ZstdCompressor(options=option)
@ -420,8 +420,8 @@ class DecompressorTestCase(unittest.TestCase):
KEY = 100001234 KEY = 100001234
options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1],
KEY: 200000000} KEY: 200000000}
pattern = r'Zstd decompression parameter.*?"unknown parameter \(key %d\)"' \ pattern = (r'Invalid zstd decompression parameter.*?'
% KEY fr'"unknown parameter \(key {KEY}\)"')
with self.assertRaisesRegex(ZstdError, pattern): with self.assertRaisesRegex(ZstdError, pattern):
ZstdDecompressor(options=options) ZstdDecompressor(options=options)
@ -507,7 +507,7 @@ class DecompressorTestCase(unittest.TestCase):
self.assertFalse(d.needs_input) self.assertFalse(d.needs_input)
def test_decompressor_arg(self): def test_decompressor_arg(self):
zd = ZstdDict(b'12345678', True) zd = ZstdDict(b'12345678', is_raw=True)
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
d = ZstdDecompressor(zstd_dict={}) d = ZstdDecompressor(zstd_dict={})
@ -1021,6 +1021,10 @@ class DecompressorFlagsTestCase(unittest.TestCase):
class ZstdDictTestCase(unittest.TestCase): class ZstdDictTestCase(unittest.TestCase):
def test_is_raw(self): def test_is_raw(self):
# must be passed as a keyword argument
with self.assertRaises(TypeError):
ZstdDict(bytes(8), True)
# content < 8 # content < 8
b = b'1234567' b = b'1234567'
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
@ -1068,9 +1072,9 @@ class ZstdDictTestCase(unittest.TestCase):
# corrupted # corrupted
zd = ZstdDict(dict_content, is_raw=False) zd = ZstdDict(dict_content, is_raw=False)
with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?corrupted'): with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'):
ZstdCompressor(zstd_dict=zd.as_digested_dict) ZstdCompressor(zstd_dict=zd.as_digested_dict)
with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?corrupted'): with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'):
ZstdDecompressor(zd) ZstdDecompressor(zd)
# wrong type # wrong type
@ -1096,7 +1100,7 @@ class ZstdDictTestCase(unittest.TestCase):
TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1)
ZstdDict(TRAINED_DICT.dict_content, False) ZstdDict(TRAINED_DICT.dict_content, is_raw=False)
self.assertNotEqual(TRAINED_DICT.dict_id, 0) self.assertNotEqual(TRAINED_DICT.dict_id, 0)
self.assertGreater(len(TRAINED_DICT.dict_content), 0) self.assertGreater(len(TRAINED_DICT.dict_content), 0)
@ -1250,7 +1254,7 @@ class ZstdDictTestCase(unittest.TestCase):
def test_as_prefix(self): def test_as_prefix(self):
# V1 # V1
V1 = THIS_FILE_BYTES V1 = THIS_FILE_BYTES
zd = ZstdDict(V1, True) zd = ZstdDict(V1, is_raw=True)
# V2 # V2
mid = len(V1) // 2 mid = len(V1) // 2
@ -1266,7 +1270,7 @@ class ZstdDictTestCase(unittest.TestCase):
self.assertEqual(decompress(dat, zd.as_prefix), V2) self.assertEqual(decompress(dat, zd.as_prefix), V2)
# use wrong prefix # use wrong prefix
zd2 = ZstdDict(SAMPLES[0], True) zd2 = ZstdDict(SAMPLES[0], is_raw=True)
try: try:
decompressed = decompress(dat, zd2.as_prefix) decompressed = decompress(dat, zd2.as_prefix)
except ZstdError: # expected except ZstdError: # expected

65
Modules/_zstd/_zstdmodule.c
View file

@ -1,7 +1,4 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
#ifndef Py_BUILD_CORE_BUILTIN #ifndef Py_BUILD_CORE_BUILTIN
# define Py_BUILD_CORE_MODULE 1 # define Py_BUILD_CORE_MODULE 1
@ -34,17 +31,17 @@ set_zstd_error(const _zstd_state* const state,
switch (type) switch (type)
{ {
case ERR_DECOMPRESS: case ERR_DECOMPRESS:
msg = "Unable to decompress zstd data: %s"; msg = "Unable to decompress Zstandard data: %s";
break; break;
case ERR_COMPRESS: case ERR_COMPRESS:
msg = "Unable to compress zstd data: %s"; msg = "Unable to compress Zstandard data: %s";
break; break;
case ERR_LOAD_D_DICT: case ERR_LOAD_D_DICT:
msg = "Unable to load zstd dictionary or prefix for decompression: %s"; msg = "Unable to load Zstandard dictionary or prefix for decompression: %s";
break; break;
case ERR_LOAD_C_DICT: case ERR_LOAD_C_DICT:
msg = "Unable to load zstd dictionary or prefix for compression: %s"; msg = "Unable to load Zstandard dictionary or prefix for compression: %s";
break; break;
case ERR_GET_C_BOUNDS: case ERR_GET_C_BOUNDS:
@ -58,10 +55,10 @@ set_zstd_error(const _zstd_state* const state,
break; break;
case ERR_TRAIN_DICT: case ERR_TRAIN_DICT:
msg = "Unable to train zstd dictionary: %s"; msg = "Unable to train the Zstandard dictionary: %s";
break; break;
case ERR_FINALIZE_DICT: case ERR_FINALIZE_DICT:
msg = "Unable to finalize zstd dictionary: %s"; msg = "Unable to finalize the Zstandard dictionary: %s";
break; break;
default: default:
@ -152,7 +149,7 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
} }
if (ZSTD_isError(bounds.error)) { if (ZSTD_isError(bounds.error)) {
PyErr_Format(state->ZstdError, PyErr_Format(state->ZstdError,
"Zstd %s parameter \"%s\" is invalid.", "Invalid zstd %s parameter \"%s\".",
type, name); type, name);
return; return;
} }
@ -187,13 +184,13 @@ _zstd.train_dict
The size of the dictionary. The size of the dictionary.
/ /
Internal function, train a zstd dictionary on sample data. Train a Zstandard dictionary on sample data.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
PyObject *samples_sizes, Py_ssize_t dict_size) PyObject *samples_sizes, Py_ssize_t dict_size)
/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/ /*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/
{ {
// TODO(emmatyping): The preamble and suffix to this function and _finalize_dict // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
// are pretty similar. We should see if we can refactor them to share that code. // are pretty similar. We should see if we can refactor them to share that code.
@ -258,7 +255,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
chunk_sizes, (uint32_t)chunks_number); chunk_sizes, (uint32_t)chunks_number);
Py_END_ALLOW_THREADS Py_END_ALLOW_THREADS
/* Check zstd dict error */ /* Check Zstandard dict error */
if (ZDICT_isError(zstd_ret)) { if (ZDICT_isError(zstd_ret)) {
_zstd_state* const mod_state = get_zstd_state(module); _zstd_state* const mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret); set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret);
@ -292,10 +289,10 @@ _zstd.finalize_dict
dict_size: Py_ssize_t dict_size: Py_ssize_t
The size of the dictionary. The size of the dictionary.
compression_level: int compression_level: int
Optimize for a specific zstd compression level, 0 means default. Optimize for a specific Zstandard compression level, 0 means default.
/ /
Internal function, finalize a zstd dictionary. Finalize a Zstandard dictionary.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
@ -303,7 +300,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
PyBytesObject *samples_bytes, PyBytesObject *samples_bytes,
PyObject *samples_sizes, Py_ssize_t dict_size, PyObject *samples_sizes, Py_ssize_t dict_size,
int compression_level) int compression_level)
/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/ /*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/
{ {
Py_ssize_t chunks_number; Py_ssize_t chunks_number;
size_t *chunk_sizes = NULL; size_t *chunk_sizes = NULL;
@ -360,7 +357,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
/* Parameters */ /* Parameters */
/* Optimize for a specific zstd compression level, 0 means default. */ /* Optimize for a specific Zstandard compression level, 0 means default. */
params.compressionLevel = compression_level; params.compressionLevel = compression_level;
/* Write log to stderr, 0 = none. */ /* Write log to stderr, 0 = none. */
params.notificationLevel = 0; params.notificationLevel = 0;
@ -376,7 +373,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
(uint32_t)chunks_number, params); (uint32_t)chunks_number, params);
Py_END_ALLOW_THREADS Py_END_ALLOW_THREADS
/* Check zstd dict error */ /* Check Zstandard dict error */
if (ZDICT_isError(zstd_ret)) { if (ZDICT_isError(zstd_ret)) {
_zstd_state* const mod_state = get_zstd_state(module); _zstd_state* const mod_state = get_zstd_state(module);
set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret); set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret);
@ -407,12 +404,12 @@ _zstd.get_param_bounds
is_compress: bool is_compress: bool
True for CompressionParameter, False for DecompressionParameter. True for CompressionParameter, False for DecompressionParameter.
Internal function, get CompressionParameter/DecompressionParameter bounds. Get CompressionParameter/DecompressionParameter bounds.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/ /*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/
{ {
ZSTD_bounds bound; ZSTD_bounds bound;
if (is_compress) { if (is_compress) {
@ -442,14 +439,12 @@ _zstd.get_frame_size
A bytes-like object, it should start from the beginning of a frame, A bytes-like object, it should start from the beginning of a frame,
and contains at least one complete frame. and contains at least one complete frame.
Get the size of a zstd frame, including frame header and 4-byte checksum if it has one. Get the size of a Zstandard frame, including the header and optional checksum.
It will iterate all blocks' headers within a frame, to accumulate the frame size.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
_zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/ /*[clinic end generated code: output=a7384c2f8780f442 input=3b9f73f8c8129d38]*/
{ {
size_t frame_size; size_t frame_size;
@ -457,9 +452,9 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
if (ZSTD_isError(frame_size)) { if (ZSTD_isError(frame_size)) {
_zstd_state* const mod_state = get_zstd_state(module); _zstd_state* const mod_state = get_zstd_state(module);
PyErr_Format(mod_state->ZstdError, PyErr_Format(mod_state->ZstdError,
"Error when finding the compressed size of a zstd frame. " "Error when finding the compressed size of a Zstandard frame. "
"Make sure the frame_buffer argument starts from the " "Ensure the frame_buffer argument starts from the "
"beginning of a frame, and its length not less than this " "beginning of a frame, and its length is not less than this "
"complete frame. Zstd error message: %s.", "complete frame. Zstd error message: %s.",
ZSTD_getErrorName(frame_size)); ZSTD_getErrorName(frame_size));
return NULL; return NULL;
@ -472,14 +467,14 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
_zstd.get_frame_info _zstd.get_frame_info
frame_buffer: Py_buffer frame_buffer: Py_buffer
A bytes-like object, containing the header of a zstd frame. A bytes-like object, containing the header of a Zstandard frame.
Internal function, get zstd frame infomation from a frame header. Get Zstandard frame infomation from a frame header.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/ /*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/
{ {
uint64_t decompressed_size; uint64_t decompressed_size;
uint32_t dict_id; uint32_t dict_id;
@ -494,9 +489,9 @@ _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
_zstd_state* const mod_state = get_zstd_state(module); _zstd_state* const mod_state = get_zstd_state(module);
PyErr_SetString(mod_state->ZstdError, PyErr_SetString(mod_state->ZstdError,
"Error when getting information from the header of " "Error when getting information from the header of "
"a zstd frame. Make sure the frame_buffer argument " "a Zstandard frame. Ensure the frame_buffer argument "
"starts from the beginning of a frame, and its length " "starts from the beginning of a frame, and its length "
"not less than the frame header (6~18 bytes)."); "is not less than the frame header (6~18 bytes).");
return NULL; return NULL;
} }
@ -518,13 +513,13 @@ _zstd.set_parameter_types
d_parameter_type: object(subclass_of='&PyType_Type') d_parameter_type: object(subclass_of='&PyType_Type')
DecompressionParameter IntEnum type object DecompressionParameter IntEnum type object
Internal function, set CompressionParameter/DecompressionParameter types for validity check. Set CompressionParameter and DecompressionParameter types for validity check.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
PyObject *d_parameter_type) PyObject *d_parameter_type)
/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/ /*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/
{ {
_zstd_state* const mod_state = get_zstd_state(module); _zstd_state* const mod_state = get_zstd_state(module);

5
Modules/_zstd/_zstdmodule.h
View file

@ -1,7 +1,4 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
/* Declarations shared between different parts of the _zstd module*/ /* Declarations shared between different parts of the _zstd module*/

5
Modules/_zstd/buffer.h
View file

@ -1,7 +1,4 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
#ifndef ZSTD_BUFFER_H #ifndef ZSTD_BUFFER_H
#define ZSTD_BUFFER_H #define ZSTD_BUFFER_H

22
Modules/_zstd/clinic/_zstdmodule.c.h generated
View file

@ -13,7 +13,7 @@ PyDoc_STRVAR(_zstd_train_dict__doc__,
"train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n"
"--\n" "--\n"
"\n" "\n"
"Internal function, train a zstd dictionary on sample data.\n" "Train a Zstandard dictionary on sample data.\n"
"\n" "\n"
" samples_bytes\n" " samples_bytes\n"
" Concatenation of samples.\n" " Concatenation of samples.\n"
@ -73,7 +73,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__,
" dict_size, compression_level, /)\n" " dict_size, compression_level, /)\n"
"--\n" "--\n"
"\n" "\n"
"Internal function, finalize a zstd dictionary.\n" "Finalize a Zstandard dictionary.\n"
"\n" "\n"
" custom_dict_bytes\n" " custom_dict_bytes\n"
" Custom dictionary content.\n" " Custom dictionary content.\n"
@ -84,7 +84,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__,
" dict_size\n" " dict_size\n"
" The size of the dictionary.\n" " The size of the dictionary.\n"
" compression_level\n" " compression_level\n"
" Optimize for a specific zstd compression level, 0 means default."); " Optimize for a specific Zstandard compression level, 0 means default.");
#define _ZSTD_FINALIZE_DICT_METHODDEF \ #define _ZSTD_FINALIZE_DICT_METHODDEF \
{"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__}, {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__},
@ -149,7 +149,7 @@ PyDoc_STRVAR(_zstd_get_param_bounds__doc__,
"get_param_bounds($module, /, parameter, is_compress)\n" "get_param_bounds($module, /, parameter, is_compress)\n"
"--\n" "--\n"
"\n" "\n"
"Internal function, get CompressionParameter/DecompressionParameter bounds.\n" "Get CompressionParameter/DecompressionParameter bounds.\n"
"\n" "\n"
" parameter\n" " parameter\n"
" The parameter to get bounds.\n" " The parameter to get bounds.\n"
@ -220,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__,
"get_frame_size($module, /, frame_buffer)\n" "get_frame_size($module, /, frame_buffer)\n"
"--\n" "--\n"
"\n" "\n"
"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n" "Get the size of a Zstandard frame, including the header and optional checksum.\n"
"\n" "\n"
" frame_buffer\n" " frame_buffer\n"
" A bytes-like object, it should start from the beginning of a frame,\n" " A bytes-like object, it should start from the beginning of a frame,\n"
" and contains at least one complete frame.\n" " and contains at least one complete frame.");
"\n"
"It will iterate all blocks\' headers within a frame, to accumulate the frame size.");
#define _ZSTD_GET_FRAME_SIZE_METHODDEF \ #define _ZSTD_GET_FRAME_SIZE_METHODDEF \
{"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__}, {"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__},
@ -291,10 +289,10 @@ PyDoc_STRVAR(_zstd_get_frame_info__doc__,
"get_frame_info($module, /, frame_buffer)\n" "get_frame_info($module, /, frame_buffer)\n"
"--\n" "--\n"
"\n" "\n"
"Internal function, get zstd frame infomation from a frame header.\n" "Get Zstandard frame infomation from a frame header.\n"
"\n" "\n"
" frame_buffer\n" " frame_buffer\n"
" A bytes-like object, containing the header of a zstd frame."); " A bytes-like object, containing the header of a Zstandard frame.");
#define _ZSTD_GET_FRAME_INFO_METHODDEF \ #define _ZSTD_GET_FRAME_INFO_METHODDEF \
{"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__}, {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__},
@ -359,7 +357,7 @@ PyDoc_STRVAR(_zstd_set_parameter_types__doc__,
"set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" "set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n"
"--\n" "--\n"
"\n" "\n"
"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n" "Set CompressionParameter and DecompressionParameter types for validity check.\n"
"\n" "\n"
" c_parameter_type\n" " c_parameter_type\n"
" CompressionParameter IntEnum type object\n" " CompressionParameter IntEnum type object\n"
@ -428,4 +426,4 @@ _zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t na
exit: exit:
return return_value; return return_value;
} }
/*[clinic end generated code: output=8445b658dcdcbb9c input=a9049054013a1b77]*/ /*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/

28
Modules/_zstd/clinic/compressor.c.h generated
View file

@ -8,30 +8,30 @@ preserve
#endif #endif
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__,
"ZstdCompressor(level=None, options=None, zstd_dict=None)\n" "ZstdCompressor(level=None, options=None, zstd_dict=None)\n"
"--\n" "--\n"
"\n" "\n"
"Create a compressor object for compressing data incrementally.\n" "Create a compressor object for compressing data incrementally.\n"
"\n" "\n"
" level\n" " level\n"
" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n" " The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n"
" options\n" " options\n"
" A dict object that contains advanced compression parameters.\n" " A dict object that contains advanced compression parameters.\n"
" zstd_dict\n" " zstd_dict\n"
" A ZstdDict object, a pre-trained zstd dictionary.\n" " A ZstdDict object, a pre-trained Zstandard dictionary.\n"
"\n" "\n"
"Thread-safe at method level. For one-shot compression, use the compress()\n" "Thread-safe at method level. For one-shot compression, use the compress()\n"
"function instead."); "function instead.");
static int static PyObject *
_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
PyObject *options, PyObject *zstd_dict); PyObject *options, PyObject *zstd_dict);
static int static PyObject *
_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{ {
int return_value = -1; PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 3 #define NUM_KEYWORDS 3
@ -89,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
} }
zstd_dict = fastargs[2]; zstd_dict = fastargs[2];
skip_optional_pos: skip_optional_pos:
return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict); return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict);
exit: exit:
return return_value; return return_value;
@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__,
" Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n"
" ZstdCompressor.FLUSH_BLOCK\n" " ZstdCompressor.FLUSH_BLOCK\n"
"\n" "\n"
"Flush any remaining data left in internal buffers. Since zstd data consists\n" "Flush any remaining data left in internal buffers. Since Zstandard data\n"
"of one or more independent frames, the compressor object can still be used\n" "consists of one or more independent frames, the compressor object can still\n"
"after this method is called."); "be used after this method is called.");
#define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \
{"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__},
@ -252,4 +252,4 @@ skip_optional_pos:
exit: exit:
return return_value; return return_value;
} }
/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/ /*[clinic end generated code: output=ee2d1dc298de790c input=a9049054013a1b77]*/

22
Modules/_zstd/clinic/decompressor.c.h generated
View file

@ -10,28 +10,28 @@ preserve
#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__,
"ZstdDecompressor(zstd_dict=None, options=None)\n" "ZstdDecompressor(zstd_dict=None, options=None)\n"
"--\n" "--\n"
"\n" "\n"
"Create a decompressor object for decompressing data incrementally.\n" "Create a decompressor object for decompressing data incrementally.\n"
"\n" "\n"
" zstd_dict\n" " zstd_dict\n"
" A ZstdDict object, a pre-trained zstd dictionary.\n" " A ZstdDict object, a pre-trained Zstandard dictionary.\n"
" options\n" " options\n"
" A dict object that contains advanced decompression parameters.\n" " A dict object that contains advanced decompression parameters.\n"
"\n" "\n"
"Thread-safe at method level. For one-shot decompression, use the decompress()\n" "Thread-safe at method level. For one-shot decompression, use the decompress()\n"
"function instead."); "function instead.");
static int static PyObject *
_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
PyObject *zstd_dict, PyObject *options); PyObject *options);
static int static PyObject *
_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{ {
int return_value = -1; PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 2 #define NUM_KEYWORDS 2
@ -82,7 +82,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs
} }
options = fastargs[1]; options = fastargs[1];
skip_optional_pos: skip_optional_pos:
return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options); return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options);
exit: exit:
return return_value; return return_value;
@ -130,7 +130,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__,
"Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n" "Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n"
"\n" "\n"
" data\n" " data\n"
" A bytes-like object, zstd data to be decompressed.\n" " A bytes-like object, Zstandard data to be decompressed.\n"
" max_length\n" " max_length\n"
" Maximum size of returned data. When it is negative, the size of\n" " Maximum size of returned data. When it is negative, the size of\n"
" output buffer is unlimited. When it is nonnegative, returns at\n" " output buffer is unlimited. When it is nonnegative, returns at\n"
@ -227,4 +227,4 @@ exit:
return return_value; return return_value;
} }
/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/ /*[clinic end generated code: output=7a4d278f9244e684 input=a9049054013a1b77]*/

48
Modules/_zstd/clinic/zstddict.c.h generated
View file

@ -9,35 +9,33 @@ preserve
#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, PyDoc_STRVAR(_zstd_ZstdDict_new__doc__,
"ZstdDict(dict_content, is_raw=False)\n" "ZstdDict(dict_content, /, *, is_raw=False)\n"
"--\n" "--\n"
"\n" "\n"
"Represents a zstd dictionary, which can be used for compression/decompression.\n" "Represents a Zstandard dictionary.\n"
"\n" "\n"
" dict_content\n" " dict_content\n"
" A bytes-like object, dictionary\'s content.\n" " The content of a Zstandard dictionary as a bytes-like object.\n"
" is_raw\n" " is_raw\n"
" This parameter is for advanced user. True means dict_content\n" " If true, perform no checks on *dict_content*, useful for some\n"
" argument is a \"raw content\" dictionary, free of any format\n" " advanced cases. Otherwise, check that the content represents\n"
" restriction. False means dict_content argument is an ordinary\n" " a Zstandard dictionary created by the zstd library or CLI.\n"
" zstd dictionary, was created by zstd functions, follow a\n"
" specified format.\n"
"\n" "\n"
"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" "The dictionary can be used for compression or decompression, and can be shared\n"
"ZstdDecompressor objects."); "by multiple ZstdCompressor or ZstdDecompressor objects.");
static int static PyObject *
_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content,
int is_raw); int is_raw);
static int static PyObject *
_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) _zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{ {
int return_value = -1; PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 2 #define NUM_KEYWORDS 1
static struct { static struct {
PyGC_Head _this_is_not_used; PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD PyObject_VAR_HEAD
@ -46,7 +44,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
} _kwtuple = { } _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_hash = -1, .ob_hash = -1,
.ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), }, .ob_item = { &_Py_ID(is_raw), },
}; };
#undef NUM_KEYWORDS #undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base) #define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -55,7 +53,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
# define KWTUPLE NULL # define KWTUPLE NULL
#endif // !Py_BUILD_CORE #endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"dict_content", "is_raw", NULL}; static const char * const _keywords[] = {"", "is_raw", NULL};
static _PyArg_Parser _parser = { static _PyArg_Parser _parser = {
.keywords = _keywords, .keywords = _keywords,
.fname = "ZstdDict", .fname = "ZstdDict",
@ -70,20 +68,20 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
int is_raw = 0; int is_raw = 0;
fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser,
/*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
if (!fastargs) { if (!fastargs) {
goto exit; goto exit;
} }
dict_content = fastargs[0]; dict_content = fastargs[0];
if (!noptargs) { if (!noptargs) {
goto skip_optional_pos; goto skip_optional_kwonly;
} }
is_raw = PyObject_IsTrue(fastargs[1]); is_raw = PyObject_IsTrue(fastargs[1]);
if (is_raw < 0) { if (is_raw < 0) {
goto exit; goto exit;
} }
skip_optional_pos: skip_optional_kwonly:
return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw); return_value = _zstd_ZstdDict_new_impl(type, dict_content, is_raw);
exit: exit:
return return_value; return return_value;
@ -204,4 +202,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context))
return return_value; return return_value;
} }
/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/ /*[clinic end generated code: output=bfb31c1187477afd input=a9049054013a1b77]*/

148
Modules/_zstd/compressor.c
View file

@ -1,7 +1,4 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
/* ZstdCompressor class definitions */ /* ZstdCompressor class definitions */
@ -21,7 +18,6 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
#include "buffer.h" #include "buffer.h"
#include "zstddict.h" #include "zstddict.h"
#include <stdbool.h> // bool
#include <stddef.h> // offsetof() #include <stddef.h> // offsetof()
#include <zstd.h> // ZSTD_*() #include <zstd.h> // ZSTD_*()
@ -42,9 +38,6 @@ typedef struct {
/* Compression level */ /* Compression level */
int compression_level; int compression_level;
/* __init__ has been called, 0 or 1. */
bool initialized;
} ZstdCompressor; } ZstdCompressor;
#define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
@ -122,7 +115,7 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
self->compression_level = value_v; self->compression_level = value_v;
} }
else if (key_v == ZSTD_c_nbWorkers) { else if (key_v == ZSTD_c_nbWorkers) {
/* From zstd library doc: /* From the zstd library docs:
1. When nbWorkers >= 1, triggers asynchronous mode when 1. When nbWorkers >= 1, triggers asynchronous mode when
used with ZSTD_compressStream2(). used with ZSTD_compressStream2().
2, Default value is `0`, aka "single-threaded mode" : no 2, Default value is `0`, aka "single-threaded mode" : no
@ -189,8 +182,8 @@ _get_CDict(ZstdDict *self, int compressionLevel)
_zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) { if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError, PyErr_SetString(mod_state->ZstdError,
"Failed to create ZSTD_CDict instance from zstd " "Failed to create a ZSTD_CDict instance from "
"dictionary content. Maybe the content is corrupted."); "Zstandard dictionary content.");
} }
goto error; goto error;
} }
@ -318,20 +311,34 @@ load:
return 0; return 0;
} }
/*[clinic input]
@classmethod
_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new
level: object = None
The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.
options: object = None
A dict object that contains advanced compression parameters.
zstd_dict: object = None
A ZstdDict object, a pre-trained Zstandard dictionary.
Create a compressor object for compressing data incrementally.
Thread-safe at method level. For one-shot compression, use the compress()
function instead.
[clinic start generated code]*/
static PyObject * static PyObject *
_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
PyObject *options, PyObject *zstd_dict)
/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/
{ {
ZstdCompressor *self; ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type);
self = PyObject_GC_New(ZstdCompressor, type);
if (self == NULL) { if (self == NULL) {
goto error; goto error;
} }
self->initialized = 0;
self->dict = NULL;
self->use_multithread = 0; self->use_multithread = 0;
/* Compression context */ /* Compression context */
self->cctx = ZSTD_createCCtx(); self->cctx = ZSTD_createCCtx();
if (self->cctx == NULL) { if (self->cctx == NULL) {
@ -346,6 +353,37 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject
/* Last mode */ /* Last mode */
self->last_mode = ZSTD_e_end; self->last_mode = ZSTD_e_end;
if (level != Py_None && options != Py_None) {
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
goto error;
}
/* Set compressLevel/options to compression context */
if (level != Py_None) {
if (_zstd_set_c_parameters(self, level, "level", "int") < 0) {
goto error;
}
}
if (options != Py_None) {
if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) {
goto error;
}
}
/* Load Zstandard dictionary to compression context */
self->dict = NULL;
if (zstd_dict != Py_None) {
if (_zstd_load_c_dict(self, zstd_dict) < 0) {
goto error;
}
Py_INCREF(zstd_dict);
self->dict = zstd_dict;
}
// We can only start GC tracking once self->dict is set.
PyObject_GC_Track(self);
return (PyObject*)self; return (PyObject*)self;
error: error:
@ -373,67 +411,6 @@ ZstdCompressor_dealloc(PyObject *ob)
Py_DECREF(tp); Py_DECREF(tp);
} }
/*[clinic input]
_zstd.ZstdCompressor.__init__
level: object = None
The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.
options: object = None
A dict object that contains advanced compression parameters.
zstd_dict: object = None
A ZstdDict object, a pre-trained zstd dictionary.
Create a compressor object for compressing data incrementally.
Thread-safe at method level. For one-shot compression, use the compress()
function instead.
[clinic start generated code]*/
static int
_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
PyObject *options, PyObject *zstd_dict)
/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/
{
if (self->initialized) {
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
return -1;
}
self->initialized = 1;
if (level != Py_None && options != Py_None) {
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
return -1;
}
/* Set compressLevel/options to compression context */
if (level != Py_None) {
if (_zstd_set_c_parameters(self, level, "level", "int") < 0) {
return -1;
}
}
if (options != Py_None) {
if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) {
return -1;
}
}
/* Load dictionary to compression context */
if (zstd_dict != Py_None) {
if (_zstd_load_c_dict(self, zstd_dict) < 0) {
return -1;
}
/* Py_INCREF the dict */
Py_INCREF(zstd_dict);
self->dict = zstd_dict;
}
// We can only start tracking self with the GC once self->dict is set.
PyObject_GC_Track(self);
return 0;
}
static PyObject * static PyObject *
compress_impl(ZstdCompressor *self, Py_buffer *data, compress_impl(ZstdCompressor *self, Py_buffer *data,
ZSTD_EndDirective end_directive) ZSTD_EndDirective end_directive)
@ -469,7 +446,7 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
} }
/* zstd stream compress */ /* Zstandard stream compress */
while (1) { while (1) {
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive); zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive);
@ -533,7 +510,7 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
goto error; goto error;
} }
/* zstd stream compress */ /* Zstandard stream compress */
while (1) { while (1) {
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
do { do {
@ -642,14 +619,14 @@ _zstd.ZstdCompressor.flush
Finish the compression process. Finish the compression process.
Flush any remaining data left in internal buffers. Since zstd data consists Flush any remaining data left in internal buffers. Since Zstandard data
of one or more independent frames, the compressor object can still be used consists of one or more independent frames, the compressor object can still
after this method is called. be used after this method is called.
[clinic start generated code]*/ [clinic start generated code]*/
static PyObject * static PyObject *
_zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/ /*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=0ab19627f323cdbc]*/
{ {
PyObject *ret; PyObject *ret;
@ -717,10 +694,9 @@ ZstdCompressor_clear(PyObject *ob)
static PyType_Slot zstdcompressor_slots[] = { static PyType_Slot zstdcompressor_slots[] = {
{Py_tp_new, _zstd_ZstdCompressor_new}, {Py_tp_new, _zstd_ZstdCompressor_new},
{Py_tp_dealloc, ZstdCompressor_dealloc}, {Py_tp_dealloc, ZstdCompressor_dealloc},
{Py_tp_init, _zstd_ZstdCompressor___init__},
{Py_tp_methods, ZstdCompressor_methods}, {Py_tp_methods, ZstdCompressor_methods},
{Py_tp_members, ZstdCompressor_members}, {Py_tp_members, ZstdCompressor_members},
{Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__}, {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__},
{Py_tp_traverse, ZstdCompressor_traverse}, {Py_tp_traverse, ZstdCompressor_traverse},
{Py_tp_clear, ZstdCompressor_clear}, {Py_tp_clear, ZstdCompressor_clear},
{0, 0} {0, 0}

119
Modules/_zstd/decompressor.c
View file

@ -1,7 +1,4 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
/* ZstdDecompressor class definition */ /* ZstdDecompressor class definition */
@ -48,9 +45,6 @@ typedef struct {
/* For ZstdDecompressor, 0 or 1. /* For ZstdDecompressor, 0 or 1.
1 means the end of the first frame has been reached. */ 1 means the end of the first frame has been reached. */
bool eof; bool eof;
/* __init__ has been called, 0 or 1. */
bool initialized;
} ZstdDecompressor; } ZstdDecompressor;
#define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
@ -81,8 +75,8 @@ _get_DDict(ZstdDict *self)
_zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
if (mod_state != NULL) { if (mod_state != NULL) {
PyErr_SetString(mod_state->ZstdError, PyErr_SetString(mod_state->ZstdError,
"Failed to create ZSTD_DDict instance from zstd " "Failed to create a ZSTD_DDict instance from "
"dictionary content. Maybe the content is corrupted."); "Zstandard dictionary content.");
} }
} }
} }
@ -265,8 +259,8 @@ load:
finish finish
ZSTD_decompressStream()'s size_t return value: ZSTD_decompressStream()'s size_t return value:
- 0 when a frame is completely decoded and fully flushed, zstd's internal - 0 when a frame is completely decoded and fully flushed,
buffer has no data. zstd's internal buffer has no data.
- An error code, which can be tested using ZSTD_isError(). - An error code, which can be tested using ZSTD_isError().
- Or any other value > 0, which means there is still some decoding or - Or any other value > 0, which means there is still some decoding or
flushing to do to complete current frame. flushing to do to complete current frame.
@ -311,7 +305,7 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
} }
/* Need to check out before in. Maybe zstd's internal buffer still has /* Need to check out before in. Maybe zstd's internal buffer still has
a few bytes can be output, grow the buffer and continue. */ a few bytes that can be output, grow the buffer and continue. */
if (out.pos == out.size) { if (out.pos == out.size) {
/* Output buffer exhausted */ /* Output buffer exhausted */
@ -373,7 +367,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
/* Check .eof flag */ /* Check .eof flag */
if (self->eof) { if (self->eof) {
PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame."); PyErr_SetString(PyExc_EOFError, "Already at the end of a Zstandard frame.");
assert(ret == NULL); assert(ret == NULL);
return NULL; return NULL;
} }
@ -530,17 +524,30 @@ error:
} }
/*[clinic input]
@classmethod
_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new
zstd_dict: object = None
A ZstdDict object, a pre-trained Zstandard dictionary.
options: object = None
A dict object that contains advanced decompression parameters.
Create a decompressor object for decompressing data incrementally.
Thread-safe at method level. For one-shot decompression, use the decompress()
function instead.
[clinic start generated code]*/
static PyObject * static PyObject *
_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
PyObject *options)
/*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/
{ {
ZstdDecompressor *self; ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type);
self = PyObject_GC_New(ZstdDecompressor, type);
if (self == NULL) { if (self == NULL) {
goto error; goto error;
} }
self->initialized = 0;
self->dict = NULL;
self->input_buffer = NULL; self->input_buffer = NULL;
self->input_buffer_size = 0; self->input_buffer_size = 0;
self->in_begin = -1; self->in_begin = -1;
@ -562,6 +569,26 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
goto error; goto error;
} }
/* Load Zstandard dictionary to decompression context */
self->dict = NULL;
if (zstd_dict != Py_None) {
if (_zstd_load_d_dict(self, zstd_dict) < 0) {
goto error;
}
Py_INCREF(zstd_dict);
self->dict = zstd_dict;
}
/* Set option to decompression context */
if (options != Py_None) {
if (_zstd_set_d_parameters(self, options) < 0) {
goto error;
}
}
// We can only start GC tracking once self->dict is set.
PyObject_GC_Track(self);
return (PyObject*)self; return (PyObject*)self;
error: error:
@ -595,55 +622,6 @@ ZstdDecompressor_dealloc(PyObject *ob)
Py_DECREF(tp); Py_DECREF(tp);
} }
/*[clinic input]
_zstd.ZstdDecompressor.__init__
zstd_dict: object = None
A ZstdDict object, a pre-trained zstd dictionary.
options: object = None
A dict object that contains advanced decompression parameters.
Create a decompressor object for decompressing data incrementally.
Thread-safe at method level. For one-shot decompression, use the decompress()
function instead.
[clinic start generated code]*/
static int
_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self,
PyObject *zstd_dict, PyObject *options)
/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/
{
/* Only called once */
if (self->initialized) {
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
return -1;
}
self->initialized = 1;
/* Load dictionary to decompression context */
if (zstd_dict != Py_None) {
if (_zstd_load_d_dict(self, zstd_dict) < 0) {
return -1;
}
/* Py_INCREF the dict */
Py_INCREF(zstd_dict);
self->dict = zstd_dict;
}
/* Set option to decompression context */
if (options != Py_None) {
if (_zstd_set_d_parameters(self, options) < 0) {
return -1;
}
}
// We can only start tracking self with the GC once self->dict is set.
PyObject_GC_Track(self);
return 0;
}
/*[clinic input] /*[clinic input]
@critical_section @critical_section
@getter @getter
@ -685,7 +663,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
_zstd.ZstdDecompressor.decompress _zstd.ZstdDecompressor.decompress
data: Py_buffer data: Py_buffer
A bytes-like object, zstd data to be decompressed. A bytes-like object, Zstandard data to be decompressed.
max_length: Py_ssize_t = -1 max_length: Py_ssize_t = -1
Maximum size of returned data. When it is negative, the size of Maximum size of returned data. When it is negative, the size of
output buffer is unlimited. When it is nonnegative, returns at output buffer is unlimited. When it is nonnegative, returns at
@ -711,7 +689,7 @@ static PyObject *
_zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
Py_buffer *data, Py_buffer *data,
Py_ssize_t max_length) Py_ssize_t max_length)
/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/ /*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/
{ {
PyObject *ret; PyObject *ret;
/* Thread-safe code */ /* Thread-safe code */
@ -769,11 +747,10 @@ ZstdDecompressor_clear(PyObject *ob)
static PyType_Slot ZstdDecompressor_slots[] = { static PyType_Slot ZstdDecompressor_slots[] = {
{Py_tp_new, _zstd_ZstdDecompressor_new}, {Py_tp_new, _zstd_ZstdDecompressor_new},
{Py_tp_dealloc, ZstdDecompressor_dealloc}, {Py_tp_dealloc, ZstdDecompressor_dealloc},
{Py_tp_init, _zstd_ZstdDecompressor___init__},
{Py_tp_methods, ZstdDecompressor_methods}, {Py_tp_methods, ZstdDecompressor_methods},
{Py_tp_members, ZstdDecompressor_members}, {Py_tp_members, ZstdDecompressor_members},
{Py_tp_getset, ZstdDecompressor_getset}, {Py_tp_getset, ZstdDecompressor_getset},
{Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__}, {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__},
{Py_tp_traverse, ZstdDecompressor_traverse}, {Py_tp_traverse, ZstdDecompressor_traverse},
{Py_tp_clear, ZstdDecompressor_clear}, {Py_tp_clear, ZstdDecompressor_clear},
{0, 0} {0, 0}

144
Modules/_zstd/zstddict.c
View file

@ -1,7 +1,4 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
/* ZstdDict class definitions */ /* ZstdDict class definitions */
@ -25,17 +22,35 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
#define ZstdDict_CAST(op) ((ZstdDict *)op) #define ZstdDict_CAST(op) ((ZstdDict *)op)
/*[clinic input]
@classmethod
_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new
dict_content: object
The content of a Zstandard dictionary as a bytes-like object.
/
*
is_raw: bool = False
If true, perform no checks on *dict_content*, useful for some
advanced cases. Otherwise, check that the content represents
a Zstandard dictionary created by the zstd library or CLI.
Represents a Zstandard dictionary.
The dictionary can be used for compression or decompression, and can be shared
by multiple ZstdCompressor or ZstdDecompressor objects.
[clinic start generated code]*/
static PyObject * static PyObject *
_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content,
int is_raw)
/*[clinic end generated code: output=3ebff839cb3be6d7 input=6b5de413869ae878]*/
{ {
ZstdDict *self; ZstdDict* self = PyObject_GC_New(ZstdDict, type);
self = PyObject_GC_New(ZstdDict, type);
if (self == NULL) { if (self == NULL) {
goto error; goto error;
} }
self->dict_content = NULL; self->dict_content = NULL;
self->initialized = 0;
self->d_dict = NULL; self->d_dict = NULL;
/* ZSTD_CDict dict */ /* ZSTD_CDict dict */
@ -44,6 +59,36 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U
goto error; goto error;
} }
/* Check dict_content's type */
self->dict_content = PyBytes_FromObject(dict_content);
if (self->dict_content == NULL) {
PyErr_SetString(PyExc_TypeError,
"dict_content argument should be bytes-like object.");
goto error;
}
/* Both ordinary dictionary and "raw content" dictionary should
at least 8 bytes */
if (Py_SIZE(self->dict_content) < 8) {
PyErr_SetString(PyExc_ValueError,
"Zstandard dictionary content should at least 8 bytes.");
goto error;
}
/* Get dict_id, 0 means "raw content" dictionary. */
self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content),
Py_SIZE(self->dict_content));
/* Check validity for ordinary dictionary */
if (!is_raw && self->dict_id == 0) {
char *msg = "Invalid Zstandard dictionary and is_raw not set.\n";
PyErr_SetString(PyExc_ValueError, msg);
goto error;
}
// Can only track self once self->dict_content is included
PyObject_GC_Track(self);
return (PyObject*)self; return (PyObject*)self;
error: error:
@ -72,83 +117,15 @@ ZstdDict_dealloc(PyObject *ob)
Py_DECREF(tp); Py_DECREF(tp);
} }
/*[clinic input]
_zstd.ZstdDict.__init__
dict_content: object
A bytes-like object, dictionary's content.
is_raw: bool = False
This parameter is for advanced user. True means dict_content
argument is a "raw content" dictionary, free of any format
restriction. False means dict_content argument is an ordinary
zstd dictionary, was created by zstd functions, follow a
specified format.
Represents a zstd dictionary, which can be used for compression/decompression.
It's thread-safe, and can be shared by multiple ZstdCompressor /
ZstdDecompressor objects.
[clinic start generated code]*/
static int
_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
int is_raw)
/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/
{
/* Only called once */
if (self->initialized) {
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
return -1;
}
self->initialized = 1;
/* Check dict_content's type */
self->dict_content = PyBytes_FromObject(dict_content);
if (self->dict_content == NULL) {
PyErr_SetString(PyExc_TypeError,
"dict_content argument should be bytes-like object.");
return -1;
}
/* Both ordinary dictionary and "raw content" dictionary should
at least 8 bytes */
if (Py_SIZE(self->dict_content) < 8) {
PyErr_SetString(PyExc_ValueError,
"Zstd dictionary content should at least 8 bytes.");
return -1;
}
/* Get dict_id, 0 means "raw content" dictionary. */
self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content),
Py_SIZE(self->dict_content));
/* Check validity for ordinary dictionary */
if (!is_raw && self->dict_id == 0) {
char *msg = "The dict_content argument is not a valid zstd "
"dictionary. The first 4 bytes of a valid zstd dictionary "
"should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n"
"If you are an advanced user, and can be sure that "
"dict_content argument is a \"raw content\" zstd "
"dictionary, set is_raw parameter to True.";
PyErr_SetString(PyExc_ValueError, msg);
return -1;
}
// Can only track self once self->dict_content is included
PyObject_GC_Track(self);
return 0;
}
PyDoc_STRVAR(ZstdDict_dictid_doc, PyDoc_STRVAR(ZstdDict_dictid_doc,
"ID of zstd dictionary, a 32-bit unsigned int value.\n\n" "the Zstandard dictionary, an int between 0 and 2**32.\n\n"
"Non-zero means ordinary dictionary, was created by zstd functions, follow\n" "A non-zero value represents an ordinary Zstandard dictionary, "
"a specified format.\n\n" "conforming to the standardised format.\n\n"
"0 means a \"raw content\" dictionary, free of any format restriction, used\n" "The special value '0' means a 'raw content' dictionary,"
"for advanced user."); "without any restrictions on format or content.");
PyDoc_STRVAR(ZstdDict_dictcontent_doc, PyDoc_STRVAR(ZstdDict_dictcontent_doc,
"The content of zstd dictionary, a bytes object, it's the same as dict_content\n" "The content of a Zstandard dictionary, as a bytes object.");
"argument in ZstdDict.__init__() method. It can be used with other programs.");
static PyObject * static PyObject *
ZstdDict_str(PyObject *ob) ZstdDict_str(PyObject *ob)
@ -266,9 +243,8 @@ static PyType_Slot zstddict_slots[] = {
{Py_tp_getset, ZstdDict_getset}, {Py_tp_getset, ZstdDict_getset},
{Py_tp_new, _zstd_ZstdDict_new}, {Py_tp_new, _zstd_ZstdDict_new},
{Py_tp_dealloc, ZstdDict_dealloc}, {Py_tp_dealloc, ZstdDict_dealloc},
{Py_tp_init, _zstd_ZstdDict___init__},
{Py_tp_str, ZstdDict_str}, {Py_tp_str, ZstdDict_str},
{Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__}, {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__},
{Py_sq_length, ZstdDict_length}, {Py_sq_length, ZstdDict_length},
{Py_tp_traverse, ZstdDict_traverse}, {Py_tp_traverse, ZstdDict_traverse},
{Py_tp_clear, ZstdDict_clear}, {Py_tp_clear, ZstdDict_clear},

9
Modules/_zstd/zstddict.h
View file

@ -1,12 +1,8 @@
/* /* Low level interface to the Zstandard algorthm & the zstd library. */
Low level interface to Meta's zstd library for use in the compression.zstd
Python module.
*/
#ifndef ZSTD_DICT_H #ifndef ZSTD_DICT_H
#define ZSTD_DICT_H #define ZSTD_DICT_H
#include <stdbool.h> // bool
#include <zstd.h> // ZSTD_DDict #include <zstd.h> // ZSTD_DDict
typedef struct { typedef struct {
@ -23,9 +19,6 @@ typedef struct {
PyObject *dict_content; PyObject *dict_content;
/* Dictionary id */ /* Dictionary id */
uint32_t dict_id; uint32_t dict_id;
/* __init__ has been called, 0 or 1. */
bool initialized;
} ZstdDict; } ZstdDict;
#endif // !ZSTD_DICT_H #endif // !ZSTD_DICT_H