mirror of
https://github.com/python/cpython.git
synced 2025-07-25 04:04:13 +00:00
gh-132983: Convert zstd `__new__` methods to Argument Clinic (#133860)
This commit is contained in:
Adam Turner
GitHub
parent
0eb448cae5
commit
d29ddbd90c
16 changed files with 275 additions and 371 deletions
|
|
@ -902,7 +902,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
|
|||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic));
|
||||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device));
|
||||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict));
|
||||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict_content));
|
||||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp));
|
||||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update));
|
||||
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest));
|
||||
|
|
|
|||
|
|
@ -393,7 +393,6 @@ struct _Py_global_strings {
|
|||
STRUCT_FOR_ID(deterministic)
|
||||
STRUCT_FOR_ID(device)
|
||||
STRUCT_FOR_ID(dict)
|
||||
STRUCT_FOR_ID(dict_content)
|
||||
STRUCT_FOR_ID(dictcomp)
|
||||
STRUCT_FOR_ID(difference_update)
|
||||
STRUCT_FOR_ID(digest)
|
||||
|
|
|
|||
1
Include/internal/pycore_runtime_init_generated.h
generated
1
Include/internal/pycore_runtime_init_generated.h
generated
|
|
@ -900,7 +900,6 @@ extern "C" {
|
|||
INIT_ID(deterministic), \
|
||||
INIT_ID(device), \
|
||||
INIT_ID(dict), \
|
||||
INIT_ID(dict_content), \
|
||||
INIT_ID(dictcomp), \
|
||||
INIT_ID(difference_update), \
|
||||
INIT_ID(digest), \
|
||||
|
|
|
|||
|
|
@ -1360,10 +1360,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
|
|||
_PyUnicode_InternStatic(interp, &string);
|
||||
assert(_PyUnicode_CheckConsistency(string, 1));
|
||||
assert(PyUnicode_GET_LENGTH(string) != 1);
|
||||
string = &_Py_ID(dict_content);
|
||||
_PyUnicode_InternStatic(interp, &string);
|
||||
assert(_PyUnicode_CheckConsistency(string, 1));
|
||||
assert(PyUnicode_GET_LENGTH(string) != 1);
|
||||
string = &_Py_ID(dictcomp);
|
||||
_PyUnicode_InternStatic(interp, &string);
|
||||
assert(_PyUnicode_CheckConsistency(string, 1));
|
||||
|
|
|
|||
|
|
@ -288,8 +288,8 @@ class CompressorTestCase(unittest.TestCase):
|
|||
KEY = 100001234
|
||||
option = {CompressionParameter.compression_level: 10,
|
||||
KEY: 200000000}
|
||||
pattern = r'Zstd compression parameter.*?"unknown parameter \(key %d\)"' \
|
||||
% KEY
|
||||
pattern = (r'Invalid zstd compression parameter.*?'
|
||||
fr'"unknown parameter \(key {KEY}\)"')
|
||||
with self.assertRaisesRegex(ZstdError, pattern):
|
||||
ZstdCompressor(options=option)
|
||||
|
||||
|
|
@ -420,8 +420,8 @@ class DecompressorTestCase(unittest.TestCase):
|
|||
KEY = 100001234
|
||||
options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1],
|
||||
KEY: 200000000}
|
||||
pattern = r'Zstd decompression parameter.*?"unknown parameter \(key %d\)"' \
|
||||
% KEY
|
||||
pattern = (r'Invalid zstd decompression parameter.*?'
|
||||
fr'"unknown parameter \(key {KEY}\)"')
|
||||
with self.assertRaisesRegex(ZstdError, pattern):
|
||||
ZstdDecompressor(options=options)
|
||||
|
||||
|
|
@ -507,7 +507,7 @@ class DecompressorTestCase(unittest.TestCase):
|
|||
self.assertFalse(d.needs_input)
|
||||
|
||||
def test_decompressor_arg(self):
|
||||
zd = ZstdDict(b'12345678', True)
|
||||
zd = ZstdDict(b'12345678', is_raw=True)
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
d = ZstdDecompressor(zstd_dict={})
|
||||
|
|
@ -1021,6 +1021,10 @@ class DecompressorFlagsTestCase(unittest.TestCase):
|
|||
class ZstdDictTestCase(unittest.TestCase):
|
||||
|
||||
def test_is_raw(self):
|
||||
# must be passed as a keyword argument
|
||||
with self.assertRaises(TypeError):
|
||||
ZstdDict(bytes(8), True)
|
||||
|
||||
# content < 8
|
||||
b = b'1234567'
|
||||
with self.assertRaises(ValueError):
|
||||
|
|
@ -1068,9 +1072,9 @@ class ZstdDictTestCase(unittest.TestCase):
|
|||
|
||||
# corrupted
|
||||
zd = ZstdDict(dict_content, is_raw=False)
|
||||
with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?corrupted'):
|
||||
with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'):
|
||||
ZstdCompressor(zstd_dict=zd.as_digested_dict)
|
||||
with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?corrupted'):
|
||||
with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'):
|
||||
ZstdDecompressor(zd)
|
||||
|
||||
# wrong type
|
||||
|
|
@ -1096,7 +1100,7 @@ class ZstdDictTestCase(unittest.TestCase):
|
|||
|
||||
|
||||
TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1)
|
||||
ZstdDict(TRAINED_DICT.dict_content, False)
|
||||
ZstdDict(TRAINED_DICT.dict_content, is_raw=False)
|
||||
|
||||
self.assertNotEqual(TRAINED_DICT.dict_id, 0)
|
||||
self.assertGreater(len(TRAINED_DICT.dict_content), 0)
|
||||
|
|
@ -1250,7 +1254,7 @@ class ZstdDictTestCase(unittest.TestCase):
|
|||
def test_as_prefix(self):
|
||||
# V1
|
||||
V1 = THIS_FILE_BYTES
|
||||
zd = ZstdDict(V1, True)
|
||||
zd = ZstdDict(V1, is_raw=True)
|
||||
|
||||
# V2
|
||||
mid = len(V1) // 2
|
||||
|
|
@ -1266,7 +1270,7 @@ class ZstdDictTestCase(unittest.TestCase):
|
|||
self.assertEqual(decompress(dat, zd.as_prefix), V2)
|
||||
|
||||
# use wrong prefix
|
||||
zd2 = ZstdDict(SAMPLES[0], True)
|
||||
zd2 = ZstdDict(SAMPLES[0], is_raw=True)
|
||||
try:
|
||||
decompressed = decompress(dat, zd2.as_prefix)
|
||||
except ZstdError: # expected
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
#ifndef Py_BUILD_CORE_BUILTIN
|
||||
# define Py_BUILD_CORE_MODULE 1
|
||||
|
|
@ -34,17 +31,17 @@ set_zstd_error(const _zstd_state* const state,
|
|||
switch (type)
|
||||
{
|
||||
case ERR_DECOMPRESS:
|
||||
msg = "Unable to decompress zstd data: %s";
|
||||
msg = "Unable to decompress Zstandard data: %s";
|
||||
break;
|
||||
case ERR_COMPRESS:
|
||||
msg = "Unable to compress zstd data: %s";
|
||||
msg = "Unable to compress Zstandard data: %s";
|
||||
break;
|
||||
|
||||
case ERR_LOAD_D_DICT:
|
||||
msg = "Unable to load zstd dictionary or prefix for decompression: %s";
|
||||
msg = "Unable to load Zstandard dictionary or prefix for decompression: %s";
|
||||
break;
|
||||
case ERR_LOAD_C_DICT:
|
||||
msg = "Unable to load zstd dictionary or prefix for compression: %s";
|
||||
msg = "Unable to load Zstandard dictionary or prefix for compression: %s";
|
||||
break;
|
||||
|
||||
case ERR_GET_C_BOUNDS:
|
||||
|
|
@ -58,10 +55,10 @@ set_zstd_error(const _zstd_state* const state,
|
|||
break;
|
||||
|
||||
case ERR_TRAIN_DICT:
|
||||
msg = "Unable to train zstd dictionary: %s";
|
||||
msg = "Unable to train the Zstandard dictionary: %s";
|
||||
break;
|
||||
case ERR_FINALIZE_DICT:
|
||||
msg = "Unable to finalize zstd dictionary: %s";
|
||||
msg = "Unable to finalize the Zstandard dictionary: %s";
|
||||
break;
|
||||
|
||||
default:
|
||||
|
|
@ -152,7 +149,7 @@ set_parameter_error(const _zstd_state* const state, int is_compress,
|
|||
}
|
||||
if (ZSTD_isError(bounds.error)) {
|
||||
PyErr_Format(state->ZstdError,
|
||||
"Zstd %s parameter \"%s\" is invalid.",
|
||||
"Invalid zstd %s parameter \"%s\".",
|
||||
type, name);
|
||||
return;
|
||||
}
|
||||
|
|
@ -187,13 +184,13 @@ _zstd.train_dict
|
|||
The size of the dictionary.
|
||||
/
|
||||
|
||||
Internal function, train a zstd dictionary on sample data.
|
||||
Train a Zstandard dictionary on sample data.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
|
||||
PyObject *samples_sizes, Py_ssize_t dict_size)
|
||||
/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/
|
||||
/*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/
|
||||
{
|
||||
// TODO(emmatyping): The preamble and suffix to this function and _finalize_dict
|
||||
// are pretty similar. We should see if we can refactor them to share that code.
|
||||
|
|
@ -258,7 +255,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes,
|
|||
chunk_sizes, (uint32_t)chunks_number);
|
||||
Py_END_ALLOW_THREADS
|
||||
|
||||
/* Check zstd dict error */
|
||||
/* Check Zstandard dict error */
|
||||
if (ZDICT_isError(zstd_ret)) {
|
||||
_zstd_state* const mod_state = get_zstd_state(module);
|
||||
set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret);
|
||||
|
|
@ -292,10 +289,10 @@ _zstd.finalize_dict
|
|||
dict_size: Py_ssize_t
|
||||
The size of the dictionary.
|
||||
compression_level: int
|
||||
Optimize for a specific zstd compression level, 0 means default.
|
||||
Optimize for a specific Zstandard compression level, 0 means default.
|
||||
/
|
||||
|
||||
Internal function, finalize a zstd dictionary.
|
||||
Finalize a Zstandard dictionary.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
|
|
@ -303,7 +300,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
|
|||
PyBytesObject *samples_bytes,
|
||||
PyObject *samples_sizes, Py_ssize_t dict_size,
|
||||
int compression_level)
|
||||
/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/
|
||||
/*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/
|
||||
{
|
||||
Py_ssize_t chunks_number;
|
||||
size_t *chunk_sizes = NULL;
|
||||
|
|
@ -360,7 +357,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
|
|||
|
||||
/* Parameters */
|
||||
|
||||
/* Optimize for a specific zstd compression level, 0 means default. */
|
||||
/* Optimize for a specific Zstandard compression level, 0 means default. */
|
||||
params.compressionLevel = compression_level;
|
||||
/* Write log to stderr, 0 = none. */
|
||||
params.notificationLevel = 0;
|
||||
|
|
@ -376,7 +373,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes,
|
|||
(uint32_t)chunks_number, params);
|
||||
Py_END_ALLOW_THREADS
|
||||
|
||||
/* Check zstd dict error */
|
||||
/* Check Zstandard dict error */
|
||||
if (ZDICT_isError(zstd_ret)) {
|
||||
_zstd_state* const mod_state = get_zstd_state(module);
|
||||
set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret);
|
||||
|
|
@ -407,12 +404,12 @@ _zstd.get_param_bounds
|
|||
is_compress: bool
|
||||
True for CompressionParameter, False for DecompressionParameter.
|
||||
|
||||
Internal function, get CompressionParameter/DecompressionParameter bounds.
|
||||
Get CompressionParameter/DecompressionParameter bounds.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress)
|
||||
/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/
|
||||
/*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/
|
||||
{
|
||||
ZSTD_bounds bound;
|
||||
if (is_compress) {
|
||||
|
|
@ -442,14 +439,12 @@ _zstd.get_frame_size
|
|||
A bytes-like object, it should start from the beginning of a frame,
|
||||
and contains at least one complete frame.
|
||||
|
||||
Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.
|
||||
|
||||
It will iterate all blocks' headers within a frame, to accumulate the frame size.
|
||||
Get the size of a Zstandard frame, including the header and optional checksum.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
|
||||
/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/
|
||||
/*[clinic end generated code: output=a7384c2f8780f442 input=3b9f73f8c8129d38]*/
|
||||
{
|
||||
size_t frame_size;
|
||||
|
||||
|
|
@ -457,9 +452,9 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
|
|||
if (ZSTD_isError(frame_size)) {
|
||||
_zstd_state* const mod_state = get_zstd_state(module);
|
||||
PyErr_Format(mod_state->ZstdError,
|
||||
"Error when finding the compressed size of a zstd frame. "
|
||||
"Make sure the frame_buffer argument starts from the "
|
||||
"beginning of a frame, and its length not less than this "
|
||||
"Error when finding the compressed size of a Zstandard frame. "
|
||||
"Ensure the frame_buffer argument starts from the "
|
||||
"beginning of a frame, and its length is not less than this "
|
||||
"complete frame. Zstd error message: %s.",
|
||||
ZSTD_getErrorName(frame_size));
|
||||
return NULL;
|
||||
|
|
@ -472,14 +467,14 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer)
|
|||
_zstd.get_frame_info
|
||||
|
||||
frame_buffer: Py_buffer
|
||||
A bytes-like object, containing the header of a zstd frame.
|
||||
A bytes-like object, containing the header of a Zstandard frame.
|
||||
|
||||
Internal function, get zstd frame infomation from a frame header.
|
||||
Get Zstandard frame infomation from a frame header.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
|
||||
/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/
|
||||
/*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/
|
||||
{
|
||||
uint64_t decompressed_size;
|
||||
uint32_t dict_id;
|
||||
|
|
@ -494,9 +489,9 @@ _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer)
|
|||
_zstd_state* const mod_state = get_zstd_state(module);
|
||||
PyErr_SetString(mod_state->ZstdError,
|
||||
"Error when getting information from the header of "
|
||||
"a zstd frame. Make sure the frame_buffer argument "
|
||||
"a Zstandard frame. Ensure the frame_buffer argument "
|
||||
"starts from the beginning of a frame, and its length "
|
||||
"not less than the frame header (6~18 bytes).");
|
||||
"is not less than the frame header (6~18 bytes).");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
@ -518,13 +513,13 @@ _zstd.set_parameter_types
|
|||
d_parameter_type: object(subclass_of='&PyType_Type')
|
||||
DecompressionParameter IntEnum type object
|
||||
|
||||
Internal function, set CompressionParameter/DecompressionParameter types for validity check.
|
||||
Set CompressionParameter and DecompressionParameter types for validity check.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type,
|
||||
PyObject *d_parameter_type)
|
||||
/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/
|
||||
/*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/
|
||||
{
|
||||
_zstd_state* const mod_state = get_zstd_state(module);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
/* Declarations shared between different parts of the _zstd module*/
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
#ifndef ZSTD_BUFFER_H
|
||||
#define ZSTD_BUFFER_H
|
||||
|
|
|
|||
22
Modules/_zstd/clinic/_zstdmodule.c.h
generated
22
Modules/_zstd/clinic/_zstdmodule.c.h
generated
|
|
@ -13,7 +13,7 @@ PyDoc_STRVAR(_zstd_train_dict__doc__,
|
|||
"train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Internal function, train a zstd dictionary on sample data.\n"
|
||||
"Train a Zstandard dictionary on sample data.\n"
|
||||
"\n"
|
||||
" samples_bytes\n"
|
||||
" Concatenation of samples.\n"
|
||||
|
|
@ -73,7 +73,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__,
|
|||
" dict_size, compression_level, /)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Internal function, finalize a zstd dictionary.\n"
|
||||
"Finalize a Zstandard dictionary.\n"
|
||||
"\n"
|
||||
" custom_dict_bytes\n"
|
||||
" Custom dictionary content.\n"
|
||||
|
|
@ -84,7 +84,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__,
|
|||
" dict_size\n"
|
||||
" The size of the dictionary.\n"
|
||||
" compression_level\n"
|
||||
" Optimize for a specific zstd compression level, 0 means default.");
|
||||
" Optimize for a specific Zstandard compression level, 0 means default.");
|
||||
|
||||
#define _ZSTD_FINALIZE_DICT_METHODDEF \
|
||||
{"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__},
|
||||
|
|
@ -149,7 +149,7 @@ PyDoc_STRVAR(_zstd_get_param_bounds__doc__,
|
|||
"get_param_bounds($module, /, parameter, is_compress)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Internal function, get CompressionParameter/DecompressionParameter bounds.\n"
|
||||
"Get CompressionParameter/DecompressionParameter bounds.\n"
|
||||
"\n"
|
||||
" parameter\n"
|
||||
" The parameter to get bounds.\n"
|
||||
|
|
@ -220,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__,
|
|||
"get_frame_size($module, /, frame_buffer)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n"
|
||||
"Get the size of a Zstandard frame, including the header and optional checksum.\n"
|
||||
"\n"
|
||||
" frame_buffer\n"
|
||||
" A bytes-like object, it should start from the beginning of a frame,\n"
|
||||
" and contains at least one complete frame.\n"
|
||||
"\n"
|
||||
"It will iterate all blocks\' headers within a frame, to accumulate the frame size.");
|
||||
" and contains at least one complete frame.");
|
||||
|
||||
#define _ZSTD_GET_FRAME_SIZE_METHODDEF \
|
||||
{"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__},
|
||||
|
|
@ -291,10 +289,10 @@ PyDoc_STRVAR(_zstd_get_frame_info__doc__,
|
|||
"get_frame_info($module, /, frame_buffer)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Internal function, get zstd frame infomation from a frame header.\n"
|
||||
"Get Zstandard frame infomation from a frame header.\n"
|
||||
"\n"
|
||||
" frame_buffer\n"
|
||||
" A bytes-like object, containing the header of a zstd frame.");
|
||||
" A bytes-like object, containing the header of a Zstandard frame.");
|
||||
|
||||
#define _ZSTD_GET_FRAME_INFO_METHODDEF \
|
||||
{"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__},
|
||||
|
|
@ -359,7 +357,7 @@ PyDoc_STRVAR(_zstd_set_parameter_types__doc__,
|
|||
"set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n"
|
||||
"Set CompressionParameter and DecompressionParameter types for validity check.\n"
|
||||
"\n"
|
||||
" c_parameter_type\n"
|
||||
" CompressionParameter IntEnum type object\n"
|
||||
|
|
@ -428,4 +426,4 @@ _zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t na
|
|||
exit:
|
||||
return return_value;
|
||||
}
|
||||
/*[clinic end generated code: output=8445b658dcdcbb9c input=a9049054013a1b77]*/
|
||||
/*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/
|
||||
|
|
|
|||
26
Modules/_zstd/clinic/compressor.c.h
generated
26
Modules/_zstd/clinic/compressor.c.h
generated
|
|
@ -8,30 +8,30 @@ preserve
|
|||
#endif
|
||||
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
|
||||
|
||||
PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__,
|
||||
PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__,
|
||||
"ZstdCompressor(level=None, options=None, zstd_dict=None)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Create a compressor object for compressing data incrementally.\n"
|
||||
"\n"
|
||||
" level\n"
|
||||
" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n"
|
||||
" The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n"
|
||||
" options\n"
|
||||
" A dict object that contains advanced compression parameters.\n"
|
||||
" zstd_dict\n"
|
||||
" A ZstdDict object, a pre-trained zstd dictionary.\n"
|
||||
" A ZstdDict object, a pre-trained Zstandard dictionary.\n"
|
||||
"\n"
|
||||
"Thread-safe at method level. For one-shot compression, use the compress()\n"
|
||||
"function instead.");
|
||||
|
||||
static int
|
||||
_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
|
||||
static PyObject *
|
||||
_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
|
||||
PyObject *options, PyObject *zstd_dict);
|
||||
|
||||
static int
|
||||
_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
||||
static PyObject *
|
||||
_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
|
||||
{
|
||||
int return_value = -1;
|
||||
PyObject *return_value = NULL;
|
||||
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
|
||||
|
||||
#define NUM_KEYWORDS 3
|
||||
|
|
@ -89,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
|||
}
|
||||
zstd_dict = fastargs[2];
|
||||
skip_optional_pos:
|
||||
return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict);
|
||||
return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict);
|
||||
|
||||
exit:
|
||||
return return_value;
|
||||
|
|
@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__,
|
|||
" Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n"
|
||||
" ZstdCompressor.FLUSH_BLOCK\n"
|
||||
"\n"
|
||||
"Flush any remaining data left in internal buffers. Since zstd data consists\n"
|
||||
"of one or more independent frames, the compressor object can still be used\n"
|
||||
"after this method is called.");
|
||||
"Flush any remaining data left in internal buffers. Since Zstandard data\n"
|
||||
"consists of one or more independent frames, the compressor object can still\n"
|
||||
"be used after this method is called.");
|
||||
|
||||
#define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \
|
||||
{"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__},
|
||||
|
|
@ -252,4 +252,4 @@ skip_optional_pos:
|
|||
exit:
|
||||
return return_value;
|
||||
}
|
||||
/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/
|
||||
/*[clinic end generated code: output=ee2d1dc298de790c input=a9049054013a1b77]*/
|
||||
|
|
|
|||
22
Modules/_zstd/clinic/decompressor.c.h
generated
22
Modules/_zstd/clinic/decompressor.c.h
generated
|
|
@ -10,28 +10,28 @@ preserve
|
|||
#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
|
||||
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
|
||||
|
||||
PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__,
|
||||
PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__,
|
||||
"ZstdDecompressor(zstd_dict=None, options=None)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Create a decompressor object for decompressing data incrementally.\n"
|
||||
"\n"
|
||||
" zstd_dict\n"
|
||||
" A ZstdDict object, a pre-trained zstd dictionary.\n"
|
||||
" A ZstdDict object, a pre-trained Zstandard dictionary.\n"
|
||||
" options\n"
|
||||
" A dict object that contains advanced decompression parameters.\n"
|
||||
"\n"
|
||||
"Thread-safe at method level. For one-shot decompression, use the decompress()\n"
|
||||
"function instead.");
|
||||
|
||||
static int
|
||||
_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self,
|
||||
PyObject *zstd_dict, PyObject *options);
|
||||
static PyObject *
|
||||
_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
|
||||
PyObject *options);
|
||||
|
||||
static int
|
||||
_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
||||
static PyObject *
|
||||
_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
|
||||
{
|
||||
int return_value = -1;
|
||||
PyObject *return_value = NULL;
|
||||
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
|
||||
|
||||
#define NUM_KEYWORDS 2
|
||||
|
|
@ -82,7 +82,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs
|
|||
}
|
||||
options = fastargs[1];
|
||||
skip_optional_pos:
|
||||
return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options);
|
||||
return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options);
|
||||
|
||||
exit:
|
||||
return return_value;
|
||||
|
|
@ -130,7 +130,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__,
|
|||
"Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n"
|
||||
"\n"
|
||||
" data\n"
|
||||
" A bytes-like object, zstd data to be decompressed.\n"
|
||||
" A bytes-like object, Zstandard data to be decompressed.\n"
|
||||
" max_length\n"
|
||||
" Maximum size of returned data. When it is negative, the size of\n"
|
||||
" output buffer is unlimited. When it is nonnegative, returns at\n"
|
||||
|
|
@ -227,4 +227,4 @@ exit:
|
|||
|
||||
return return_value;
|
||||
}
|
||||
/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/
|
||||
/*[clinic end generated code: output=7a4d278f9244e684 input=a9049054013a1b77]*/
|
||||
|
|
|
|||
46
Modules/_zstd/clinic/zstddict.c.h
generated
46
Modules/_zstd/clinic/zstddict.c.h
generated
|
|
@ -9,35 +9,33 @@ preserve
|
|||
#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION()
|
||||
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
|
||||
|
||||
PyDoc_STRVAR(_zstd_ZstdDict___init____doc__,
|
||||
"ZstdDict(dict_content, is_raw=False)\n"
|
||||
PyDoc_STRVAR(_zstd_ZstdDict_new__doc__,
|
||||
"ZstdDict(dict_content, /, *, is_raw=False)\n"
|
||||
"--\n"
|
||||
"\n"
|
||||
"Represents a zstd dictionary, which can be used for compression/decompression.\n"
|
||||
"Represents a Zstandard dictionary.\n"
|
||||
"\n"
|
||||
" dict_content\n"
|
||||
" A bytes-like object, dictionary\'s content.\n"
|
||||
" The content of a Zstandard dictionary as a bytes-like object.\n"
|
||||
" is_raw\n"
|
||||
" This parameter is for advanced user. True means dict_content\n"
|
||||
" argument is a \"raw content\" dictionary, free of any format\n"
|
||||
" restriction. False means dict_content argument is an ordinary\n"
|
||||
" zstd dictionary, was created by zstd functions, follow a\n"
|
||||
" specified format.\n"
|
||||
" If true, perform no checks on *dict_content*, useful for some\n"
|
||||
" advanced cases. Otherwise, check that the content represents\n"
|
||||
" a Zstandard dictionary created by the zstd library or CLI.\n"
|
||||
"\n"
|
||||
"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n"
|
||||
"ZstdDecompressor objects.");
|
||||
"The dictionary can be used for compression or decompression, and can be shared\n"
|
||||
"by multiple ZstdCompressor or ZstdDecompressor objects.");
|
||||
|
||||
static int
|
||||
_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
|
||||
static PyObject *
|
||||
_zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content,
|
||||
int is_raw);
|
||||
|
||||
static int
|
||||
_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
||||
static PyObject *
|
||||
_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
|
||||
{
|
||||
int return_value = -1;
|
||||
PyObject *return_value = NULL;
|
||||
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
|
||||
|
||||
#define NUM_KEYWORDS 2
|
||||
#define NUM_KEYWORDS 1
|
||||
static struct {
|
||||
PyGC_Head _this_is_not_used;
|
||||
PyObject_VAR_HEAD
|
||||
|
|
@ -46,7 +44,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
|||
} _kwtuple = {
|
||||
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
|
||||
.ob_hash = -1,
|
||||
.ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), },
|
||||
.ob_item = { &_Py_ID(is_raw), },
|
||||
};
|
||||
#undef NUM_KEYWORDS
|
||||
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
|
||||
|
|
@ -55,7 +53,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
|||
# define KWTUPLE NULL
|
||||
#endif // !Py_BUILD_CORE
|
||||
|
||||
static const char * const _keywords[] = {"dict_content", "is_raw", NULL};
|
||||
static const char * const _keywords[] = {"", "is_raw", NULL};
|
||||
static _PyArg_Parser _parser = {
|
||||
.keywords = _keywords,
|
||||
.fname = "ZstdDict",
|
||||
|
|
@ -70,20 +68,20 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs)
|
|||
int is_raw = 0;
|
||||
|
||||
fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser,
|
||||
/*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
|
||||
/*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf);
|
||||
if (!fastargs) {
|
||||
goto exit;
|
||||
}
|
||||
dict_content = fastargs[0];
|
||||
if (!noptargs) {
|
||||
goto skip_optional_pos;
|
||||
goto skip_optional_kwonly;
|
||||
}
|
||||
is_raw = PyObject_IsTrue(fastargs[1]);
|
||||
if (is_raw < 0) {
|
||||
goto exit;
|
||||
}
|
||||
skip_optional_pos:
|
||||
return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw);
|
||||
skip_optional_kwonly:
|
||||
return_value = _zstd_ZstdDict_new_impl(type, dict_content, is_raw);
|
||||
|
||||
exit:
|
||||
return return_value;
|
||||
|
|
@ -204,4 +202,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context))
|
|||
|
||||
return return_value;
|
||||
}
|
||||
/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/
|
||||
/*[clinic end generated code: output=bfb31c1187477afd input=a9049054013a1b77]*/
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
/* ZstdCompressor class definitions */
|
||||
|
||||
|
|
@ -21,7 +18,6 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
|
|||
#include "buffer.h"
|
||||
#include "zstddict.h"
|
||||
|
||||
#include <stdbool.h> // bool
|
||||
#include <stddef.h> // offsetof()
|
||||
#include <zstd.h> // ZSTD_*()
|
||||
|
||||
|
|
@ -42,9 +38,6 @@ typedef struct {
|
|||
|
||||
/* Compression level */
|
||||
int compression_level;
|
||||
|
||||
/* __init__ has been called, 0 or 1. */
|
||||
bool initialized;
|
||||
} ZstdCompressor;
|
||||
|
||||
#define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
|
||||
|
|
@ -122,7 +115,7 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
|
|||
self->compression_level = value_v;
|
||||
}
|
||||
else if (key_v == ZSTD_c_nbWorkers) {
|
||||
/* From zstd library doc:
|
||||
/* From the zstd library docs:
|
||||
1. When nbWorkers >= 1, triggers asynchronous mode when
|
||||
used with ZSTD_compressStream2().
|
||||
2, Default value is `0`, aka "single-threaded mode" : no
|
||||
|
|
@ -189,8 +182,8 @@ _get_CDict(ZstdDict *self, int compressionLevel)
|
|||
_zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
|
||||
if (mod_state != NULL) {
|
||||
PyErr_SetString(mod_state->ZstdError,
|
||||
"Failed to create ZSTD_CDict instance from zstd "
|
||||
"dictionary content. Maybe the content is corrupted.");
|
||||
"Failed to create a ZSTD_CDict instance from "
|
||||
"Zstandard dictionary content.");
|
||||
}
|
||||
goto error;
|
||||
}
|
||||
|
|
@ -318,20 +311,34 @@ load:
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*[clinic input]
|
||||
@classmethod
|
||||
_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new
|
||||
level: object = None
|
||||
The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.
|
||||
options: object = None
|
||||
A dict object that contains advanced compression parameters.
|
||||
zstd_dict: object = None
|
||||
A ZstdDict object, a pre-trained Zstandard dictionary.
|
||||
|
||||
Create a compressor object for compressing data incrementally.
|
||||
|
||||
Thread-safe at method level. For one-shot compression, use the compress()
|
||||
function instead.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
|
||||
_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level,
|
||||
PyObject *options, PyObject *zstd_dict)
|
||||
/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/
|
||||
{
|
||||
ZstdCompressor *self;
|
||||
self = PyObject_GC_New(ZstdCompressor, type);
|
||||
ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type);
|
||||
if (self == NULL) {
|
||||
goto error;
|
||||
}
|
||||
|
||||
self->initialized = 0;
|
||||
self->dict = NULL;
|
||||
self->use_multithread = 0;
|
||||
|
||||
|
||||
/* Compression context */
|
||||
self->cctx = ZSTD_createCCtx();
|
||||
if (self->cctx == NULL) {
|
||||
|
|
@ -346,6 +353,37 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject
|
|||
/* Last mode */
|
||||
self->last_mode = ZSTD_e_end;
|
||||
|
||||
if (level != Py_None && options != Py_None) {
|
||||
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
|
||||
goto error;
|
||||
}
|
||||
|
||||
/* Set compressLevel/options to compression context */
|
||||
if (level != Py_None) {
|
||||
if (_zstd_set_c_parameters(self, level, "level", "int") < 0) {
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
|
||||
if (options != Py_None) {
|
||||
if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) {
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
|
||||
/* Load Zstandard dictionary to compression context */
|
||||
self->dict = NULL;
|
||||
if (zstd_dict != Py_None) {
|
||||
if (_zstd_load_c_dict(self, zstd_dict) < 0) {
|
||||
goto error;
|
||||
}
|
||||
Py_INCREF(zstd_dict);
|
||||
self->dict = zstd_dict;
|
||||
}
|
||||
|
||||
// We can only start GC tracking once self->dict is set.
|
||||
PyObject_GC_Track(self);
|
||||
|
||||
return (PyObject*)self;
|
||||
|
||||
error:
|
||||
|
|
@ -373,67 +411,6 @@ ZstdCompressor_dealloc(PyObject *ob)
|
|||
Py_DECREF(tp);
|
||||
}
|
||||
|
||||
/*[clinic input]
|
||||
_zstd.ZstdCompressor.__init__
|
||||
|
||||
level: object = None
|
||||
The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.
|
||||
options: object = None
|
||||
A dict object that contains advanced compression parameters.
|
||||
zstd_dict: object = None
|
||||
A ZstdDict object, a pre-trained zstd dictionary.
|
||||
|
||||
Create a compressor object for compressing data incrementally.
|
||||
|
||||
Thread-safe at method level. For one-shot compression, use the compress()
|
||||
function instead.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static int
|
||||
_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
|
||||
PyObject *options, PyObject *zstd_dict)
|
||||
/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/
|
||||
{
|
||||
if (self->initialized) {
|
||||
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
|
||||
return -1;
|
||||
}
|
||||
self->initialized = 1;
|
||||
|
||||
if (level != Py_None && options != Py_None) {
|
||||
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Set compressLevel/options to compression context */
|
||||
if (level != Py_None) {
|
||||
if (_zstd_set_c_parameters(self, level, "level", "int") < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (options != Py_None) {
|
||||
if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Load dictionary to compression context */
|
||||
if (zstd_dict != Py_None) {
|
||||
if (_zstd_load_c_dict(self, zstd_dict) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Py_INCREF the dict */
|
||||
Py_INCREF(zstd_dict);
|
||||
self->dict = zstd_dict;
|
||||
}
|
||||
|
||||
// We can only start tracking self with the GC once self->dict is set.
|
||||
PyObject_GC_Track(self);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
compress_impl(ZstdCompressor *self, Py_buffer *data,
|
||||
ZSTD_EndDirective end_directive)
|
||||
|
|
@ -469,7 +446,7 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
|
|||
}
|
||||
|
||||
|
||||
/* zstd stream compress */
|
||||
/* Zstandard stream compress */
|
||||
while (1) {
|
||||
Py_BEGIN_ALLOW_THREADS
|
||||
zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive);
|
||||
|
|
@ -533,7 +510,7 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
|
|||
goto error;
|
||||
}
|
||||
|
||||
/* zstd stream compress */
|
||||
/* Zstandard stream compress */
|
||||
while (1) {
|
||||
Py_BEGIN_ALLOW_THREADS
|
||||
do {
|
||||
|
|
@ -642,14 +619,14 @@ _zstd.ZstdCompressor.flush
|
|||
|
||||
Finish the compression process.
|
||||
|
||||
Flush any remaining data left in internal buffers. Since zstd data consists
|
||||
of one or more independent frames, the compressor object can still be used
|
||||
after this method is called.
|
||||
Flush any remaining data left in internal buffers. Since Zstandard data
|
||||
consists of one or more independent frames, the compressor object can still
|
||||
be used after this method is called.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode)
|
||||
/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/
|
||||
/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=0ab19627f323cdbc]*/
|
||||
{
|
||||
PyObject *ret;
|
||||
|
||||
|
|
@ -717,10 +694,9 @@ ZstdCompressor_clear(PyObject *ob)
|
|||
static PyType_Slot zstdcompressor_slots[] = {
|
||||
{Py_tp_new, _zstd_ZstdCompressor_new},
|
||||
{Py_tp_dealloc, ZstdCompressor_dealloc},
|
||||
{Py_tp_init, _zstd_ZstdCompressor___init__},
|
||||
{Py_tp_methods, ZstdCompressor_methods},
|
||||
{Py_tp_members, ZstdCompressor_members},
|
||||
{Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__},
|
||||
{Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__},
|
||||
{Py_tp_traverse, ZstdCompressor_traverse},
|
||||
{Py_tp_clear, ZstdCompressor_clear},
|
||||
{0, 0}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
/* ZstdDecompressor class definition */
|
||||
|
||||
|
|
@ -48,9 +45,6 @@ typedef struct {
|
|||
/* For ZstdDecompressor, 0 or 1.
|
||||
1 means the end of the first frame has been reached. */
|
||||
bool eof;
|
||||
|
||||
/* __init__ has been called, 0 or 1. */
|
||||
bool initialized;
|
||||
} ZstdDecompressor;
|
||||
|
||||
#define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
|
||||
|
|
@ -81,8 +75,8 @@ _get_DDict(ZstdDict *self)
|
|||
_zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self));
|
||||
if (mod_state != NULL) {
|
||||
PyErr_SetString(mod_state->ZstdError,
|
||||
"Failed to create ZSTD_DDict instance from zstd "
|
||||
"dictionary content. Maybe the content is corrupted.");
|
||||
"Failed to create a ZSTD_DDict instance from "
|
||||
"Zstandard dictionary content.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -265,8 +259,8 @@ load:
|
|||
finish
|
||||
|
||||
ZSTD_decompressStream()'s size_t return value:
|
||||
- 0 when a frame is completely decoded and fully flushed, zstd's internal
|
||||
buffer has no data.
|
||||
- 0 when a frame is completely decoded and fully flushed,
|
||||
zstd's internal buffer has no data.
|
||||
- An error code, which can be tested using ZSTD_isError().
|
||||
- Or any other value > 0, which means there is still some decoding or
|
||||
flushing to do to complete current frame.
|
||||
|
|
@ -311,7 +305,7 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in,
|
|||
}
|
||||
|
||||
/* Need to check out before in. Maybe zstd's internal buffer still has
|
||||
a few bytes can be output, grow the buffer and continue. */
|
||||
a few bytes that can be output, grow the buffer and continue. */
|
||||
if (out.pos == out.size) {
|
||||
/* Output buffer exhausted */
|
||||
|
||||
|
|
@ -373,7 +367,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length
|
|||
|
||||
/* Check .eof flag */
|
||||
if (self->eof) {
|
||||
PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame.");
|
||||
PyErr_SetString(PyExc_EOFError, "Already at the end of a Zstandard frame.");
|
||||
assert(ret == NULL);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -530,17 +524,30 @@ error:
|
|||
}
|
||||
|
||||
|
||||
/*[clinic input]
|
||||
@classmethod
|
||||
_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new
|
||||
zstd_dict: object = None
|
||||
A ZstdDict object, a pre-trained Zstandard dictionary.
|
||||
options: object = None
|
||||
A dict object that contains advanced decompression parameters.
|
||||
|
||||
Create a decompressor object for decompressing data incrementally.
|
||||
|
||||
Thread-safe at method level. For one-shot decompression, use the decompress()
|
||||
function instead.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
|
||||
_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
|
||||
PyObject *options)
|
||||
/*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/
|
||||
{
|
||||
ZstdDecompressor *self;
|
||||
self = PyObject_GC_New(ZstdDecompressor, type);
|
||||
ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type);
|
||||
if (self == NULL) {
|
||||
goto error;
|
||||
}
|
||||
|
||||
self->initialized = 0;
|
||||
self->dict = NULL;
|
||||
self->input_buffer = NULL;
|
||||
self->input_buffer_size = 0;
|
||||
self->in_begin = -1;
|
||||
|
|
@ -562,6 +569,26 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
|
|||
goto error;
|
||||
}
|
||||
|
||||
/* Load Zstandard dictionary to decompression context */
|
||||
self->dict = NULL;
|
||||
if (zstd_dict != Py_None) {
|
||||
if (_zstd_load_d_dict(self, zstd_dict) < 0) {
|
||||
goto error;
|
||||
}
|
||||
Py_INCREF(zstd_dict);
|
||||
self->dict = zstd_dict;
|
||||
}
|
||||
|
||||
/* Set option to decompression context */
|
||||
if (options != Py_None) {
|
||||
if (_zstd_set_d_parameters(self, options) < 0) {
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
|
||||
// We can only start GC tracking once self->dict is set.
|
||||
PyObject_GC_Track(self);
|
||||
|
||||
return (PyObject*)self;
|
||||
|
||||
error:
|
||||
|
|
@ -595,55 +622,6 @@ ZstdDecompressor_dealloc(PyObject *ob)
|
|||
Py_DECREF(tp);
|
||||
}
|
||||
|
||||
/*[clinic input]
|
||||
_zstd.ZstdDecompressor.__init__
|
||||
|
||||
zstd_dict: object = None
|
||||
A ZstdDict object, a pre-trained zstd dictionary.
|
||||
options: object = None
|
||||
A dict object that contains advanced decompression parameters.
|
||||
|
||||
Create a decompressor object for decompressing data incrementally.
|
||||
|
||||
Thread-safe at method level. For one-shot decompression, use the decompress()
|
||||
function instead.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static int
|
||||
_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self,
|
||||
PyObject *zstd_dict, PyObject *options)
|
||||
/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/
|
||||
{
|
||||
/* Only called once */
|
||||
if (self->initialized) {
|
||||
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
|
||||
return -1;
|
||||
}
|
||||
self->initialized = 1;
|
||||
|
||||
/* Load dictionary to decompression context */
|
||||
if (zstd_dict != Py_None) {
|
||||
if (_zstd_load_d_dict(self, zstd_dict) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Py_INCREF the dict */
|
||||
Py_INCREF(zstd_dict);
|
||||
self->dict = zstd_dict;
|
||||
}
|
||||
|
||||
/* Set option to decompression context */
|
||||
if (options != Py_None) {
|
||||
if (_zstd_set_d_parameters(self, options) < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
// We can only start tracking self with the GC once self->dict is set.
|
||||
PyObject_GC_Track(self);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*[clinic input]
|
||||
@critical_section
|
||||
@getter
|
||||
|
|
@ -685,7 +663,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
|
|||
_zstd.ZstdDecompressor.decompress
|
||||
|
||||
data: Py_buffer
|
||||
A bytes-like object, zstd data to be decompressed.
|
||||
A bytes-like object, Zstandard data to be decompressed.
|
||||
max_length: Py_ssize_t = -1
|
||||
Maximum size of returned data. When it is negative, the size of
|
||||
output buffer is unlimited. When it is nonnegative, returns at
|
||||
|
|
@ -711,7 +689,7 @@ static PyObject *
|
|||
_zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
|
||||
Py_buffer *data,
|
||||
Py_ssize_t max_length)
|
||||
/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/
|
||||
/*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/
|
||||
{
|
||||
PyObject *ret;
|
||||
/* Thread-safe code */
|
||||
|
|
@ -769,11 +747,10 @@ ZstdDecompressor_clear(PyObject *ob)
|
|||
static PyType_Slot ZstdDecompressor_slots[] = {
|
||||
{Py_tp_new, _zstd_ZstdDecompressor_new},
|
||||
{Py_tp_dealloc, ZstdDecompressor_dealloc},
|
||||
{Py_tp_init, _zstd_ZstdDecompressor___init__},
|
||||
{Py_tp_methods, ZstdDecompressor_methods},
|
||||
{Py_tp_members, ZstdDecompressor_members},
|
||||
{Py_tp_getset, ZstdDecompressor_getset},
|
||||
{Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__},
|
||||
{Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__},
|
||||
{Py_tp_traverse, ZstdDecompressor_traverse},
|
||||
{Py_tp_clear, ZstdDecompressor_clear},
|
||||
{0, 0}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,4 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
/* ZstdDict class definitions */
|
||||
|
||||
|
|
@ -25,17 +22,35 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
|
|||
|
||||
#define ZstdDict_CAST(op) ((ZstdDict *)op)
|
||||
|
||||
/*[clinic input]
|
||||
@classmethod
|
||||
_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new
|
||||
dict_content: object
|
||||
The content of a Zstandard dictionary as a bytes-like object.
|
||||
/
|
||||
*
|
||||
is_raw: bool = False
|
||||
If true, perform no checks on *dict_content*, useful for some
|
||||
advanced cases. Otherwise, check that the content represents
|
||||
a Zstandard dictionary created by the zstd library or CLI.
|
||||
|
||||
Represents a Zstandard dictionary.
|
||||
|
||||
The dictionary can be used for compression or decompression, and can be shared
|
||||
by multiple ZstdCompressor or ZstdDecompressor objects.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static PyObject *
|
||||
_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
|
||||
_zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content,
|
||||
int is_raw)
|
||||
/*[clinic end generated code: output=3ebff839cb3be6d7 input=6b5de413869ae878]*/
|
||||
{
|
||||
ZstdDict *self;
|
||||
self = PyObject_GC_New(ZstdDict, type);
|
||||
ZstdDict* self = PyObject_GC_New(ZstdDict, type);
|
||||
if (self == NULL) {
|
||||
goto error;
|
||||
}
|
||||
|
||||
self->dict_content = NULL;
|
||||
self->initialized = 0;
|
||||
self->d_dict = NULL;
|
||||
|
||||
/* ZSTD_CDict dict */
|
||||
|
|
@ -44,6 +59,36 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U
|
|||
goto error;
|
||||
}
|
||||
|
||||
/* Check dict_content's type */
|
||||
self->dict_content = PyBytes_FromObject(dict_content);
|
||||
if (self->dict_content == NULL) {
|
||||
PyErr_SetString(PyExc_TypeError,
|
||||
"dict_content argument should be bytes-like object.");
|
||||
goto error;
|
||||
}
|
||||
|
||||
/* Both ordinary dictionary and "raw content" dictionary should
|
||||
at least 8 bytes */
|
||||
if (Py_SIZE(self->dict_content) < 8) {
|
||||
PyErr_SetString(PyExc_ValueError,
|
||||
"Zstandard dictionary content should at least 8 bytes.");
|
||||
goto error;
|
||||
}
|
||||
|
||||
/* Get dict_id, 0 means "raw content" dictionary. */
|
||||
self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content),
|
||||
Py_SIZE(self->dict_content));
|
||||
|
||||
/* Check validity for ordinary dictionary */
|
||||
if (!is_raw && self->dict_id == 0) {
|
||||
char *msg = "Invalid Zstandard dictionary and is_raw not set.\n";
|
||||
PyErr_SetString(PyExc_ValueError, msg);
|
||||
goto error;
|
||||
}
|
||||
|
||||
// Can only track self once self->dict_content is included
|
||||
PyObject_GC_Track(self);
|
||||
|
||||
return (PyObject*)self;
|
||||
|
||||
error:
|
||||
|
|
@ -72,83 +117,15 @@ ZstdDict_dealloc(PyObject *ob)
|
|||
Py_DECREF(tp);
|
||||
}
|
||||
|
||||
/*[clinic input]
|
||||
_zstd.ZstdDict.__init__
|
||||
|
||||
dict_content: object
|
||||
A bytes-like object, dictionary's content.
|
||||
is_raw: bool = False
|
||||
This parameter is for advanced user. True means dict_content
|
||||
argument is a "raw content" dictionary, free of any format
|
||||
restriction. False means dict_content argument is an ordinary
|
||||
zstd dictionary, was created by zstd functions, follow a
|
||||
specified format.
|
||||
|
||||
Represents a zstd dictionary, which can be used for compression/decompression.
|
||||
|
||||
It's thread-safe, and can be shared by multiple ZstdCompressor /
|
||||
ZstdDecompressor objects.
|
||||
[clinic start generated code]*/
|
||||
|
||||
static int
|
||||
_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
|
||||
int is_raw)
|
||||
/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/
|
||||
{
|
||||
/* Only called once */
|
||||
if (self->initialized) {
|
||||
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
|
||||
return -1;
|
||||
}
|
||||
self->initialized = 1;
|
||||
|
||||
/* Check dict_content's type */
|
||||
self->dict_content = PyBytes_FromObject(dict_content);
|
||||
if (self->dict_content == NULL) {
|
||||
PyErr_SetString(PyExc_TypeError,
|
||||
"dict_content argument should be bytes-like object.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Both ordinary dictionary and "raw content" dictionary should
|
||||
at least 8 bytes */
|
||||
if (Py_SIZE(self->dict_content) < 8) {
|
||||
PyErr_SetString(PyExc_ValueError,
|
||||
"Zstd dictionary content should at least 8 bytes.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Get dict_id, 0 means "raw content" dictionary. */
|
||||
self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content),
|
||||
Py_SIZE(self->dict_content));
|
||||
|
||||
/* Check validity for ordinary dictionary */
|
||||
if (!is_raw && self->dict_id == 0) {
|
||||
char *msg = "The dict_content argument is not a valid zstd "
|
||||
"dictionary. The first 4 bytes of a valid zstd dictionary "
|
||||
"should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n"
|
||||
"If you are an advanced user, and can be sure that "
|
||||
"dict_content argument is a \"raw content\" zstd "
|
||||
"dictionary, set is_raw parameter to True.";
|
||||
PyErr_SetString(PyExc_ValueError, msg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Can only track self once self->dict_content is included
|
||||
PyObject_GC_Track(self);
|
||||
return 0;
|
||||
}
|
||||
|
||||
PyDoc_STRVAR(ZstdDict_dictid_doc,
|
||||
"ID of zstd dictionary, a 32-bit unsigned int value.\n\n"
|
||||
"Non-zero means ordinary dictionary, was created by zstd functions, follow\n"
|
||||
"a specified format.\n\n"
|
||||
"0 means a \"raw content\" dictionary, free of any format restriction, used\n"
|
||||
"for advanced user.");
|
||||
"the Zstandard dictionary, an int between 0 and 2**32.\n\n"
|
||||
"A non-zero value represents an ordinary Zstandard dictionary, "
|
||||
"conforming to the standardised format.\n\n"
|
||||
"The special value '0' means a 'raw content' dictionary,"
|
||||
"without any restrictions on format or content.");
|
||||
|
||||
PyDoc_STRVAR(ZstdDict_dictcontent_doc,
|
||||
"The content of zstd dictionary, a bytes object, it's the same as dict_content\n"
|
||||
"argument in ZstdDict.__init__() method. It can be used with other programs.");
|
||||
"The content of a Zstandard dictionary, as a bytes object.");
|
||||
|
||||
static PyObject *
|
||||
ZstdDict_str(PyObject *ob)
|
||||
|
|
@ -266,9 +243,8 @@ static PyType_Slot zstddict_slots[] = {
|
|||
{Py_tp_getset, ZstdDict_getset},
|
||||
{Py_tp_new, _zstd_ZstdDict_new},
|
||||
{Py_tp_dealloc, ZstdDict_dealloc},
|
||||
{Py_tp_init, _zstd_ZstdDict___init__},
|
||||
{Py_tp_str, ZstdDict_str},
|
||||
{Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__},
|
||||
{Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__},
|
||||
{Py_sq_length, ZstdDict_length},
|
||||
{Py_tp_traverse, ZstdDict_traverse},
|
||||
{Py_tp_clear, ZstdDict_clear},
|
||||
|
|
|
|||
|
|
@ -1,12 +1,8 @@
|
|||
/*
|
||||
Low level interface to Meta's zstd library for use in the compression.zstd
|
||||
Python module.
|
||||
*/
|
||||
/* Low level interface to the Zstandard algorthm & the zstd library. */
|
||||
|
||||
#ifndef ZSTD_DICT_H
|
||||
#define ZSTD_DICT_H
|
||||
|
||||
#include <stdbool.h> // bool
|
||||
#include <zstd.h> // ZSTD_DDict
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -23,9 +19,6 @@ typedef struct {
|
|||
PyObject *dict_content;
|
||||
/* Dictionary id */
|
||||
uint32_t dict_id;
|
||||
|
||||
/* __init__ has been called, 0 or 1. */
|
||||
bool initialized;
|
||||
} ZstdDict;
|
||||
|
||||
#endif // !ZSTD_DICT_H
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue