gh-132983: Reduce the size of `_zstdmodule.h` (#133793)

Makefile.pre.in

@@ -3346,7 +3346,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes
 MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
 MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
 MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
-MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h
+MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
 
 CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h
 MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS)

Modules/_zstd/_zstdmodule.c

@@ -7,7 +7,13 @@ Python module.
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include "Python.h"
+
 #include "_zstdmodule.h"
+#include "zstddict.h"
+
+#include <zstd.h>                 // ZSTD_*()
+#include <zdict.h>                // ZDICT_*()
 
 /*[clinic input]
 module _zstd
@@ -727,7 +733,7 @@ static struct PyModuleDef_Slot _zstd_slots[] = {
     {0, NULL},
 };
 
-struct PyModuleDef _zstdmodule = {
+static struct PyModuleDef _zstdmodule = {
     .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "_zstd",
     .m_doc = "Implementation module for Zstandard compression.",

Modules/_zstd/_zstdmodule.h

@@ -1,4 +1,3 @@
-#pragma once
 /*
 Low level interface to Meta's zstd library for use in the compression.zstd
 Python module.
@@ -6,127 +5,25 @@ Python module.
 
 /* Declarations shared between different parts of the _zstd module*/
 
-#include "Python.h"
+#ifndef ZSTD_MODULE_H
-
+#define ZSTD_MODULE_H
-#include "zstd.h"
-#include "zdict.h"
-
-
-/* Forward declaration of module state */
-typedef struct _zstd_state _zstd_state;
-
-/* Forward reference of module def */
-extern PyModuleDef _zstdmodule;
-
-/* For clinic type calculations */
-static inline _zstd_state *
-get_zstd_state_from_type(PyTypeObject *type)
-{
-    PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
-    if (module == NULL) {
-        return NULL;
-    }
-    void *state = PyModule_GetState(module);
-    assert(state != NULL);
-    return (_zstd_state *)state;
-}
 
+/* Type specs */
 extern PyType_Spec zstd_dict_type_spec;
 extern PyType_Spec zstd_compressor_type_spec;
 extern PyType_Spec zstd_decompressor_type_spec;
 
-struct _zstd_state {
+typedef struct {
+    /* Module heap types. */
     PyTypeObject *ZstdDict_type;
     PyTypeObject *ZstdCompressor_type;
     PyTypeObject *ZstdDecompressor_type;
     PyObject *ZstdError;
 
+    /* enum types set by set_parameter_types. */
     PyTypeObject *CParameter_type;
     PyTypeObject *DParameter_type;
-};
+} _zstd_state;
-
-typedef struct {
-    PyObject_HEAD
-
-    /* Reusable compress/decompress dictionary, they are created once and
-       can be shared by multiple threads concurrently, since its usage is
-       read-only.
-       c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */
-    ZSTD_DDict *d_dict;
-    PyObject *c_dicts;
-
-    /* Content of the dictionary, bytes object. */
-    PyObject *dict_content;
-    /* Dictionary id */
-    uint32_t dict_id;
-
-    /* __init__ has been called, 0 or 1. */
-    int inited;
-} ZstdDict;
-
-typedef struct {
-    PyObject_HEAD
-
-    /* Compression context */
-    ZSTD_CCtx *cctx;
-
-    /* ZstdDict object in use */
-    PyObject *dict;
-
-    /* Last mode, initialized to ZSTD_e_end */
-    int last_mode;
-
-    /* (nbWorker >= 1) ? 1 : 0 */
-    int use_multithread;
-
-    /* Compression level */
-    int compression_level;
-
-    /* __init__ has been called, 0 or 1. */
-    int inited;
-} ZstdCompressor;
-
-typedef struct {
-    PyObject_HEAD
-
-    /* Decompression context */
-    ZSTD_DCtx *dctx;
-
-    /* ZstdDict object in use */
-    PyObject *dict;
-
-    /* Unconsumed input data */
-    char *input_buffer;
-    size_t input_buffer_size;
-    size_t in_begin, in_end;
-
-    /* Unused data */
-    PyObject *unused_data;
-
-    /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
-    char needs_input;
-
-    /* For decompress(), 0 or 1.
-       1 when both input and output streams are at a frame edge, means a
-       frame is completely decoded and fully flushed, or the decompressor
-       just be initialized. */
-    char at_frame_edge;
-
-    /* For ZstdDecompressor, 0 or 1.
-       1 means the end of the first frame has been reached. */
-    char eof;
-
-    /* Used for fast reset above three variables */
-    char _unused_char_for_align;
-
-    /* __init__ has been called, 0 or 1. */
-    int inited;
-} ZstdDecompressor;
-
-typedef enum {
-    TYPE_DECOMPRESSOR,          // <D>, ZstdDecompressor class
-    TYPE_ENDLESS_DECOMPRESSOR,  // <E>, decompress() function
-} decompress_type;
 
 typedef enum {
     ERR_DECOMPRESS,
@@ -149,12 +46,6 @@ typedef enum {
     DICT_TYPE_PREFIX = 2
 } dictionary_type;
 
-static inline int
-mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
-{
-    return in->size == in->pos && out->size != out->pos;
-}
-
 /* Format error message and set ZstdError. */
 extern void
 set_zstd_error(const _zstd_state* const state,
@@ -164,4 +55,4 @@ extern void
 set_parameter_error(const _zstd_state* const state, int is_compress,
                     int key_v, int value_v);
 
-static const char init_twice_msg[] = "__init__ method is called twice.";
+#endif  // !ZSTD_MODULE_H

Modules/_zstd/buffer.h

@@ -3,9 +3,13 @@ Low level interface to Meta's zstd library for use in the compression.zstd
 Python module.
 */
 
-#include "_zstdmodule.h"
+#ifndef ZSTD_BUFFER_H
+#define ZSTD_BUFFER_H
+
 #include "pycore_blocks_output_buffer.h"
 
+#include <zstd.h>                 // ZSTD_outBuffer
+
 /* Blocks output buffer wrapper code */
 
 /* Initialize the buffer, and grow the buffer.
@@ -102,3 +106,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob)
 
     return buffer->allocated == buffer->max_length;
 }
+
+#endif  // !ZSTD_BUFFER_H

Modules/_zstd/compressor.c

@@ -7,24 +7,50 @@ Python module.
 
 /*[clinic input]
 module _zstd
-class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type"
+class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
 [clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/
-
 
 #ifndef Py_BUILD_CORE_BUILTIN
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include "Python.h"
+
 #include "_zstdmodule.h"
-
 #include "buffer.h"
+#include "zstddict.h"
 
+#include <stdbool.h>              // bool
 #include <stddef.h>               // offsetof()
+#include <zstd.h>                 // ZSTD_*()
 
+typedef struct {
+    PyObject_HEAD
+
+    /* Compression context */
+    ZSTD_CCtx *cctx;
+
+    /* ZstdDict object in use */
+    PyObject *dict;
+
+    /* Last mode, initialized to ZSTD_e_end */
+    int last_mode;
+
+    /* (nbWorker >= 1) ? 1 : 0 */
+    int use_multithread;
+
+    /* Compression level */
+    int compression_level;
+
+    /* __init__ has been called, 0 or 1. */
+    bool initialized;
+} ZstdCompressor;
 
 #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
 
+#include "clinic/compressor.c.h"
+
 static int
 _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
                        const char *arg_name, const char* arg_type)
@@ -292,10 +318,6 @@ load:
     return 0;
 }
 
-#define clinic_state() (get_zstd_state_from_type(type))
-#include "clinic/compressor.c.h"
-#undef clinic_state
-
 static PyObject *
 _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
 {
@@ -305,7 +327,7 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject
         goto error;
     }
 
-    self->inited = 0;
+    self->initialized = 0;
     self->dict = NULL;
     self->use_multithread = 0;
 
@@ -372,12 +394,11 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
                                    PyObject *options, PyObject *zstd_dict)
 /*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/
 {
-    /* Only called once */
+    if (self->initialized) {
-    if (self->inited) {
+        PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
-        PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
         return -1;
     }
-    self->inited = 1;
+    self->initialized = 1;
 
     if (level != Py_None && options != Py_None) {
         PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
@@ -488,6 +509,12 @@ error:
     return NULL;
 }
 
+static inline int
+mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
+{
+    return in->size == in->pos && out->size != out->pos;
+}
+
 static PyObject *
 compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
 {

Modules/_zstd/decompressor.c

@@ -7,22 +7,65 @@ Python module.
 
 /*[clinic input]
 module _zstd
-class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type"
+class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec"
 [clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/
 
 #ifndef Py_BUILD_CORE_BUILTIN
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include "Python.h"
+
 #include "_zstdmodule.h"
-
 #include "buffer.h"
+#include "zstddict.h"
 
+#include <stdbool.h>              // bool
 #include <stddef.h>               // offsetof()
+#include <zstd.h>                 // ZSTD_*()
+
+typedef struct {
+    PyObject_HEAD
+
+    /* Decompression context */
+    ZSTD_DCtx *dctx;
+
+    /* ZstdDict object in use */
+    PyObject *dict;
+
+    /* Unconsumed input data */
+    char *input_buffer;
+    size_t input_buffer_size;
+    size_t in_begin, in_end;
+
+    /* Unused data */
+    PyObject *unused_data;
+
+    /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
+    char needs_input;
+
+    /* For decompress(), 0 or 1.
+       1 when both input and output streams are at a frame edge, means a
+       frame is completely decoded and fully flushed, or the decompressor
+       just be initialized. */
+    char at_frame_edge;
+
+    /* For ZstdDecompressor, 0 or 1.
+       1 means the end of the first frame has been reached. */
+    char eof;
+
+    /* Used for fast reset above three variables */
+    char _unused_char_for_align;
+
+    /* __init__ has been called, 0 or 1. */
+    bool initialized;
+} ZstdDecompressor;
 
 #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
 
+#include "clinic/decompressor.c.h"
+
 static inline ZSTD_DDict *
 _get_DDict(ZstdDict *self)
 {
@@ -215,17 +258,14 @@ load:
     return 0;
 }
 
-
+typedef enum {
+    TYPE_DECOMPRESSOR,          // <D>, ZstdDecompressor class
+    TYPE_ENDLESS_DECOMPRESSOR,  // <E>, decompress() function
+} decompress_type;
 
 /*
-    Given the two types of decompressors (defined in _zstdmodule.h):
+    Given the two types of decompressors (defined above),
-
+    decompress implementation for <D>, <E>, pseudo code:
-        typedef enum {
-            TYPE_DECOMPRESSOR,          // <D>, ZstdDecompressor class
-            TYPE_ENDLESS_DECOMPRESSOR,  // <E>, decompress() function
-        } decompress_type;
-
-    Decompress implementation for <D>, <E>, pseudo code:
 
         initialize_output_buffer
         while True:
@@ -616,7 +656,7 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         goto error;
     }
 
-    self->inited = 0;
+    self->initialized = 0;
     self->dict = NULL;
     self->input_buffer = NULL;
     self->input_buffer_size = 0;
@@ -695,11 +735,11 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self,
 /*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/
 {
     /* Only called once */
-    if (self->inited) {
+    if (self->initialized) {
-        PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
+        PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
         return -1;
     }
-    self->inited = 1;
+    self->initialized = 1;
 
     /* Load dictionary to decompression context */
     if (zstd_dict != Py_None) {
@@ -802,10 +842,6 @@ _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
     return ret;
 }
 
-#define clinic_state() (get_zstd_state_from_type(type))
-#include "clinic/decompressor.c.h"
-#undef clinic_state
-
 static PyMethodDef ZstdDecompressor_methods[] = {
     _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF
     {NULL, NULL}

Modules/_zstd/zstddict.c

@@ -7,17 +7,21 @@ Python module.
 
 /*[clinic input]
 module _zstd
-class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type"
+class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
 [clinic start generated code]*/
-/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/
 
 #ifndef Py_BUILD_CORE_BUILTIN
 #  define Py_BUILD_CORE_MODULE 1
 #endif
 
-#include "_zstdmodule.h"
+#include "Python.h"
 
-#include <stddef.h>               // offsetof()
+#include "_zstdmodule.h"
+#include "zstddict.h"
+#include "clinic/zstddict.c.h"
+
+#include <zstd.h>                 // ZSTD_freeDDict(), ZSTD_getDictID_fromDict()
 
 #define ZstdDict_CAST(op) ((ZstdDict *)op)
 
@@ -31,7 +35,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U
     }
 
     self->dict_content = NULL;
-    self->inited = 0;
+    self->initialized = 0;
     self->d_dict = NULL;
 
     /* ZSTD_CDict dict */
@@ -92,11 +96,11 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
 /*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/
 {
     /* Only called once */
-    if (self->inited) {
+    if (self->initialized) {
-        PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
+        PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
         return -1;
     }
-    self->inited = 1;
+    self->initialized = 1;
 
     /* Check dict_content's type */
     self->dict_content = PyBytes_FromObject(dict_content);
@@ -135,10 +139,6 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content,
     return 0;
 }
 
-#define clinic_state() (get_zstd_state(type))
-#include "clinic/zstddict.c.h"
-#undef clinic_state
-
 PyDoc_STRVAR(ZstdDict_dictid_doc,
 "ID of zstd dictionary, a 32-bit unsigned int value.\n\n"
 "Non-zero means ordinary dictionary, was created by zstd functions, follow\n"

Modules/_zstd/zstddict.h

@@ -0,0 +1,31 @@
+/*
+Low level interface to Meta's zstd library for use in the compression.zstd
+Python module.
+*/
+
+#ifndef ZSTD_DICT_H
+#define ZSTD_DICT_H
+
+#include <stdbool.h>              // bool
+#include <zstd.h>                 // ZSTD_DDict
+
+typedef struct {
+    PyObject_HEAD
+
+    /* Reusable compress/decompress dictionary, they are created once and
+       can be shared by multiple threads concurrently, since its usage is
+       read-only.
+       c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */
+    ZSTD_DDict *d_dict;
+    PyObject *c_dicts;
+
+    /* Content of the dictionary, bytes object. */
+    PyObject *dict_content;
+    /* Dictionary id */
+    uint32_t dict_id;
+
+    /* __init__ has been called, 0 or 1. */
+    bool initialized;
+} ZstdDict;
+
+#endif  // !ZSTD_DICT_H

PCbuild/_zstd.vcxproj

@@ -137,6 +137,7 @@
   <ItemGroup>
     <ClInclude Include="..\Modules\_zstd\_zstdmodule.h" />
     <ClInclude Include="..\Modules\_zstd\buffer.h" />
+    <ClInclude Include="..\Modules\_zstd\zstddict.h" />
     <ClInclude Include="$(zstdDir)lib\common\bitstream.h" />
     <ClInclude Include="$(zstdDir)lib\common\error_private.h" />
     <ClInclude Include="$(zstdDir)lib\common\fse.h" />

PCbuild/_zstd.vcxproj.filters

@@ -128,6 +128,9 @@
     <ClInclude Include="..\Modules\_zstd\buffer.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\Modules\_zstd\zstddict.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="$(zstdDir)lib\zstd.h">
       <Filter>Header Files\zstd</Filter>
     </ClInclude>