gh-134160: Use multi-phase init in documentation examples (#134296)

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>

Doc/c-api/allocation.rst

@@ -153,6 +153,6 @@ Allocating Objects on the Heap
 
 .. seealso::
 
-   :c:func:`PyModule_Create`
+   :ref:`moduleobjects`
       To allocate and create extension modules.
 

Doc/c-api/intro.rst

@@ -127,7 +127,7 @@ complete listing.
    item defined in the module file. Example::
 
        static struct PyModuleDef spam_module = {
-           PyModuleDef_HEAD_INIT,
+           .m_base = PyModuleDef_HEAD_INIT,
            .m_name = "spam",
            ...
        };
@@ -135,7 +135,7 @@ complete listing.
        PyMODINIT_FUNC
        PyInit_spam(void)
        {
-           return PyModule_Create(&spam_module);
+           return PyModuleDef_Init(&spam_module);
        }
 
 

Doc/extending/building.rst

@@ -23,10 +23,10 @@ instance. See :ref:`initializing-modules` for details.
 .. highlight:: python
 
 For modules with ASCII-only names, the function must be named
-``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the
+:samp:`PyInit_{<name>}`, with ``<name>`` replaced by the name of the module.
-module. When using :ref:`multi-phase-initialization`, non-ASCII module names
+When using :ref:`multi-phase-initialization`, non-ASCII module names
 are allowed. In this case, the initialization function name is
-``PyInitU_<modulename>``, with ``<modulename>`` encoded using Python's
+:samp:`PyInitU_{<name>}`, with ``<name>`` encoded using Python's
 *punycode* encoding with hyphens replaced by underscores. In Python::
 
     def initfunc_name(name):

Doc/extending/embedding.rst

@@ -245,21 +245,23 @@ Python extension.  For example::
        return PyLong_FromLong(numargs);
    }
 
-   static PyMethodDef EmbMethods[] = {
+   static PyMethodDef emb_module_methods[] = {
        {"numargs", emb_numargs, METH_VARARGS,
         "Return the number of arguments received by the process."},
        {NULL, NULL, 0, NULL}
    };
 
-   static PyModuleDef EmbModule = {
+   static struct PyModuleDef emb_module = {
-       PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods,
+       .m_base = PyModuleDef_HEAD_INIT,
-       NULL, NULL, NULL, NULL
+       .m_name = "emb",
+       .m_size = 0,
+       .m_methods = emb_module_methods,
    };
 
    static PyObject*
    PyInit_emb(void)
    {
-       return PyModule_Create(&EmbModule);
+       return PyModuleDef_Init(&emb_module);
    }
 
 Insert the above code just above the :c:func:`main` function. Also, insert the

Doc/extending/extending.rst

@@ -203,31 +203,42 @@ function usually raises :c:data:`PyExc_TypeError`.  If you have an argument whos
 value must be in a particular range or must satisfy other conditions,
 :c:data:`PyExc_ValueError` is appropriate.
 
-You can also define a new exception that is unique to your module. For this, you
+You can also define a new exception that is unique to your module.
-usually declare a static object variable at the beginning of your file::
+For this, you can declare a static global object variable at the beginning
+of the file::
 
    static PyObject *SpamError;
 
-and initialize it in your module's initialization function (:c:func:`!PyInit_spam`)
+and initialize it with an exception object in the module's
-with an exception object::
+:c:data:`Py_mod_exec` function (:c:func:`!spam_module_exec`)::
+
+   static int
+   spam_module_exec(PyObject *m)
+   {
+       SpamError = PyErr_NewException("spam.error", NULL, NULL);
+       if (PyModule_AddObjectRef(m, "SpamError", SpamError) < 0) {
+           return -1;
+       }
+
+       return 0;
+   }
+
+   static PyModuleDef_Slot spam_module_slots[] = {
+       {Py_mod_exec, spam_module_exec},
+       {0, NULL}
+   };
+
+   static struct PyModuleDef spam_module = {
+       .m_base = PyModuleDef_HEAD_INIT,
+       .m_name = "spam",
+       .m_size = 0,  // non-negative
+       .m_slots = spam_module_slots,
+   };
 
    PyMODINIT_FUNC
    PyInit_spam(void)
    {
-       PyObject *m;
+       return PyModuleDef_Init(&spam_module);
-
-       m = PyModule_Create(&spammodule);
-       if (m == NULL)
-           return NULL;
-
-       SpamError = PyErr_NewException("spam.error", NULL, NULL);
-       if (PyModule_AddObjectRef(m, "error", SpamError) < 0) {
-           Py_CLEAR(SpamError);
-           Py_DECREF(m);
-           return NULL;
-       }
-
-       return m;
    }
 
 Note that the Python name for the exception object is :exc:`!spam.error`.  The
@@ -318,7 +329,7 @@ The Module's Method Table and Initialization Function
 I promised to show how :c:func:`!spam_system` is called from Python programs.
 First, we need to list its name and address in a "method table"::
 
-   static PyMethodDef SpamMethods[] = {
+   static PyMethodDef spam_methods[] = {
        ...
        {"system",  spam_system, METH_VARARGS,
         "Execute a shell command."},
@@ -343,13 +354,10 @@ function.
 
 The method table must be referenced in the module definition structure::
 
-   static struct PyModuleDef spammodule = {
+   static struct PyModuleDef spam_module = {
-       PyModuleDef_HEAD_INIT,
+       ...
-       "spam",   /* name of module */
+       .m_methods = spam_methods,
-       spam_doc, /* module documentation, may be NULL */
+       ...
-       -1,       /* size of per-interpreter state of the module,
-                    or -1 if the module keeps state in global variables. */
-       SpamMethods
    };
 
 This structure, in turn, must be passed to the interpreter in the module's
@@ -360,23 +368,17 @@ only non-\ ``static`` item defined in the module file::
    PyMODINIT_FUNC
    PyInit_spam(void)
    {
-       return PyModule_Create(&spammodule);
+       return PyModuleDef_Init(&spam_module);
    }
 
 Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type,
 declares any special linkage declarations required by the platform, and for C++
 declares the function as ``extern "C"``.
 
-When the Python program imports module :mod:`!spam` for the first time,
+:c:func:`!PyInit_spam` is called when each interpreter imports its module
-:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.)
+:mod:`!spam` for the first time.  (See below for comments about embedding Python.)
-It calls :c:func:`PyModule_Create`, which returns a module object, and
+A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`,
-inserts built-in function objects into the newly created module based upon the
+so that the import machinery can create the module and store it in ``sys.modules``.
-table (an array of :c:type:`PyMethodDef` structures) found in the module definition.
-:c:func:`PyModule_Create` returns a pointer to the module object
-that it creates.  It may abort with a fatal error for
-certain errors, or return ``NULL`` if the module could not be initialized
-satisfactorily. The init function must return the module object to its caller,
-so that it then gets inserted into ``sys.modules``.
 
 When embedding Python, the :c:func:`!PyInit_spam` function is not called
 automatically unless there's an entry in the :c:data:`PyImport_Inittab` table.
@@ -433,23 +435,19 @@ optionally followed by an import of the module::
 
 .. note::
 
-   Removing entries from ``sys.modules`` or importing compiled modules into
+   If you declare a global variable or a local static one, the module may
-   multiple interpreters within a process (or following a :c:func:`fork` without an
+   experience unintended side-effects on re-initialisation, for example when
-   intervening :c:func:`exec`) can create problems for some extension modules.
+   removing entries from ``sys.modules`` or importing compiled modules into
-   Extension module authors should exercise caution when initializing internal data
+   multiple interpreters within a process
-   structures.
+   (or following a :c:func:`fork` without an intervening :c:func:`exec`).
+   If module state is not yet fully :ref:`isolated <isolating-extensions-howto>`,
+   authors should consider marking the module as having no support for subinterpreters
+   (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`).
 
 A more substantial example module is included in the Python source distribution
-as :file:`Modules/xxmodule.c`.  This file may be used as a  template or simply
+as :file:`Modules/xxlimited.c`.  This file may be used as a template or simply
 read as an example.
 
-.. note::
-
-   Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization*
-   (new in Python 3.5), where a PyModuleDef structure is returned from
-   ``PyInit_spam``, and creation of the module is left to the import machinery.
-   For details on multi-phase initialization, see :PEP:`489`.
-
 
 .. _compilation:
 
@@ -790,18 +788,17 @@ Philbrick (philbrick@hks.com)::
        {NULL, NULL, 0, NULL}   /* sentinel */
    };
 
-   static struct PyModuleDef keywdargmodule = {
+   static struct PyModuleDef keywdarg_module = {
-       PyModuleDef_HEAD_INIT,
+       .m_base = PyModuleDef_HEAD_INIT,
-       "keywdarg",
+       .m_name = "keywdarg",
-       NULL,
+       .m_size = 0,
-       -1,
+       .m_methods = keywdarg_methods,
-       keywdarg_methods
    };
 
    PyMODINIT_FUNC
    PyInit_keywdarg(void)
    {
-       return PyModule_Create(&keywdargmodule);
+       return PyModuleDef_Init(&keywdarg_module);
    }
 
 
@@ -1072,8 +1069,9 @@ why his :meth:`!__del__` methods would fail...
 
 The second case of problems with a borrowed reference is a variant involving
 threads.  Normally, multiple threads in the Python interpreter can't get in each
-other's way, because there is a global lock protecting Python's entire object
+other's way, because there is a :term:`global lock <global interpreter lock>`
-space.  However, it is possible to temporarily release this lock using the macro
+protecting Python's entire object space.
+However, it is possible to temporarily release this lock using the macro
 :c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using
 :c:macro:`Py_END_ALLOW_THREADS`.  This is common around blocking I/O calls, to
 let other threads use the processor while waiting for the I/O to complete.
@@ -1259,20 +1257,15 @@ two more lines must be added::
    #include "spammodule.h"
 
 The ``#define`` is used to tell the header file that it is being included in the
-exporting module, not a client module. Finally, the module's initialization
+exporting module, not a client module. Finally, the module's :c:data:`mod_exec
-function must take care of initializing the C API pointer array::
+<Py_mod_exec>` function must take care of initializing the C API pointer array::
 
-   PyMODINIT_FUNC
+   static int
-   PyInit_spam(void)
+   spam_module_exec(PyObject *m)
    {
-       PyObject *m;
        static void *PySpam_API[PySpam_API_pointers];
        PyObject *c_api_object;
 
-       m = PyModule_Create(&spammodule);
-       if (m == NULL)
-           return NULL;
-
        /* Initialize the C API pointer array */
        PySpam_API[PySpam_System_NUM] = (void *)PySpam_System;
 
@@ -1280,11 +1273,10 @@ function must take care of initializing the C API pointer array::
        c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL);
 
        if (PyModule_Add(m, "_C_API", c_api_object) < 0) {
-           Py_DECREF(m);
+           return -1;
-           return NULL;
        }
 
-       return m;
+       return 0;
    }
 
 Note that ``PySpam_API`` is declared ``static``; otherwise the pointer
@@ -1343,20 +1335,16 @@ like this::
 
 All that a client module must do in order to have access to the function
 :c:func:`!PySpam_System` is to call the function (or rather macro)
-:c:func:`!import_spam` in its initialization function::
+:c:func:`!import_spam` in its :c:data:`mod_exec <Py_mod_exec>` function::
 
-   PyMODINIT_FUNC
+   static int
-   PyInit_client(void)
+   client_module_exec(PyObject *m)
    {
-       PyObject *m;
+       if (import_spam() < 0) {
-
+           return -1;
-       m = PyModule_Create(&clientmodule);
+       }
-       if (m == NULL)
-           return NULL;
-       if (import_spam() < 0)
-           return NULL;
        /* additional initialization can happen here */
-       return m;
+       return 0;
    }
 
 The main disadvantage of this approach is that the file :file:`spammodule.h` is

Doc/extending/index.rst

@@ -49,6 +49,10 @@ assistance from third party tools. It is intended primarily for creators
 of those tools, rather than being a recommended way to create your own
 C extensions.
 
+.. seealso::
+
+   :pep:`489` -- Multi-phase extension module initialization
+
 .. toctree::
    :maxdepth: 2
    :numbered:

Doc/extending/newtypes_tutorial.rst

@@ -55,8 +55,10 @@ from the previous chapter.  This file defines three things:
 #. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct,
    which defines a set of flags and function pointers that the interpreter
    inspects when specific operations are requested.
-#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom``
+#. How to define and execute the :mod:`!custom` module: this is the
-   function and the associated ``custommodule`` struct.
+   ``PyInit_custom`` function and the associated ``custom_module`` struct for
+   defining the module, and the ``custom_module_exec`` function to set up
+   a fresh module object.
 
 The first bit is::
 
@@ -171,18 +173,18 @@ implementation provided by the API function :c:func:`PyType_GenericNew`. ::
    .tp_new = PyType_GenericNew,
 
 Everything else in the file should be familiar, except for some code in
-:c:func:`!PyInit_custom`::
+:c:func:`!custom_module_exec`::
 
-   if (PyType_Ready(&CustomType) < 0)
+   if (PyType_Ready(&CustomType) < 0) {
-       return;
+       return -1;
+   }
 
 This initializes the :class:`!Custom` type, filling in a number of members
 to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially
 set to ``NULL``. ::
 
    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
-       Py_DECREF(m);
+       return -1;
-       return NULL;
    }
 
 This adds the type to the module dictionary.  This allows us to create
@@ -875,27 +877,22 @@ but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_
 The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base`
 specifying the type's concrete base class.  Due to cross-platform compiler
 issues, you can't fill that field directly with a reference to
-:c:type:`PyList_Type`; it should be done later in the module initialization
+:c:type:`PyList_Type`; it should be done in the :c:data:`Py_mod_exec`
 function::
 
-   PyMODINIT_FUNC
+   static int
-   PyInit_sublist(void)
+   sublist_module_exec(PyObject *m)
    {
-       PyObject* m;
        SubListType.tp_base = &PyList_Type;
-       if (PyType_Ready(&SubListType) < 0)
+       if (PyType_Ready(&SubListType) < 0) {
-           return NULL;
+           return -1;
-
-       m = PyModule_Create(&sublistmodule);
-       if (m == NULL)
-           return NULL;
-
-       if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) {
-           Py_DECREF(m);
-           return NULL;
        }
 
-       return m;
+       if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) {
+           return -1;
+       }
+
+       return 0;
    }
 
 Before calling :c:func:`PyType_Ready`, the type structure must have the

Doc/includes/newtypes/custom.c

@@ -16,28 +16,37 @@ static PyTypeObject CustomType = {
     .tp_new = PyType_GenericNew,
 };
 
-static PyModuleDef custommodule = {
+static int
+custom_module_exec(PyObject *m)
+{
+    if (PyType_Ready(&CustomType) < 0) {
+        return -1;
+    }
+
+    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static PyModuleDef_Slot custom_module_slots[] = {
+    {Py_mod_exec, custom_module_exec},
+    // Just use this while using static types
+    {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+    {0, NULL}
+};
+
+static PyModuleDef custom_module = {
     .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "custom",
     .m_doc = "Example module that creates an extension type.",
-    .m_size = -1,
+    .m_size = 0,
+    .m_slots = custom_module_slots,
 };
 
 PyMODINIT_FUNC
 PyInit_custom(void)
 {
-    PyObject *m;
+    return PyModuleDef_Init(&custom_module);
-    if (PyType_Ready(&CustomType) < 0)
-        return NULL;
-
-    m = PyModule_Create(&custommodule);
-    if (m == NULL)
-        return NULL;
-
-    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
-        Py_DECREF(m);
-        return NULL;
-    }
-
-    return m;
 }

Doc/includes/newtypes/custom2.c

@@ -106,28 +106,36 @@ static PyTypeObject CustomType = {
     .tp_methods = Custom_methods,
 };
 
-static PyModuleDef custommodule = {
+static int
-    .m_base =PyModuleDef_HEAD_INIT,
+custom_module_exec(PyObject *m)
+{
+    if (PyType_Ready(&CustomType) < 0) {
+        return -1;
+    }
+
+    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static PyModuleDef_Slot custom_module_slots[] = {
+    {Py_mod_exec, custom_module_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+    {0, NULL}
+};
+
+static PyModuleDef custom_module = {
+    .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "custom2",
     .m_doc = "Example module that creates an extension type.",
-    .m_size = -1,
+    .m_size = 0,
+    .m_slots = custom_module_slots,
 };
 
 PyMODINIT_FUNC
 PyInit_custom2(void)
 {
-    PyObject *m;
+    return PyModuleDef_Init(&custom_module);
-    if (PyType_Ready(&CustomType) < 0)
-        return NULL;
-
-    m = PyModule_Create(&custommodule);
-    if (m == NULL)
-        return NULL;
-
-    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
-        Py_DECREF(m);
-        return NULL;
-    }
-
-    return m;
 }

Doc/includes/newtypes/custom3.c

@@ -151,28 +151,36 @@ static PyTypeObject CustomType = {
     .tp_getset = Custom_getsetters,
 };
 
-static PyModuleDef custommodule = {
+static int
+custom_module_exec(PyObject *m)
+{
+    if (PyType_Ready(&CustomType) < 0) {
+        return -1;
+    }
+
+    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static PyModuleDef_Slot custom_module_slots[] = {
+    {Py_mod_exec, custom_module_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+    {0, NULL}
+};
+
+static PyModuleDef custom_module = {
     .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "custom3",
     .m_doc = "Example module that creates an extension type.",
-    .m_size = -1,
+    .m_size = 0,
+    .m_slots = custom_module_slots,
 };
 
 PyMODINIT_FUNC
 PyInit_custom3(void)
 {
-    PyObject *m;
+    return PyModuleDef_Init(&custom_module);
-    if (PyType_Ready(&CustomType) < 0)
-        return NULL;
-
-    m = PyModule_Create(&custommodule);
-    if (m == NULL)
-        return NULL;
-
-    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
-        Py_DECREF(m);
-        return NULL;
-    }
-
-    return m;
 }

Doc/includes/newtypes/custom4.c

@@ -170,28 +170,36 @@ static PyTypeObject CustomType = {
     .tp_getset = Custom_getsetters,
 };
 
-static PyModuleDef custommodule = {
+static int
+custom_module_exec(PyObject *m)
+{
+    if (PyType_Ready(&CustomType) < 0) {
+        return -1;
+    }
+
+    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static PyModuleDef_Slot custom_module_slots[] = {
+    {Py_mod_exec, custom_module_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+    {0, NULL}
+};
+
+static PyModuleDef custom_module = {
     .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "custom4",
     .m_doc = "Example module that creates an extension type.",
-    .m_size = -1,
+    .m_size = 0,
+    .m_slots = custom_module_slots,
 };
 
 PyMODINIT_FUNC
 PyInit_custom4(void)
 {
-    PyObject *m;
+    return PyModuleDef_Init(&custom_module);
-    if (PyType_Ready(&CustomType) < 0)
-        return NULL;
-
-    m = PyModule_Create(&custommodule);
-    if (m == NULL)
-        return NULL;
-
-    if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) {
-        Py_DECREF(m);
-        return NULL;
-    }
-
-    return m;
 }

Doc/includes/newtypes/sublist.c

@@ -31,7 +31,7 @@ SubList_init(PyObject *op, PyObject *args, PyObject *kwds)
 }
 
 static PyTypeObject SubListType = {
-    PyVarObject_HEAD_INIT(NULL, 0)
+    .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
     .tp_name = "sublist.SubList",
     .tp_doc = PyDoc_STR("SubList objects"),
     .tp_basicsize = sizeof(SubListObject),
@@ -41,29 +41,37 @@ static PyTypeObject SubListType = {
     .tp_methods = SubList_methods,
 };
 
-static PyModuleDef sublistmodule = {
+static int
-    PyModuleDef_HEAD_INIT,
+sublist_module_exec(PyObject *m)
+{
+    SubListType.tp_base = &PyList_Type;
+    if (PyType_Ready(&SubListType) < 0) {
+        return -1;
+    }
+
+    if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static PyModuleDef_Slot sublist_module_slots[] = {
+    {Py_mod_exec, sublist_module_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},
+    {0, NULL}
+};
+
+static PyModuleDef sublist_module = {
+    .m_base = PyModuleDef_HEAD_INIT,
     .m_name = "sublist",
     .m_doc = "Example module that creates an extension type.",
-    .m_size = -1,
+    .m_size = 0,
+    .m_slots = sublist_module_slots,
 };
 
 PyMODINIT_FUNC
 PyInit_sublist(void)
 {
-    PyObject *m;
+    return PyModuleDef_Init(&sublist_module);
-    SubListType.tp_base = &PyList_Type;
-    if (PyType_Ready(&SubListType) < 0)
-        return NULL;
-
-    m = PyModule_Create(&sublistmodule);
-    if (m == NULL)
-        return NULL;
-
-    if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) {
-        Py_DECREF(m);
-        return NULL;
-    }
-
-    return m;
 }