bpo-46008: Make runtime-global object/type lifecycle functions and state consistent. (gh-29998)

This change is strictly renames and moving code around.  It helps in the following ways:

* ensures type-related init functions focus strictly on one of the three aspects (state, objects, types)
* passes in PyInterpreterState * to all those functions, simplifying work on moving types/objects/state to the interpreter
* consistent naming conventions help make what's going on more clear
* keeping API related to a type in the corresponding header file makes it more obvious where to look for it

https://bugs.python.org/issue46008

Objects/bytesobject.c

@@ -5,6 +5,7 @@
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
 #include "pycore_bytes_methods.h" // _Py_bytes_startswith()
+#include "pycore_bytesobject.h"   // struct _Py_bytes_state
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_format.h"        // F_LJUST
 #include "pycore_initconfig.h"    // _PyStatus_OK()
@@ -3086,7 +3087,7 @@ error:
 
 
 PyStatus
-_PyBytes_Init(PyInterpreterState *interp)
+_PyBytes_InitGlobalObjects(PyInterpreterState *interp)
 {
     struct _Py_bytes_state *state = &interp->bytes;
     if (bytes_create_empty_string_singleton(state) < 0) {
@@ -3096,6 +3097,25 @@ _PyBytes_Init(PyInterpreterState *interp)
 }
 
 
+PyStatus
+_PyBytes_InitTypes(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
+    if (PyType_Ready(&PyBytes_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize bytes type");
+    }
+
+    if (PyType_Ready(&PyBytesIter_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize bytes iterator type");
+    }
+
+    return _PyStatus_OK();
+}
+
+
 void
 _PyBytes_Fini(PyInterpreterState *interp)
 {

Objects/exceptions.c

@@ -7,6 +7,7 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <stdbool.h>
+#include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_initconfig.h"
 #include "pycore_object.h"
 #include "structmember.h"         // PyMemberDef
@@ -3189,10 +3190,8 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning,
 #endif /* MS_WINDOWS */
 
 PyStatus
-_PyExc_Init(PyInterpreterState *interp)
+_PyExc_InitTypes(PyInterpreterState *interp)
 {
-    struct _Py_exc_state *state = &interp->exc_state;
-
 #define PRE_INIT(TYPE) \
     if (!(_PyExc_ ## TYPE.tp_flags & Py_TPFLAGS_READY)) { \
         if (PyType_Ready(&_PyExc_ ## TYPE) < 0) { \
@@ -3201,17 +3200,6 @@ _PyExc_Init(PyInterpreterState *interp)
         Py_INCREF(PyExc_ ## TYPE); \
     }
 
-#define ADD_ERRNO(TYPE, CODE) \
-    do { \
-        PyObject *_code = PyLong_FromLong(CODE); \
-        assert(_PyObject_RealIsSubclass(PyExc_ ## TYPE, PyExc_OSError)); \
-        if (!_code || PyDict_SetItem(state->errnomap, _code, PyExc_ ## TYPE)) { \
-            Py_XDECREF(_code); \
-            return _PyStatus_ERR("errmap insertion problem."); \
-        } \
-        Py_DECREF(_code); \
-    } while (0)
-
     PRE_INIT(BaseException);
     PRE_INIT(BaseExceptionGroup);
     PRE_INIT(Exception);
@@ -3282,10 +3270,37 @@ _PyExc_Init(PyInterpreterState *interp)
     PRE_INIT(ProcessLookupError);
     PRE_INIT(TimeoutError);
 
+    return _PyStatus_OK();
+
+#undef PRE_INIT
+}
+
+PyStatus
+_PyExc_InitGlobalObjects(PyInterpreterState *interp)
+{
     if (preallocate_memerrors() < 0) {
         return _PyStatus_NO_MEMORY();
     }
 
+    return _PyStatus_OK();
+}
+
+PyStatus
+_PyExc_InitState(PyInterpreterState *interp)
+{
+    struct _Py_exc_state *state = &interp->exc_state;
+
+#define ADD_ERRNO(TYPE, CODE) \
+    do { \
+        PyObject *_code = PyLong_FromLong(CODE); \
+        assert(_PyObject_RealIsSubclass(PyExc_ ## TYPE, PyExc_OSError)); \
+        if (!_code || PyDict_SetItem(state->errnomap, _code, PyExc_ ## TYPE)) { \
+            Py_XDECREF(_code); \
+            return _PyStatus_ERR("errmap insertion problem."); \
+        } \
+        Py_DECREF(_code); \
+    } while (0)
+
     /* Add exceptions to errnomap */
     assert(state->errnomap == NULL);
     state->errnomap = PyDict_New();
@@ -3317,7 +3332,6 @@ _PyExc_Init(PyInterpreterState *interp)
 
     return _PyStatus_OK();
 
-#undef PRE_INIT
 #undef ADD_ERRNO
 }
 

Objects/floatobject.c

@@ -6,6 +6,7 @@
 #include "Python.h"
 #include "pycore_dtoa.h"          // _Py_dg_dtoa()
 #include "pycore_floatobject.h"   // _PyFloat_FormatAdvancedWriter()
+#include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_interp.h"        // _PyInterpreterState.float_state
 #include "pycore_long.h"          // _PyLong_GetOne()
 #include "pycore_object.h"        // _PyObject_Init()
@@ -1981,8 +1982,12 @@ PyTypeObject PyFloat_Type = {
 };
 
 void
-_PyFloat_Init(void)
+_PyFloat_InitState(PyInterpreterState *interp)
 {
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
     /* We attempt to determine if this machine is using IEEE
        floating point formats by peering at the bits of some
        carefully chosen values.  If it looks like we are on an
@@ -2030,16 +2035,25 @@ _PyFloat_Init(void)
     float_format = detected_float_format;
 }
 
-int
-_PyFloat_InitTypes(void)
+PyStatus
+_PyFloat_InitTypes(PyInterpreterState *interp)
 {
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
+    if (PyType_Ready(&PyFloat_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize float type");
+    }
+
     /* Init float info */
     if (FloatInfoType.tp_name == NULL) {
         if (PyStructSequence_InitType2(&FloatInfoType, &floatinfo_desc) < 0) {
-            return -1;
+            return _PyStatus_ERR("can't init float info type");
         }
     }
-    return 0;
+
+    return _PyStatus_OK();
 }
 
 void

Objects/genobject.c

@@ -3,6 +3,7 @@
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_EvalFrame()
+#include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
 #include "pycore_pyerrors.h"      // _PyErr_ClearExcState()
 #include "pycore_pystate.h"       // _PyThreadState_GET()

Objects/listobject.c

@@ -3,6 +3,7 @@
 #include "Python.h"
 #include "pycore_abstract.h"      // _PyIndex_Check()
 #include "pycore_interp.h"        // PyInterpreterState.list
+#include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
 #include "pycore_tuple.h"         // _PyTuple_FromArray()
 #include <stddef.h>

Objects/longobject.c

@@ -4,10 +4,11 @@
 
 #include "Python.h"
 #include "pycore_bitutils.h"      // _Py_popcount32()
-#include "pycore_runtime.h"       // _PY_NSMALLPOSINTS
+#include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_long.h"          // _Py_SmallInts
 #include "pycore_object.h"        // _PyObject_InitVar()
 #include "pycore_pystate.h"       // _Py_IsMainInterpreter()
+#include "pycore_runtime.h"       // _PY_NSMALLPOSINTS
 
 #include <ctype.h>
 #include <float.h>
@@ -48,7 +49,7 @@ static PyObject *
 get_small_int(sdigit ival)
 {
     assert(IS_SMALL_INT(ival));
-    PyObject *v = (PyObject *)&_PyRuntime.small_ints[_PY_NSMALLNEGINTS + ival];
+    PyObject *v = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + ival];
     Py_INCREF(v);
     return v;
 }
@@ -5828,31 +5829,51 @@ PyLong_GetInfo(void)
     return int_info;
 }
 
+
+/* runtime lifecycle */
+
 void
-_PyLong_Init(PyInterpreterState *interp)
+_PyLong_InitGlobalObjects(PyInterpreterState *interp)
 {
-    if (_PyRuntime.small_ints[0].ob_base.ob_base.ob_refcnt == 0) {
-        for (Py_ssize_t i=0; i < _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS; i++) {
-            sdigit ival = (sdigit)i - _PY_NSMALLNEGINTS;
-            int size = (ival < 0) ? -1 : ((ival == 0) ? 0 : 1);
-            _PyRuntime.small_ints[i].ob_base.ob_base.ob_refcnt = 1;
-            _PyRuntime.small_ints[i].ob_base.ob_base.ob_type = &PyLong_Type;
-            _PyRuntime.small_ints[i].ob_base.ob_size = size;
-            _PyRuntime.small_ints[i].ob_digit[0] = (digit)abs(ival);
-        }
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
+    PyLongObject *small_ints = _PyLong_SMALL_INTS;
+    if (small_ints[0].ob_base.ob_base.ob_refcnt != 0) {
+        // Py_Initialize() must be running a second time.
+        return;
+    }
+
+    for (Py_ssize_t i=0; i < _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS; i++) {
+        sdigit ival = (sdigit)i - _PY_NSMALLNEGINTS;
+        int size = (ival < 0) ? -1 : ((ival == 0) ? 0 : 1);
+        small_ints[i].ob_base.ob_base.ob_refcnt = 1;
+        small_ints[i].ob_base.ob_base.ob_type = &PyLong_Type;
+        small_ints[i].ob_base.ob_size = size;
+        small_ints[i].ob_digit[0] = (digit)abs(ival);
     }
 }
 
-int
-_PyLong_InitTypes(void)
+PyStatus
+_PyLong_InitTypes(PyInterpreterState *interp)
 {
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
+    if (PyType_Ready(&PyLong_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize int type");
+    }
+
     /* initialize int_info */
     if (Int_InfoType.tp_name == NULL) {
         if (PyStructSequence_InitType2(&Int_InfoType, &int_info_desc) < 0) {
-            return -1;
+            return _PyStatus_ERR("can't init int info type");
         }
     }
-    return 0;
+
+    return _PyStatus_OK();
 }
 
 void

Objects/object.c

@@ -10,10 +10,10 @@
 #include "pycore_namespace.h"     // _PyNamespace_Type
 #include "pycore_object.h"        // _PyType_CheckConsistency()
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
-#include "pycore_pylifecycle.h"   // _PyTypes_InitSlotDefs()
 #include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_symtable.h"      // PySTEntry_Type
+#include "pycore_typeobject.h"    // _PyTypes_InitSlotDefs()
 #include "pycore_unionobject.h"   // _PyUnion_Type
 #include "frameobject.h"          // PyFrame_Type
 #include "pycore_interpreteridobject.h"  // _PyInterpreterID_Type
@@ -1823,13 +1823,27 @@ PyObject _Py_NotImplementedStruct = {
 };
 
 PyStatus
-_PyTypes_Init(void)
+_PyTypes_InitState(PyInterpreterState *interp)
 {
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
     PyStatus status = _PyTypes_InitSlotDefs();
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
 
+    return _PyStatus_OK();
+}
+
+PyStatus
+_PyTypes_InitTypes(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
 #define INIT_TYPE(TYPE) \
     do { \
         if (PyType_Ready(&(TYPE)) < 0) { \
@@ -1843,13 +1857,11 @@ _PyTypes_Init(void)
     assert(PyBaseObject_Type.tp_base == NULL);
     assert(PyType_Type.tp_base == &PyBaseObject_Type);
 
-    // All other static types
+    // All other static types (unless initialized elsewhere)
     INIT_TYPE(PyAsyncGen_Type);
     INIT_TYPE(PyBool_Type);
     INIT_TYPE(PyByteArrayIter_Type);
     INIT_TYPE(PyByteArray_Type);
-    INIT_TYPE(PyBytesIter_Type);
-    INIT_TYPE(PyBytes_Type);
     INIT_TYPE(PyCFunction_Type);
     INIT_TYPE(PyCMethod_Type);
     INIT_TYPE(PyCallIter_Type);
@@ -1873,7 +1885,6 @@ _PyTypes_Init(void)
     INIT_TYPE(PyDict_Type);
     INIT_TYPE(PyEllipsis_Type);
     INIT_TYPE(PyEnum_Type);
-    INIT_TYPE(PyFloat_Type);
     INIT_TYPE(PyFrame_Type);
     INIT_TYPE(PyFrozenSet_Type);
     INIT_TYPE(PyFunction_Type);
@@ -1884,7 +1895,6 @@ _PyTypes_Init(void)
     INIT_TYPE(PyListRevIter_Type);
     INIT_TYPE(PyList_Type);
     INIT_TYPE(PyLongRangeIter_Type);
-    INIT_TYPE(PyLong_Type);
     INIT_TYPE(PyMemberDescr_Type);
     INIT_TYPE(PyMemoryView_Type);
     INIT_TYPE(PyMethodDescr_Type);
@@ -1910,10 +1920,6 @@ _PyTypes_Init(void)
     INIT_TYPE(PyStdPrinter_Type);
     INIT_TYPE(PySuper_Type);
     INIT_TYPE(PyTraceBack_Type);
-    INIT_TYPE(PyTupleIter_Type);
-    INIT_TYPE(PyTuple_Type);
-    INIT_TYPE(PyUnicodeIter_Type);
-    INIT_TYPE(PyUnicode_Type);
     INIT_TYPE(PyWrapperDescr_Type);
     INIT_TYPE(Py_GenericAliasType);
     INIT_TYPE(_PyAnextAwaitable_Type);

Objects/structseq.c

@@ -12,6 +12,7 @@
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
 #include "structmember.h"         // PyMemberDef
 #include "pycore_structseq.h"     // PyStructSequence_InitType()
+#include "pycore_initconfig.h"    // _PyStatus_OK()
 
 static const char visible_length_key[] = "n_sequence_fields";
 static const char real_length_key[] = "n_fields";
@@ -583,13 +584,20 @@ PyStructSequence_NewType(PyStructSequence_Desc *desc)
     return type;
 }
 
-int _PyStructSequence_Init(void)
+
+/* runtime lifecycle */
+
+PyStatus _PyStructSequence_InitState(PyInterpreterState *interp)
 {
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
     if (_PyUnicode_FromId(&PyId_n_sequence_fields) == NULL
         || _PyUnicode_FromId(&PyId_n_fields) == NULL
         || _PyUnicode_FromId(&PyId_n_unnamed_fields) == NULL)
     {
-        return -1;
+        return _PyStatus_ERR("can't initialize structseq state");
     }
-    return 0;
+    return _PyStatus_OK();
 }

Objects/tupleobject.c

@@ -7,6 +7,7 @@
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_object.h"        // _PyObject_GC_TRACK()
 #include "pycore_pyerrors.h"      // _Py_FatalRefcountError()
+#include "pycore_tuple.h"         // struct _Py_tuple_state()
 
 /*[clinic input]
 class tuple "PyTupleObject *" "&PyTuple_Type"
@@ -1066,7 +1067,7 @@ _PyTuple_ClearFreeList(PyInterpreterState *interp)
 
 
 PyStatus
-_PyTuple_Init(PyInterpreterState *interp)
+_PyTuple_InitGlobalObjects(PyInterpreterState *interp)
 {
     struct _Py_tuple_state *state = &interp->tuple;
     if (tuple_create_empty_tuple_singleton(state) < 0) {
@@ -1076,6 +1077,24 @@ _PyTuple_Init(PyInterpreterState *interp)
 }
 
 
+PyStatus
+_PyTuple_InitTypes(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
+    if (PyType_Ready(&PyTuple_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize tuple type");
+    }
+
+    if (PyType_Ready(&PyTupleIter_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize tuple iterator type");
+    }
+
+    return _PyStatus_OK();
+}
+
 void
 _PyTuple_Fini(PyInterpreterState *interp)
 {

Objects/typeobject.c

@@ -9,6 +9,7 @@
 #include "pycore_object.h"        // _PyType_HasFeature()
 #include "pycore_pyerrors.h"      // _PyErr_Occurred()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_typeobject.h"    // struct type_cache
 #include "pycore_unionobject.h"   // _Py_union_type_or
 #include "frameobject.h"          // PyFrameObject
 #include "pycore_frame.h"         // InterpreterFrame
@@ -294,7 +295,7 @@ PyType_ClearCache(void)
 
 
 void
-_PyType_Fini(PyInterpreterState *interp)
+_PyTypes_Fini(PyInterpreterState *interp)
 {
     struct type_cache *cache = &interp->type_cache;
     type_cache_clear(cache, NULL);

Objects/unicodeobject.c

@@ -53,6 +53,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #include "pycore_pylifecycle.h"   // _Py_SetFileSystemEncoding()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
+#include "pycore_unicodeobject.h" // struct _Py_unicode_state
 #include "stringlib/eq.h"         // unicode_eq()
 
 #ifdef MS_WINDOWS
@@ -15504,41 +15505,56 @@ PyTypeObject PyUnicode_Type = {
 
 /* Initialize the Unicode implementation */
 
+void
+_PyUnicode_InitState(PyInterpreterState *interp)
+{
+    if (!_Py_IsMainInterpreter(interp)) {
+        return;
+    }
+
+    /* initialize the linebreak bloom filter */
+    const Py_UCS2 linebreak[] = {
+        0x000A, /* LINE FEED */
+        0x000D, /* CARRIAGE RETURN */
+        0x001C, /* FILE SEPARATOR */
+        0x001D, /* GROUP SEPARATOR */
+        0x001E, /* RECORD SEPARATOR */
+        0x0085, /* NEXT LINE */
+        0x2028, /* LINE SEPARATOR */
+        0x2029, /* PARAGRAPH SEPARATOR */
+    };
+    bloom_linebreak = make_bloom_mask(
+        PyUnicode_2BYTE_KIND, linebreak,
+        Py_ARRAY_LENGTH(linebreak));
+}
+
+
 PyStatus
-_PyUnicode_Init(PyInterpreterState *interp)
+_PyUnicode_InitGlobalObjects(PyInterpreterState *interp)
 {
     struct _Py_unicode_state *state = &interp->unicode;
     if (unicode_create_empty_string_singleton(state) < 0) {
         return _PyStatus_NO_MEMORY();
     }
 
-    if (_Py_IsMainInterpreter(interp)) {
-        /* initialize the linebreak bloom filter */
-        const Py_UCS2 linebreak[] = {
-            0x000A, /* LINE FEED */
-            0x000D, /* CARRIAGE RETURN */
-            0x001C, /* FILE SEPARATOR */
-            0x001D, /* GROUP SEPARATOR */
-            0x001E, /* RECORD SEPARATOR */
-            0x0085, /* NEXT LINE */
-            0x2028, /* LINE SEPARATOR */
-            0x2029, /* PARAGRAPH SEPARATOR */
-        };
-        bloom_linebreak = make_bloom_mask(
-            PyUnicode_2BYTE_KIND, linebreak,
-            Py_ARRAY_LENGTH(linebreak));
-    }
-
     return _PyStatus_OK();
 }
 
 
 PyStatus
-_PyUnicode_InitTypes(void)
+_PyUnicode_InitTypes(PyInterpreterState *interp)
 {
+    if (!_Py_IsMainInterpreter(interp)) {
+        return _PyStatus_OK();
+    }
+
     if (PyType_Ready(&PyUnicode_Type) < 0) {
         return _PyStatus_ERR("Can't initialize unicode type");
     }
+    if (PyType_Ready(&PyUnicodeIter_Type) < 0) {
+        return _PyStatus_ERR("Can't initialize unicode iterator type");
+    }
+
     if (PyType_Ready(&EncodingMapType) < 0) {
          return _PyStatus_ERR("Can't initialize encoding map type");
     }