gh-128384: Use a context variable for warnings.catch_warnings (gh-130010)

Make `warnings.catch_warnings()` use a context variable for holding
the warning filtering state if the `sys.flags.context_aware_warnings`
flag is set to true.  This makes using the context manager thread-safe in
multi-threaded programs.

Add the `sys.flags.thread_inherit_context` flag.  If true, starting a new
thread with `threading.Thread` will use a copy of the context
from the caller of `Thread.start()`.

Both these flags are set to true by default for the free-threaded build
and false for the default build.

Move the Python implementation of warnings.py into _py_warnings.py.

Make _contextvars a builtin module.

Co-authored-by: Kumar Aditya <kumaraditya@python.org>

Python/_contextvars.c

@@ -0,0 +1,68 @@
+#include "Python.h"
+
+#include "clinic/_contextvars.c.h"
+
+/*[clinic input]
+module _contextvars
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a0955718c8b8cea6]*/
+
+
+/*[clinic input]
+_contextvars.copy_context
+[clinic start generated code]*/
+
+static PyObject *
+_contextvars_copy_context_impl(PyObject *module)
+/*[clinic end generated code: output=1fcd5da7225c4fa9 input=89bb9ae485888440]*/
+{
+    return PyContext_CopyCurrent();
+}
+
+
+PyDoc_STRVAR(module_doc, "Context Variables");
+
+static PyMethodDef _contextvars_methods[] = {
+    _CONTEXTVARS_COPY_CONTEXT_METHODDEF
+    {NULL, NULL}
+};
+
+static int
+_contextvars_exec(PyObject *m)
+{
+    if (PyModule_AddType(m, &PyContext_Type) < 0) {
+        return -1;
+    }
+    if (PyModule_AddType(m, &PyContextVar_Type) < 0) {
+        return -1;
+    }
+    if (PyModule_AddType(m, &PyContextToken_Type) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+static struct PyModuleDef_Slot _contextvars_slots[] = {
+    {Py_mod_exec, _contextvars_exec},
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
+    {0, NULL}
+};
+
+static struct PyModuleDef _contextvarsmodule = {
+    PyModuleDef_HEAD_INIT,      /* m_base */
+    "_contextvars",             /* m_name */
+    module_doc,                 /* m_doc */
+    0,                          /* m_size */
+    _contextvars_methods,       /* m_methods */
+    _contextvars_slots,         /* m_slots */
+    NULL,                       /* m_traverse */
+    NULL,                       /* m_clear */
+    NULL,                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__contextvars(void)
+{
+    return PyModuleDef_Init(&_contextvarsmodule);
+}

Python/_warnings.c

@@ -69,6 +69,7 @@ warnings_clear_state(WarningsState *st)
     Py_CLEAR(st->filters);
     Py_CLEAR(st->once_registry);
     Py_CLEAR(st->default_action);
+    Py_CLEAR(st->context);
 }
 
 #ifndef Py_DEBUG
@@ -156,6 +157,13 @@ _PyWarnings_InitState(PyInterpreterState *interp)
         }
     }
 
+    if (st->context == NULL) {
+        st->context = PyContextVar_New("_warnings_context", NULL);
+        if (st->context == NULL) {
+            return -1;
+        }
+    }
+
     st->filters_version = 0;
     return 0;
 }
@@ -256,6 +264,68 @@ warnings_lock_held(WarningsState *st)
     return PyMutex_IsLocked(&st->lock.mutex);
 }
 
+static PyObject *
+get_warnings_context(PyInterpreterState *interp)
+{
+    WarningsState *st = warnings_get_state(interp);
+    assert(PyContextVar_CheckExact(st->context));
+    PyObject *ctx;
+    if (PyContextVar_Get(st->context, NULL, &ctx) < 0) {
+        return NULL;
+    }
+    if (ctx == NULL) {
+        Py_RETURN_NONE;
+    }
+    return ctx;
+}
+
+static PyObject *
+get_warnings_context_filters(PyInterpreterState *interp)
+{
+    PyObject *ctx = get_warnings_context(interp);
+    if (ctx == NULL) {
+        return NULL;
+    }
+    if (ctx == Py_None) {
+        Py_RETURN_NONE;
+    }
+    PyObject *context_filters = PyObject_GetAttr(ctx, &_Py_ID(_filters));
+    Py_DECREF(ctx);
+    if (context_filters == NULL) {
+        return NULL;
+    }
+    if (!PyList_Check(context_filters)) {
+        PyErr_SetString(PyExc_ValueError,
+                        "_filters of warnings._warnings_context must be a list");
+        Py_DECREF(context_filters);
+        return NULL;
+    }
+    return context_filters;
+}
+
+// Returns a borrowed reference to the list.
+static PyObject *
+get_warnings_filters(PyInterpreterState *interp)
+{
+    WarningsState *st = warnings_get_state(interp);
+    PyObject *warnings_filters = GET_WARNINGS_ATTR(interp, filters, 0);
+    if (warnings_filters == NULL) {
+        if (PyErr_Occurred())
+            return NULL;
+    }
+    else {
+        Py_SETREF(st->filters, warnings_filters);
+    }
+
+    PyObject *filters = st->filters;
+    if (filters == NULL || !PyList_Check(filters)) {
+        PyErr_SetString(PyExc_ValueError,
+                        MODULE_NAME ".filters must be a list");
+        return NULL;
+    }
+    return filters;
+}
+
 /*[clinic input]
 _acquire_lock as warnings_acquire_lock
 
@@ -349,35 +419,17 @@ get_default_action(PyInterpreterState *interp)
     return default_action;
 }
 
-
-/* The item is a new reference. */
-static PyObject*
-get_filter(PyInterpreterState *interp, PyObject *category,
-           PyObject *text, Py_ssize_t lineno,
-           PyObject *module, PyObject **item)
-{
-    WarningsState *st = warnings_get_state(interp);
-    assert(st != NULL);
-
-    assert(warnings_lock_held(st));
-
-    PyObject *warnings_filters = GET_WARNINGS_ATTR(interp, filters, 0);
-    if (warnings_filters == NULL) {
-        if (PyErr_Occurred())
-            return NULL;
-    }
-    else {
-        Py_SETREF(st->filters, warnings_filters);
-    }
-
-    PyObject *filters = st->filters;
-    if (filters == NULL || !PyList_Check(filters)) {
-        PyErr_SetString(PyExc_ValueError,
-                        MODULE_NAME ".filters must be a list");
-        return NULL;
-    }
-
-    /* WarningsState.filters could change while we are iterating over it. */
+/* Search filters list of match, returns false on error.  If no match
+ * then 'matched_action' is NULL.  */
+static bool
+filter_search(PyInterpreterState *interp, PyObject *category,
+              PyObject *text, Py_ssize_t lineno,
+              PyObject *module, char *list_name, PyObject *filters,
+              PyObject **item, PyObject **matched_action) {
+    bool result = true;
+    *matched_action = NULL;
+    /* Avoid the filters list changing while we iterate over it. */
+    Py_BEGIN_CRITICAL_SECTION(filters);
     for (Py_ssize_t i = 0; i < PyList_GET_SIZE(filters); i++) {
         PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj;
         Py_ssize_t ln;
@@ -386,8 +438,9 @@ get_filter(PyInterpreterState *interp, PyObject *category,
         tmp_item = PyList_GET_ITEM(filters, i);
         if (!PyTuple_Check(tmp_item) || PyTuple_GET_SIZE(tmp_item) != 5) {
             PyErr_Format(PyExc_ValueError,
-                         MODULE_NAME ".filters item %zd isn't a 5-tuple", i);
-            return NULL;
+                         "warnings.%s item %zd isn't a 5-tuple", list_name, i);
+            result = false;
+            break;
         }
 
         /* Python code: action, msg, cat, mod, ln = item */
@@ -403,42 +456,102 @@ get_filter(PyInterpreterState *interp, PyObject *category,
                          "action must be a string, not '%.200s'",
                          Py_TYPE(action)->tp_name);
             Py_DECREF(tmp_item);
-            return NULL;
+            result = false;
+            break;
         }
 
         good_msg = check_matched(interp, msg, text);
         if (good_msg == -1) {
             Py_DECREF(tmp_item);
-            return NULL;
+            result = false;
+            break;
         }
 
         good_mod = check_matched(interp, mod, module);
         if (good_mod == -1) {
             Py_DECREF(tmp_item);
-            return NULL;
+            result = false;
+            break;
         }
 
         is_subclass = PyObject_IsSubclass(category, cat);
         if (is_subclass == -1) {
             Py_DECREF(tmp_item);
-            return NULL;
+            result = false;
+            break;
         }
 
         ln = PyLong_AsSsize_t(ln_obj);
         if (ln == -1 && PyErr_Occurred()) {
             Py_DECREF(tmp_item);
-            return NULL;
+            result = false;
+            break;
         }
 
         if (good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln)) {
             *item = tmp_item;
-            return action;
+            *matched_action = action;
+            result = true;
+            break;
         }
 
         Py_DECREF(tmp_item);
     }
+    Py_END_CRITICAL_SECTION();
+    return result;
+}
 
-    PyObject *action = get_default_action(interp);
+/* The item is a new reference. */
+static PyObject*
+get_filter(PyInterpreterState *interp, PyObject *category,
+           PyObject *text, Py_ssize_t lineno,
+           PyObject *module, PyObject **item)
+{
+#ifdef Py_DEBUG
+    WarningsState *st = warnings_get_state(interp);
+    assert(st != NULL);
+    assert(warnings_lock_held(st));
+#endif
+
+    /* check _warning_context _filters list */
+    PyObject *context_filters = get_warnings_context_filters(interp);
+    if (context_filters == NULL) {
+        return NULL;
+    }
+    bool use_global_filters = false;
+    if (context_filters == Py_None) {
+        use_global_filters = true;
+    } else {
+        PyObject *context_action = NULL;
+        if (!filter_search(interp, category, text, lineno, module, "_warnings_context _filters",
+                           context_filters, item, &context_action)) {
+            Py_DECREF(context_filters);
+            return NULL;
+        }
+        Py_DECREF(context_filters);
+        if (context_action != NULL) {
+            return context_action;
+        }
+    }
+
+    PyObject *action;
+
+    if (use_global_filters) {
+        /* check warnings.filters list */
+        PyObject *filters = get_warnings_filters(interp);
+        if (filters == NULL) {
+            return NULL;
+        }
+        if (!filter_search(interp, category, text, lineno, module, "filters",
+                           filters, item, &action)) {
+            return NULL;
+        }
+        if (action != NULL) {
+            return action;
+        }
+    }
+
+    action = get_default_action(interp);
     if (action != NULL) {
         *item = Py_NewRef(Py_None);
         return action;
@@ -1540,6 +1653,9 @@ warnings_module_exec(PyObject *module)
     if (PyModule_AddObjectRef(module, "_defaultaction", st->default_action) < 0) {
         return -1;
     }
+    if (PyModule_AddObjectRef(module, "_warnings_context", st->context) < 0) {
+        return -1;
+    }
     return 0;
 }
 

Python/clinic/_contextvars.c.h

@@ -0,0 +1,21 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+PyDoc_STRVAR(_contextvars_copy_context__doc__,
+"copy_context($module, /)\n"
+"--\n"
+"\n");
+
+#define _CONTEXTVARS_COPY_CONTEXT_METHODDEF    \
+    {"copy_context", (PyCFunction)_contextvars_copy_context, METH_NOARGS, _contextvars_copy_context__doc__},
+
+static PyObject *
+_contextvars_copy_context_impl(PyObject *module);
+
+static PyObject *
+_contextvars_copy_context(PyObject *module, PyObject *Py_UNUSED(ignored))
+{
+    return _contextvars_copy_context_impl(module);
+}
+/*[clinic end generated code: output=26e07024451baf52 input=a9049054013a1b77]*/

Python/initconfig.c

@@ -151,6 +151,8 @@ static const PyConfigSpec PYCONFIG_SPEC[] = {
     SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS),
     SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS),
     SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS),
+    SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS),
+    SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS),
     SPEC(import_time, BOOL, READ_ONLY, NO_SYS),
     SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS),
     SPEC(isolated, BOOL, READ_ONLY, NO_SYS),  // sys.flags.isolated
@@ -339,6 +341,14 @@ The following implementation-specific options are available:\n\
          PYTHON_TLBC\n"
 #endif
 "\
+-X thread_inherit_context=[0|1]: enable (1) or disable (0) threads inheriting\n\
+         context vars by default; enabled by default in the free-threaded\n\
+         build and disabled otherwise; also PYTHON_THREAD_INHERIT_CONTEXT\n\
+-X context_aware_warnings=[0|1]: if true (1) then the warnings module will\n\
+         use a context variables; if false (0) then the warnings module will\n\
+         use module globals, which is not concurrent-safe; set to true for\n\
+         free-threaded builds and false otherwise; also\n\
+         PYTHON_CONTEXT_AWARE_WARNINGS\n\
 -X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n \
          of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\
 -X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\
@@ -426,6 +436,10 @@ static const char usage_envvars[] =
 #ifdef Py_GIL_DISABLED
 "PYTHON_TLBC     : when set to 0, disables thread-local bytecode (-X tlbc)\n"
 #endif
+"PYTHON_THREAD_INHERIT_CONTEXT: if true (1), threads inherit context vars\n"
+"                   (-X thread_inherit_context)\n"
+"PYTHON_CONTEXT_AWARE_WARNINGS: if true (1), enable thread-safe warnings module\n"
+"                   behaviour (-X context_aware_warnings)\n"
 "PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n"
 "PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n"
 "PYTHONUTF8      : control the UTF-8 mode (-X utf8)\n"
@@ -923,6 +937,8 @@ config_check_consistency(const PyConfig *config)
     assert(config->cpu_count != 0);
     // config->use_frozen_modules is initialized later
     // by _PyConfig_InitImportConfig().
+    assert(config->thread_inherit_context >= 0);
+    assert(config->context_aware_warnings >= 0);
 #ifdef __APPLE__
     assert(config->use_system_logger >= 0);
 #endif
@@ -1029,6 +1045,13 @@ _PyConfig_InitCompatConfig(PyConfig *config)
     config->_is_python_build = 0;
     config->code_debug_ranges = 1;
     config->cpu_count = -1;
+#ifdef Py_GIL_DISABLED
+    config->thread_inherit_context = 1;
+    config->context_aware_warnings = 1;
+#else
+    config->thread_inherit_context = 0;
+    config->context_aware_warnings = 0;
+#endif
 #ifdef __APPLE__
     config->use_system_logger = USE_SYSTEM_LOGGER_DEFAULT;
 #endif
@@ -1061,6 +1084,13 @@ config_init_defaults(PyConfig *config)
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = 0;
 #endif
+#ifdef Py_GIL_DISABLED
+    config->thread_inherit_context = 1;
+    config->context_aware_warnings = 1;
+#else
+    config->thread_inherit_context = 0;
+    config->context_aware_warnings = 0;
+#endif
 #ifdef __APPLE__
     config->use_system_logger = USE_SYSTEM_LOGGER_DEFAULT;
 #endif
@@ -1095,6 +1125,11 @@ PyConfig_InitIsolatedConfig(PyConfig *config)
     config->int_max_str_digits = _PY_LONG_DEFAULT_MAX_STR_DIGITS;
     config->safe_path = 1;
     config->pathconfig_warnings = 0;
+#ifdef Py_GIL_DISABLED
+    config->thread_inherit_context = 1;
+#else
+    config->thread_inherit_context = 0;
+#endif
 #ifdef MS_WINDOWS
     config->legacy_windows_stdio = 0;
 #endif
@@ -1924,6 +1959,58 @@ error:
                          "n must be greater than 0");
 }
 
+static PyStatus
+config_init_thread_inherit_context(PyConfig *config)
+{
+    const char *env = config_get_env(config, "PYTHON_THREAD_INHERIT_CONTEXT");
+    if (env) {
+        int enabled;
+        if (_Py_str_to_int(env, &enabled) < 0 || (enabled < 0) || (enabled > 1)) {
+            return _PyStatus_ERR(
+                "PYTHON_THREAD_INHERIT_CONTEXT=N: N is missing or invalid");
+        }
+        config->thread_inherit_context = enabled;
+    }
+
+    const wchar_t *xoption = config_get_xoption(config, L"thread_inherit_context");
+    if (xoption) {
+        int enabled;
+        const wchar_t *sep = wcschr(xoption, L'=');
+        if (!sep || (config_wstr_to_int(sep + 1, &enabled) < 0) || (enabled < 0) || (enabled > 1)) {
+            return _PyStatus_ERR(
+                "-X thread_inherit_context=n: n is missing or invalid");
+        }
+        config->thread_inherit_context = enabled;
+    }
+    return _PyStatus_OK();
+}
+
+static PyStatus
+config_init_context_aware_warnings(PyConfig *config)
+{
+    const char *env = config_get_env(config, "PYTHON_CONTEXT_AWARE_WARNINGS");
+    if (env) {
+        int enabled;
+        if (_Py_str_to_int(env, &enabled) < 0 || (enabled < 0) || (enabled > 1)) {
+            return _PyStatus_ERR(
+                "PYTHON_CONTEXT_AWARE_WARNINGS=N: N is missing or invalid");
+        }
+        config->context_aware_warnings = enabled;
+    }
+
+    const wchar_t *xoption = config_get_xoption(config, L"context_aware_warnings");
+    if (xoption) {
+        int enabled;
+        const wchar_t *sep = wcschr(xoption, L'=');
+        if (!sep || (config_wstr_to_int(sep + 1, &enabled) < 0) || (enabled < 0) || (enabled > 1)) {
+            return _PyStatus_ERR(
+                "-X context_aware_warnings=n: n is missing or invalid");
+        }
+        config->context_aware_warnings = enabled;
+    }
+    return _PyStatus_OK();
+}
+
 static PyStatus
 config_init_tlbc(PyConfig *config)
 {
@@ -2232,6 +2319,16 @@ config_read_complex_options(PyConfig *config)
     }
 #endif
 
+    status = config_init_thread_inherit_context(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
+    status = config_init_context_aware_warnings(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     status = config_init_tlbc(config);
     if (_PyStatus_EXCEPTION(status)) {
         return status;

Python/stdlib_module_names.h

@@ -64,6 +64,7 @@ static const char* _Py_stdlib_module_names[] = {
 "_posixshmem",
 "_posixsubprocess",
 "_py_abc",
+"_py_warnings",
 "_pydatetime",
 "_pydecimal",
 "_pyio",

Python/sysmodule.c

@@ -3335,6 +3335,8 @@ static PyStructSequence_Field flags_fields[] = {
     {"safe_path", "-P"},
     {"int_max_str_digits",      "-X int_max_str_digits"},
     {"gil",                     "-X gil"},
+    {"thread_inherit_context",  "-X thread_inherit_context"},
+    {"context_aware_warnings",    "-X context_aware_warnings"},
     {0}
 };
 
@@ -3435,6 +3437,8 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags)
 #else
     SetFlagObj(PyLong_FromLong(1));
 #endif
+    SetFlag(config->thread_inherit_context);
+    SetFlag(config->context_aware_warnings);
 #undef SetFlagObj
 #undef SetFlag
     return 0;