mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-24 20:30:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

[3.14] gh-137109: refactor warning about threads when forking (GH-141438) (GH-141614)

Browse source 
This splits the OS API specific functionality to get the number of threads out
from the fallback Python method and warning raising code itself.  This way the
OS APIs can be queried before we've run
`os.register_at_fork(after_in_parent=...)` registered functions which
themselves may (re)start threads that would otherwise be detected.

This is best effort.  If the OS APIs are either unavailable or fail, the
warning generating code still falls back to looking at the Python threading
state after the CPython interpreter world has been restarted and the
after_in_parent calls have been made.  The common case for most Linux and macOS
environments should work today.

This also lines up with the existing TODO refactoring, we may choose to expose
this API to get the number of OS threads in the `os` module in the future.

Note: This is a simplified backport that maintains the void return type
for warn_about_fork_with_threads() and keeps PyErr_Clear() in the warning path,
as the error handling changes from fd8f42d3d1 are not needed in 3.14.
This commit is contained in:
Gregory P. Smith 2025-11-16 11:45:29 -08:00 • committed by GitHub
parent 58c9d25675
commit 0d8fb0b852
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 65 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst Normal file
View file

@ -0,0 +1,5 @@
The :mod:`os.fork` and related forking APIs will no longer warn in the
common case where Linux or macOS platform APIs return the number of threads
in a process and find the answer to be 1 even when a
:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a
thread.

103
Modules/posixmodule.c
View file

@ -7975,53 +7975,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before,
// running in the process. Best effort, silent if unable to count threads.
// Constraint: Quick. Never overcounts. Never leaves an error set.
//
// This should only be called from the parent process after
// This MUST only be called from the parent process after
// PyOS_AfterFork_Parent().
static void
warn_about_fork_with_threads(const char* name)
warn_about_fork_with_threads(
const char* name, // Name of the API to use in the warning message.
const Py_ssize_t num_os_threads // Only trusted when >= 1.
)
{
// It's not safe to issue the warning while the world is stopped, because
// other threads might be holding locks that we need, which would deadlock.
assert(!_PyRuntime.stoptheworld.world_stopped);
// TODO: Consider making an `os` module API to return the current number
// of threads in the process. That'd presumably use this platform code but
// raise an error rather than using the inaccurate fallback.
Py_ssize_t num_python_threads = 0;
#if defined(__APPLE__) && defined(HAVE_GETPID)
mach_port_t macos_self = mach_task_self();
mach_port_t macos_task;
if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) {
thread_array_t macos_threads;
mach_msg_type_number_t macos_n_threads;
if (task_threads(macos_task, &macos_threads,
&macos_n_threads) == KERN_SUCCESS) {
num_python_threads = macos_n_threads;
}
}
#elif defined(__linux__)
// Linux /proc/self/stat 20th field is the number of threads.
FILE* proc_stat = fopen("/proc/self/stat", "r");
if (proc_stat) {
size_t n;
// Size chosen arbitrarily. ~60% more bytes than a 20th column index
// observed on the author's workstation.
char stat_line[160];
n = fread(&stat_line, 1, 159, proc_stat);
stat_line[n] = '\0';
fclose(proc_stat);
char *saveptr = NULL;
char *field = strtok_r(stat_line, " ", &saveptr);
unsigned int idx;
for (idx = 19; idx && field; --idx) {
field = strtok_r(NULL, " ", &saveptr);
}
if (idx == 0 && field) { // found the 20th field
num_python_threads = atoi(field); // 0 on error
}
}
#endif
Py_ssize_t num_python_threads = num_os_threads;
if (num_python_threads <= 0) {
// Fall back to just the number our threading module knows about.
// An incomplete view of the world, but better than nothing.
@ -8074,6 +8040,51 @@ warn_about_fork_with_threads(const char* name)
PyErr_Clear();
}
}
// If this returns <= 0, we were unable to successfully use any OS APIs.
// Returns a positive number of threads otherwise.
static Py_ssize_t get_number_of_os_threads(void)
{
// TODO: Consider making an `os` module API to return the current number
// of threads in the process. That'd presumably use this platform code but
// raise an error rather than using the inaccurate fallback.
Py_ssize_t num_python_threads = 0;
#if defined(__APPLE__) && defined(HAVE_GETPID)
mach_port_t macos_self = mach_task_self();
mach_port_t macos_task;
if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) {
thread_array_t macos_threads;
mach_msg_type_number_t macos_n_threads;
if (task_threads(macos_task, &macos_threads,
&macos_n_threads) == KERN_SUCCESS) {
num_python_threads = macos_n_threads;
}
}
#elif defined(__linux__)
// Linux /proc/self/stat 20th field is the number of threads.
FILE* proc_stat = fopen("/proc/self/stat", "r");
if (proc_stat) {
size_t n;
// Size chosen arbitrarily. ~60% more bytes than a 20th column index
// observed on the author's workstation.
char stat_line[160];
n = fread(&stat_line, 1, 159, proc_stat);
stat_line[n] = '\0';
fclose(proc_stat);
char *saveptr = NULL;
char *field = strtok_r(stat_line, " ", &saveptr);
unsigned int idx;
for (idx = 19; idx && field; --idx) {
field = strtok_r(NULL, " ", &saveptr);
}
if (idx == 0 && field) { // found the 20th field
num_python_threads = atoi(field); // 0 on error
}
}
#endif
return num_python_threads;
}
#endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK
#ifdef HAVE_FORK1
@ -8108,10 +8119,12 @@ os_fork1_impl(PyObject *module)
/* child: this clobbers and resets the import lock. */
PyOS_AfterFork_Child();
} else {
// Called before AfterFork_Parent in case those hooks start threads.
Py_ssize_t num_os_threads = get_number_of_os_threads();
/* parent: release the import lock. */
PyOS_AfterFork_Parent();
// After PyOS_AfterFork_Parent() starts the world to avoid deadlock.
warn_about_fork_with_threads("fork1");
warn_about_fork_with_threads("fork1", num_os_threads);
}
if (pid == -1) {
errno = saved_errno;
@ -8157,10 +8170,12 @@ os_fork_impl(PyObject *module)
/* child: this clobbers and resets the import lock. */
PyOS_AfterFork_Child();
} else {
// Called before AfterFork_Parent in case those hooks start threads.
Py_ssize_t num_os_threads = get_number_of_os_threads();
/* parent: release the import lock. */
PyOS_AfterFork_Parent();
// After PyOS_AfterFork_Parent() starts the world to avoid deadlock.
warn_about_fork_with_threads("fork");
warn_about_fork_with_threads("fork", num_os_threads);
}
if (pid == -1) {
errno = saved_errno;
@ -9014,10 +9029,12 @@ os_forkpty_impl(PyObject *module)
/* child: this clobbers and resets the import lock. */
PyOS_AfterFork_Child();
} else {
// Called before AfterFork_Parent in case those hooks start threads.
Py_ssize_t num_os_threads = get_number_of_os_threads();
/* parent: release the import lock. */
PyOS_AfterFork_Parent();
// After PyOS_AfterFork_Parent() starts the world to avoid deadlock.
warn_about_fork_with_threads("forkpty");
warn_about_fork_with_threads("forkpty", num_os_threads);
}
if (pid == -1) {
return posix_error();