GH-96177: Move GIL and eval breaker code out of ceval.c into ceval_gil.c. (GH-96204)

This commit is contained in:
Mark Shannon 2022-08-24 14:21:01 +01:00 committed by GitHub
parent 4de06e3cc0
commit a4a9f2e879
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 1005 additions and 985 deletions

View file

@ -13,13 +13,11 @@
#include "pycore_ceval.h" // _PyEval_SignalAsyncExc()
#include "pycore_code.h"
#include "pycore_function.h"
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_opcode.h" // EXTRA_CASES
#include "pycore_pyerrors.h" // _PyErr_Fetch()
#include "pycore_pylifecycle.h" // _PyErr_Print()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_range.h" // _PyRangeIterObject
@ -237,582 +235,9 @@ is_tstate_valid(PyThreadState *tstate)
#endif
/* This can set eval_breaker to 0 even though gil_drop_request became
1. We believe this is all right because the eval loop will release
the GIL eventually anyway. */
static inline void
COMPUTE_EVAL_BREAKER(PyInterpreterState *interp,
struct _ceval_runtime_state *ceval,
struct _ceval_state *ceval2)
{
_Py_atomic_store_relaxed(&ceval2->eval_breaker,
_Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request)
| (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)
&& _Py_ThreadCanHandleSignals(interp))
| (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)
&& _Py_ThreadCanHandlePendingCalls())
| ceval2->pending.async_exc);
}
static inline void
SET_GIL_DROP_REQUEST(PyInterpreterState *interp)
{
struct _ceval_state *ceval2 = &interp->ceval;
_Py_atomic_store_relaxed(&ceval2->gil_drop_request, 1);
_Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
}
static inline void
RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)
{
struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
struct _ceval_state *ceval2 = &interp->ceval;
_Py_atomic_store_relaxed(&ceval2->gil_drop_request, 0);
COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}
static inline void
SIGNAL_PENDING_CALLS(PyInterpreterState *interp)
{
struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
struct _ceval_state *ceval2 = &interp->ceval;
_Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 1);
COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}
static inline void
UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)
{
struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
struct _ceval_state *ceval2 = &interp->ceval;
_Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0);
COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}
static inline void
SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp, int force)
{
struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
struct _ceval_state *ceval2 = &interp->ceval;
_Py_atomic_store_relaxed(&ceval->signals_pending, 1);
if (force) {
_Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
}
else {
/* eval_breaker is not set to 1 if thread_can_handle_signals() is false */
COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}
}
static inline void
UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp)
{
struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
struct _ceval_state *ceval2 = &interp->ceval;
_Py_atomic_store_relaxed(&ceval->signals_pending, 0);
COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}
static inline void
SIGNAL_ASYNC_EXC(PyInterpreterState *interp)
{
struct _ceval_state *ceval2 = &interp->ceval;
ceval2->pending.async_exc = 1;
_Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
}
static inline void
UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp)
{
struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
struct _ceval_state *ceval2 = &interp->ceval;
ceval2->pending.async_exc = 0;
COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}
#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#include "ceval_gil.h"
void _Py_NO_RETURN
_Py_FatalError_TstateNULL(const char *func)
{
_Py_FatalErrorFunc(func,
"the function must be called with the GIL held, "
"after Python initialization and before Python finalization, "
"but the GIL is released (the current Python thread state is NULL)");
}
int
_PyEval_ThreadsInitialized(_PyRuntimeState *runtime)
{
return gil_created(&runtime->ceval.gil);
}
int
PyEval_ThreadsInitialized(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
return _PyEval_ThreadsInitialized(runtime);
}
PyStatus
_PyEval_InitGIL(PyThreadState *tstate)
{
if (!_Py_IsMainInterpreter(tstate->interp)) {
/* Currently, the GIL is shared by all interpreters,
and only the main interpreter is responsible to create
and destroy it. */
return _PyStatus_OK();
}
struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
assert(!gil_created(gil));
PyThread_init_thread();
create_gil(gil);
take_gil(tstate);
assert(gil_created(gil));
return _PyStatus_OK();
}
void
_PyEval_FiniGIL(PyInterpreterState *interp)
{
if (!_Py_IsMainInterpreter(interp)) {
/* Currently, the GIL is shared by all interpreters,
and only the main interpreter is responsible to create
and destroy it. */
return;
}
struct _gil_runtime_state *gil = &interp->runtime->ceval.gil;
if (!gil_created(gil)) {
/* First Py_InitializeFromConfig() call: the GIL doesn't exist
yet: do nothing. */
return;
}
destroy_gil(gil);
assert(!gil_created(gil));
}
void
PyEval_InitThreads(void)
{
/* Do nothing: kept for backward compatibility */
}
void
_PyEval_Fini(void)
{
#ifdef Py_STATS
_Py_PrintSpecializationStats(1);
#endif
}
void
PyEval_AcquireLock(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
_Py_EnsureTstateNotNULL(tstate);
take_gil(tstate);
}
void
PyEval_ReleaseLock(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
/* This function must succeed when the current thread state is NULL.
We therefore avoid PyThreadState_Get() which dumps a fatal error
in debug mode. */
struct _ceval_runtime_state *ceval = &runtime->ceval;
struct _ceval_state *ceval2 = &tstate->interp->ceval;
drop_gil(ceval, ceval2, tstate);
}
void
_PyEval_ReleaseLock(PyThreadState *tstate)
{
struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval;
struct _ceval_state *ceval2 = &tstate->interp->ceval;
drop_gil(ceval, ceval2, tstate);
}
void
PyEval_AcquireThread(PyThreadState *tstate)
{
_Py_EnsureTstateNotNULL(tstate);
take_gil(tstate);
struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
if (_PyThreadState_Swap(gilstate, tstate) != NULL) {
Py_FatalError("non-NULL old thread state");
}
}
void
PyEval_ReleaseThread(PyThreadState *tstate)
{
assert(is_tstate_valid(tstate));
_PyRuntimeState *runtime = tstate->interp->runtime;
PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
if (new_tstate != tstate) {
Py_FatalError("wrong thread state");
}
struct _ceval_runtime_state *ceval = &runtime->ceval;
struct _ceval_state *ceval2 = &tstate->interp->ceval;
drop_gil(ceval, ceval2, tstate);
}
#ifdef HAVE_FORK
/* This function is called from PyOS_AfterFork_Child to destroy all threads
which are not running in the child process, and clear internal locks
which might be held by those threads. */
PyStatus
_PyEval_ReInitThreads(PyThreadState *tstate)
{
_PyRuntimeState *runtime = tstate->interp->runtime;
struct _gil_runtime_state *gil = &runtime->ceval.gil;
if (!gil_created(gil)) {
return _PyStatus_OK();
}
recreate_gil(gil);
take_gil(tstate);
struct _pending_calls *pending = &tstate->interp->ceval.pending;
if (_PyThread_at_fork_reinit(&pending->lock) < 0) {
return _PyStatus_ERR("Can't reinitialize pending calls lock");
}
/* Destroy all threads except the current one */
_PyThreadState_DeleteExcept(runtime, tstate);
return _PyStatus_OK();
}
#endif
/* This function is used to signal that async exceptions are waiting to be
raised. */
void
_PyEval_SignalAsyncExc(PyInterpreterState *interp)
{
SIGNAL_ASYNC_EXC(interp);
}
PyThreadState *
PyEval_SaveThread(void)
{
_PyRuntimeState *runtime = &_PyRuntime;
PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
_Py_EnsureTstateNotNULL(tstate);
struct _ceval_runtime_state *ceval = &runtime->ceval;
struct _ceval_state *ceval2 = &tstate->interp->ceval;
assert(gil_created(&ceval->gil));
drop_gil(ceval, ceval2, tstate);
return tstate;
}
void
PyEval_RestoreThread(PyThreadState *tstate)
{
_Py_EnsureTstateNotNULL(tstate);
take_gil(tstate);
struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
_PyThreadState_Swap(gilstate, tstate);
}
/* Mechanism whereby asynchronously executing callbacks (e.g. UNIX
signal handlers or Mac I/O completion routines) can schedule calls
to a function to be called synchronously.
The synchronous function is called with one void* argument.
It should return 0 for success or -1 for failure -- failure should
be accompanied by an exception.
If registry succeeds, the registry function returns 0; if it fails
(e.g. due to too many pending calls) it returns -1 (without setting
an exception condition).
Note that because registry may occur from within signal handlers,
or other asynchronous events, calling malloc() is unsafe!
Any thread can schedule pending calls, but only the main thread
will execute them.
There is no facility to schedule calls to a particular thread, but
that should be easy to change, should that ever be required. In
that case, the static variables here should go into the python
threadstate.
*/
void
_PyEval_SignalReceived(PyInterpreterState *interp)
{
#ifdef MS_WINDOWS
// bpo-42296: On Windows, _PyEval_SignalReceived() is called from a signal
// handler which can run in a thread different than the Python thread, in
// which case _Py_ThreadCanHandleSignals() is wrong. Ignore
// _Py_ThreadCanHandleSignals() and always set eval_breaker to 1.
//
// The next eval_frame_handle_pending() call will call
// _Py_ThreadCanHandleSignals() to recompute eval_breaker.
int force = 1;
#else
int force = 0;
#endif
/* bpo-30703: Function called when the C signal handler of Python gets a
signal. We cannot queue a callback using _PyEval_AddPendingCall() since
that function is not async-signal-safe. */
SIGNAL_PENDING_SIGNALS(interp, force);
}
/* Push one item onto the queue while holding the lock. */
static int
_push_pending_call(struct _pending_calls *pending,
int (*func)(void *), void *arg)
{
int i = pending->last;
int j = (i + 1) % NPENDINGCALLS;
if (j == pending->first) {
return -1; /* Queue full */
}
pending->calls[i].func = func;
pending->calls[i].arg = arg;
pending->last = j;
return 0;
}
/* Pop one item off the queue while holding the lock. */
static void
_pop_pending_call(struct _pending_calls *pending,
int (**func)(void *), void **arg)
{
int i = pending->first;
if (i == pending->last) {
return; /* Queue empty */
}
*func = pending->calls[i].func;
*arg = pending->calls[i].arg;
pending->first = (i + 1) % NPENDINGCALLS;
}
/* This implementation is thread-safe. It allows
scheduling to be made from any thread, and even from an executing
callback.
*/
int
_PyEval_AddPendingCall(PyInterpreterState *interp,
int (*func)(void *), void *arg)
{
struct _pending_calls *pending = &interp->ceval.pending;
/* Ensure that _PyEval_InitState() was called
and that _PyEval_FiniState() is not called yet. */
assert(pending->lock != NULL);
PyThread_acquire_lock(pending->lock, WAIT_LOCK);
int result = _push_pending_call(pending, func, arg);
PyThread_release_lock(pending->lock);
/* signal main loop */
SIGNAL_PENDING_CALLS(interp);
return result;
}
int
Py_AddPendingCall(int (*func)(void *), void *arg)
{
/* Best-effort to support subinterpreters and calls with the GIL released.
First attempt _PyThreadState_GET() since it supports subinterpreters.
If the GIL is released, _PyThreadState_GET() returns NULL . In this
case, use PyGILState_GetThisThreadState() which works even if the GIL
is released.
Sadly, PyGILState_GetThisThreadState() doesn't support subinterpreters:
see bpo-10915 and bpo-15751.
Py_AddPendingCall() doesn't require the caller to hold the GIL. */
PyThreadState *tstate = _PyThreadState_GET();
if (tstate == NULL) {
tstate = PyGILState_GetThisThreadState();
}
PyInterpreterState *interp;
if (tstate != NULL) {
interp = tstate->interp;
}
else {
/* Last resort: use the main interpreter */
interp = _PyInterpreterState_Main();
}
return _PyEval_AddPendingCall(interp, func, arg);
}
static int
handle_signals(PyThreadState *tstate)
{
assert(is_tstate_valid(tstate));
if (!_Py_ThreadCanHandleSignals(tstate->interp)) {
return 0;
}
UNSIGNAL_PENDING_SIGNALS(tstate->interp);
if (_PyErr_CheckSignalsTstate(tstate) < 0) {
/* On failure, re-schedule a call to handle_signals(). */
SIGNAL_PENDING_SIGNALS(tstate->interp, 0);
return -1;
}
return 0;
}
static int
make_pending_calls(PyInterpreterState *interp)
{
/* only execute pending calls on main thread */
if (!_Py_ThreadCanHandlePendingCalls()) {
return 0;
}
/* don't perform recursive pending calls */
static int busy = 0;
if (busy) {
return 0;
}
busy = 1;
/* unsignal before starting to call callbacks, so that any callback
added in-between re-signals */
UNSIGNAL_PENDING_CALLS(interp);
int res = 0;
/* perform a bounded number of calls, in case of recursion */
struct _pending_calls *pending = &interp->ceval.pending;
for (int i=0; i<NPENDINGCALLS; i++) {
int (*func)(void *) = NULL;
void *arg = NULL;
/* pop one item off the queue while holding the lock */
PyThread_acquire_lock(pending->lock, WAIT_LOCK);
_pop_pending_call(pending, &func, &arg);
PyThread_release_lock(pending->lock);
/* having released the lock, perform the callback */
if (func == NULL) {
break;
}
res = func(arg);
if (res) {
goto error;
}
}
busy = 0;
return res;
error:
busy = 0;
SIGNAL_PENDING_CALLS(interp);
return res;
}
void
_Py_FinishPendingCalls(PyThreadState *tstate)
{
assert(PyGILState_Check());
assert(is_tstate_valid(tstate));
struct _pending_calls *pending = &tstate->interp->ceval.pending;
if (!_Py_atomic_load_relaxed_int32(&(pending->calls_to_do))) {
return;
}
if (make_pending_calls(tstate->interp) < 0) {
PyObject *exc, *val, *tb;
_PyErr_Fetch(tstate, &exc, &val, &tb);
PyErr_BadInternalCall();
_PyErr_ChainExceptions(exc, val, tb);
_PyErr_Print(tstate);
}
}
/* Py_MakePendingCalls() is a simple wrapper for the sake
of backward-compatibility. */
int
Py_MakePendingCalls(void)
{
assert(PyGILState_Check());
PyThreadState *tstate = _PyThreadState_GET();
assert(is_tstate_valid(tstate));
/* Python signal handler doesn't really queue a callback: it only signals
that a signal was received, see _PyEval_SignalReceived(). */
int res = handle_signals(tstate);
if (res != 0) {
return res;
}
res = make_pending_calls(tstate->interp);
if (res != 0) {
return res;
}
return 0;
}
/* The interpreter's recursion limit */
void
_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)
{
_gil_initialize(&ceval->gil);
}
void
_PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock)
{
struct _pending_calls *pending = &ceval->pending;
assert(pending->lock == NULL);
pending->lock = pending_lock;
}
void
_PyEval_FiniState(struct _ceval_state *ceval)
{
struct _pending_calls *pending = &ceval->pending;
if (pending->lock != NULL) {
PyThread_free_lock(pending->lock);
pending->lock = NULL;
}
}
int
Py_GetRecursionLimit(void)
@ -1182,71 +607,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
}
/* Handle signals, pending calls, GIL drop request
and asynchronous exception */
static int
eval_frame_handle_pending(PyThreadState *tstate)
{
_PyRuntimeState * const runtime = &_PyRuntime;
struct _ceval_runtime_state *ceval = &runtime->ceval;
/* Pending signals */
if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) {
if (handle_signals(tstate) != 0) {
return -1;
}
}
/* Pending calls */
struct _ceval_state *ceval2 = &tstate->interp->ceval;
if (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) {
if (make_pending_calls(tstate->interp) != 0) {
return -1;
}
}
/* GIL drop request */
if (_Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request)) {
/* Give another thread a chance */
if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) {
Py_FatalError("tstate mix-up");
}
drop_gil(ceval, ceval2, tstate);
/* Other threads may run now */
take_gil(tstate);
if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {
Py_FatalError("orphan tstate");
}
}
/* Check for asynchronous exception. */
if (tstate->async_exc != NULL) {
PyObject *exc = tstate->async_exc;
tstate->async_exc = NULL;
UNSIGNAL_ASYNC_EXC(tstate->interp);
_PyErr_SetNone(tstate, exc);
Py_DECREF(exc);
return -1;
}
#ifdef MS_WINDOWS
// bpo-42296: On Windows, _PyEval_SignalReceived() can be called in a
// different thread than the Python thread, in which case
// _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the
// current Python thread with the correct _Py_ThreadCanHandleSignals()
// value. It prevents to interrupt the eval loop at every instruction if
// the current Python thread cannot handle signals (if
// _Py_ThreadCanHandleSignals() is false).
COMPUTE_EVAL_BREAKER(tstate->interp, ceval, ceval2);
#endif
return 0;
}
/* Computed GOTOs, or
the-optimization-commonly-but-improperly-known-as-"threaded code"
using gcc's labels-as-values extension
@ -1750,7 +1110,7 @@ handle_eval_breaker:
* All loops should include a check of the eval breaker.
* We also check on return from any builtin function.
*/
if (eval_frame_handle_pending(tstate) != 0) {
if (_Py_HandlePending(tstate) != 0) {
goto error;
}
DISPATCH();