mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-07 07:31:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

bpo-45256: Remove the usage of the C stack in Python to Python calls (GH-28488)

Browse source 
Ths commit inlines calls to Python functions in the eval loop and steals all the arguments in the call from the caller for
performance.
This commit is contained in:
Pablo Galindo Salgado 2021-10-09 16:51:30 +01:00 • committed by GitHub
parent ec04db74e2
commit b4903afd4d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 224 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

149
Python/ceval.c
View file

@ -98,6 +98,12 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg
static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);
static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int);
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static InterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFrameConstructor *con,
PyObject *locals, PyObject* const* args,
size_t argcount, PyObject *kwnames, int steal_args);
static int
_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame);
#define NAME_ERROR_MSG \
"name '%.200s' is not defined"
@ -1516,6 +1522,12 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
return 0;
}
static PyObject *
make_coro(PyThreadState *tstate, PyFrameConstructor *con,
PyObject *locals,
PyObject* const* args, size_t argcount,
PyObject *kwnames);
static int
skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
_Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code);
@ -1543,10 +1555,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
PyObject *retval = NULL; /* Return value */
_Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
if (_Py_EnterRecursiveCall(tstate, "")) {
return NULL;
}
CFrame cframe;
/* WARNING: Because the CFrame lives on the C stack,
@ -1558,9 +1566,18 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
cframe.previous = prev_cframe;
tstate->cframe = &cframe;
assert(frame->depth == 0);
/* push frame */
tstate->frame = frame;
start_frame:
if (_Py_EnterRecursiveCall(tstate, "")) {
tstate->recursion_depth++;
goto exit_eval_frame;
}
assert(frame == tstate->frame);
if (cframe.use_tracing) {
if (trace_function_entry(tstate, frame)) {
goto exit_eval_frame;
@ -1582,7 +1599,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
}
}
resume_frame:
co = frame->f_code;
PyObject *names = co->co_names;
PyObject *consts = co->co_consts;
_Py_CODEUNIT *first_instr = co->co_firstinstr;
@ -1594,12 +1612,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
multiple values.
When the PREDICT() macros are enabled, some opcode pairs follow in
direct succession without updating frame->f_lasti. A successful
prediction effectively links the two codes together as if they
were a single new opcode; accordingly,frame->f_lasti will point to
the first code in the pair (for instance, GET_ITER followed by
FOR_ITER is effectively a single opcode and frame->f_lasti will point
to the beginning of the combined pair.)
direct succession. A successful prediction effectively links the two
codes together as if they were a single new opcode, but the value
of frame->f_lasti is correctly updated so potential inlined calls
or lookups of frame->f_lasti are aways correct when the macros are used.
*/
assert(frame->f_lasti >= -1);
_Py_CODEUNIT *next_instr = first_instr + frame->f_lasti + 1;
@ -1625,6 +1641,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
#endif
if (throwflag) { /* support for generator.throw() */
throwflag = 0;
goto error;
}
@ -4591,10 +4608,44 @@ check_eval_breaker:
TARGET(CALL_FUNCTION) {
PREDICTED(CALL_FUNCTION);
PyObject **sp, *res;
sp = stack_pointer;
res = call_function(tstate, &sp, oparg, NULL, cframe.use_tracing);
stack_pointer = sp;
PyObject *res;
// Check if the call can be inlined or not
PyObject *function = PEEK(oparg + 1);
if (Py_TYPE(function) == &PyFunction_Type) {
PyCodeObject *code = (PyCodeObject*)PyFunction_GET_CODE(function);
PyObject *locals = code->co_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == 0) {
InterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, PyFunction_AS_FRAME_CONSTRUCTOR(function), locals, stack_pointer-oparg, oparg, NULL, 1);
if (new_frame == NULL) {
// When we exit here, we own all variables in the stack (the frame creation has not stolen
// any variable) so we need to clean the whole stack (done in the "error" label).
goto error;
}
STACK_SHRINK(oparg + 1);
assert(tstate->interp->eval_frame != NULL);
// The frame has stolen all the arguments from the stack, so there is no need to clean them up.```
Py_DECREF(function);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->depth = frame->depth + 1;
tstate->frame = frame = new_frame;
goto start_frame;
}
else {
/* Callable is a generator or coroutine function: create coroutine or generator. */
res = make_coro(tstate, PyFunction_AS_FRAME_CONSTRUCTOR(function), locals, stack_pointer-oparg, oparg, NULL);
STACK_SHRINK(oparg + 1);
for (int i = 0; i < oparg + 1; i++) {
Py_DECREF(stack_pointer[i]);
}
}
}
else {
PyObject **sp = stack_pointer;
res = call_function(tstate, &sp, oparg, NULL, cframe.use_tracing);
stack_pointer = sp;
}
PUSH(res);
if (res == NULL) {
goto error;
@ -5018,14 +5069,28 @@ exiting:
/* pop frame */
exit_eval_frame:
/* Restore previous cframe */
tstate->cframe = cframe.previous;
tstate->cframe->use_tracing = cframe.use_tracing;
if (PyDTrace_FUNCTION_RETURN_ENABLED())
dtrace_function_return(frame);
_Py_LeaveRecursiveCall(tstate);
if (frame->depth) {
_PyFrame_StackPush(frame->previous, retval);
if (_PyEvalFrameClearAndPop(tstate, frame)) {
retval = NULL;
}
frame = tstate->frame;
if (retval == NULL) {
assert(_PyErr_Occurred(tstate));
throwflag = 1;
}
retval = NULL;
goto resume_frame;
}
tstate->frame = frame->previous;
/* Restore previous cframe */
tstate->cframe = cframe.previous;
tstate->cframe->use_tracing = cframe.use_tracing;
return _Py_CheckFunctionResult(tstate, NULL, retval, __func__);
}
@ -5336,7 +5401,7 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i
static int
initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
PyObject **localsplus, PyObject *const *args,
Py_ssize_t argcount, PyObject *kwnames)
Py_ssize_t argcount, PyObject *kwnames, int steal_args)
{
PyCodeObject *co = (PyCodeObject*)con->fc_code;
const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
@ -5346,8 +5411,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
Py_ssize_t i;
if (co->co_flags & CO_VARKEYWORDS) {
kwdict = PyDict_New();
if (kwdict == NULL)
if (kwdict == NULL) {
goto fail;
}
i = total_args;
if (co->co_flags & CO_VARARGS) {
i++;
@ -5369,14 +5435,21 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
}
for (j = 0; j < n; j++) {
PyObject *x = args[j];
Py_INCREF(x);
if (!steal_args) {
Py_INCREF(x);
}
assert(localsplus[j] == NULL);
localsplus[j] = x;
}
/* Pack other positional arguments into the *args argument */
if (co->co_flags & CO_VARARGS) {
PyObject *u = _PyTuple_FromArray(args + n, argcount - n);
PyObject *u = NULL;
if (steal_args) {
u = _PyTuple_FromArraySteal(args + n, argcount - n);
} else {
u = _PyTuple_FromArray(args + n, argcount - n);
}
if (u == NULL) {
goto fail;
}
@ -5442,6 +5515,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
if (PyDict_SetItem(kwdict, keyword, value) == -1) {
goto fail;
}
if (steal_args) {
Py_DECREF(value);
}
continue;
kw_found:
@ -5451,7 +5527,9 @@ initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
con->fc_qualname, keyword);
goto fail;
}
Py_INCREF(value);
if (!steal_args) {
Py_INCREF(value);
}
localsplus[j] = value;
}
}
@ -5555,7 +5633,7 @@ make_coro_frame(PyThreadState *tstate,
}
_PyFrame_InitializeSpecials(frame, con, locals, code->co_nlocalsplus);
assert(frame->frame_obj == NULL);
if (initialize_locals(tstate, con, frame->localsplus, args, argcount, kwnames)) {
if (initialize_locals(tstate, con, frame->localsplus, args, argcount, kwnames, 0)) {
_PyFrame_Clear(frame, 1);
return NULL;
}
@ -5581,17 +5659,30 @@ make_coro(PyThreadState *tstate, PyFrameConstructor *con,
return gen;
}
// If *steal_args* is set, the function will steal the references to all the arguments.
// In case of error, the function returns null and if *steal_args* is set, the caller
// will still own all the arguments.
static InterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFrameConstructor *con,
PyObject *locals, PyObject* const* args,
size_t argcount, PyObject *kwnames)
size_t argcount, PyObject *kwnames, int steal_args)
{
InterpreterFrame * frame = _PyThreadState_PushFrame(tstate, con, locals);
if (frame == NULL) {
return NULL;
}
PyObject **localsarray = _PyFrame_GetLocalsArray(frame);
if (initialize_locals(tstate, con, localsarray, args, argcount, kwnames)) {
if (initialize_locals(tstate, con, localsarray, args, argcount, kwnames, steal_args)) {
if (steal_args) {
// If we failed to initialize locals, make sure the caller still own all the
// arguments. Notice that we only need to increase the reference count of the
// *valid* arguments (i.e. the ones that fit into the frame).
PyCodeObject *co = (PyCodeObject*)con->fc_code;
const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
for (Py_ssize_t i = 0; i < Py_MIN(argcount, total_args); i++) {
Py_XINCREF(frame->localsplus[i]);
}
}
_PyFrame_Clear(frame, 0);
return NULL;
}
@ -5606,9 +5697,9 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame)
++tstate->recursion_depth;
assert(frame->frame_obj == NULL || frame->frame_obj->f_own_locals_memory == 0);
if (_PyFrame_Clear(frame, 0)) {
--tstate->recursion_depth;
return -1;
}
assert(frame->frame_obj == NULL);
--tstate->recursion_depth;
tstate->frame = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
@ -5628,7 +5719,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con,
return make_coro(tstate, con, locals, args, argcount, kwnames);
}
InterpreterFrame *frame = _PyEvalFramePushAndInit(
tstate, con, locals, args, argcount, kwnames);
tstate, con, locals, args, argcount, kwnames, 0);
if (frame == NULL) {
return NULL;
}