mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-16 07:45:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-46329: Split calls into precall and call instructions. (GH-30855)

Browse source 
* Add PRECALL_FUNCTION opcode.

* Move 'call shape' varaibles into struct.

* Replace CALL_NO_KW and CALL_KW with KW_NAMES and CALL instructions.

* Specialize for builtin methods taking using the METH_FASTCALL | METH_KEYWORDS protocol.

* Allow kwnames for specialized calls to builtin types.

* Specialize calls to tuple(arg) and str(arg).
This commit is contained in:
Mark Shannon 2022-01-28 12:42:30 +00:00 • committed by GitHub
parent 5a9e423473
commit 89fd7c3452
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 957 additions and 674 deletions
Download patch file Download diff file Expand all files Collapse all files

521
Python/ceval.c
View file

@ -1636,6 +1636,17 @@ pop_frame(PyThreadState *tstate, InterpreterFrame *frame)
return prev_frame;
}
/* It is only between a PRECALL_METHOD/FUNCTION instruction and the following CALL,
* that these values have any meaning.
*/
typedef struct {
PyObject *callable;
PyObject *kwnames;
int total_args;
int postcall_shrink;
} CallShape;
PyObject* _Py_HOT_FUNCTION
_PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
{
@ -1654,22 +1665,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
_Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
CFrame cframe;
/* Variables used for making calls */
PyObject *kwnames;
int nargs;
/*
* It is only between a PRECALL_METHOD instruction and the following instruction,
* that these two values can be anything other than their defaults. */
int postcall_shrink = 1;
int extra_args = 0;
#define RESET_STACK_ADJUST_FOR_CALLS \
do { \
postcall_shrink = 1; \
extra_args = 0; \
} while (0)
#define STACK_ADJUST_IS_RESET \
(postcall_shrink == 1 && extra_args == 0)
CallShape call_shape;
call_shape.kwnames = NULL; // Borrowed reference
call_shape.postcall_shrink = 0;
call_shape.total_args = 0;
call_shape.callable = NULL; // Strong reference
/* WARNING: Because the CFrame lives on the C stack,
* but can be accessed from a heap allocated object (tstate)
@ -2532,12 +2532,12 @@ handle_eval_breaker:
PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
if (iter == NULL) {
int opcode_at_minus_3 = 0;
if ((next_instr - first_instr) > 2) {
opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]);
int opcode_at_minus_4 = 0;
if ((next_instr - first_instr) > 4) {
opcode_at_minus_4 = _Py_OPCODE(next_instr[-4]);
}
format_awaitable_error(tstate, Py_TYPE(iterable),
opcode_at_minus_3,
opcode_at_minus_4,
_Py_OPCODE(next_instr[-2]));
}
@ -4173,7 +4173,6 @@ handle_eval_breaker:
if (iter == NULL)
goto error;
PREDICT(FOR_ITER);
PREDICT(CALL_NO_KW);
DISPATCH();
}
@ -4494,6 +4493,16 @@ handle_eval_breaker:
NOTRACE_DISPATCH();
}
TARGET(PRECALL_FUNCTION) {
/* Move ownership of reference from stack to call_shape */
call_shape.callable = PEEK(oparg + 1);
call_shape.postcall_shrink = 1;
call_shape.total_args = oparg;
call_shape.kwnames = NULL;
DISPATCH();
}
TARGET(PRECALL_METHOD) {
/* Designed to work in tamdem with LOAD_METHOD. */
/* `meth` is NULL when LOAD_METHOD thinks that it's not
@ -4522,54 +4531,57 @@ handle_eval_breaker:
make it accept the `self` as a first argument.
*/
int is_method = (PEEK(oparg + 2) != NULL);
extra_args = is_method;
postcall_shrink = 2-is_method;
int nargs = oparg + is_method;
/* Move ownership of reference from stack to call_shape
* and make sure that NULL is cleared from stack */
call_shape.callable = PEEK(nargs + 1);
call_shape.postcall_shrink = 2-is_method;
call_shape.total_args = nargs;
call_shape.kwnames = NULL;
DISPATCH();
}
TARGET(CALL_KW) {
kwnames = POP();
oparg += extra_args;
extra_args = 0;
nargs = oparg - (int)PyTuple_GET_SIZE(kwnames);
goto call_function;
TARGET(KW_NAMES) {
assert(call_shape.kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts));
call_shape.kwnames = GETITEM(consts, oparg);
DISPATCH();
}
TARGET(CALL_NO_KW) {
TARGET(CALL) {
PREDICTED(CALL);
PyObject *function;
PREDICTED(CALL_NO_KW);
kwnames = NULL;
oparg += extra_args;
nargs = oparg;
assert((oparg == 0 && call_shape.kwnames == NULL)
|| (oparg != 0 && oparg == PyTuple_GET_SIZE(call_shape.kwnames)));
call_function:
function = PEEK(oparg + 1);
function = call_shape.callable;
if (Py_TYPE(function) == &PyMethod_Type) {
PyObject *meth = ((PyMethodObject *)function)->im_func;
PyObject *self = ((PyMethodObject *)function)->im_self;
Py_INCREF(meth);
Py_INCREF(self);
PEEK(oparg + 1) = self;
PEEK(call_shape.total_args + 1) = self;
Py_DECREF(function);
function = meth;
oparg++;
nargs++;
assert(postcall_shrink >= 1);
postcall_shrink--;
call_shape.total_args++;
assert(call_shape.postcall_shrink >= 1);
call_shape.postcall_shrink--;
}
int total_args = call_shape.total_args;
int positional_args = total_args - oparg;
// Check if the call can be inlined or not
if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) {
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
STACK_SHRINK(oparg);
STACK_SHRINK(total_args);
InterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)function, locals,
stack_pointer, nargs, kwnames
stack_pointer, positional_args, call_shape.kwnames
);
STACK_SHRINK(postcall_shrink);
RESET_STACK_ADJUST_FOR_CALLS;
STACK_SHRINK(call_shape.postcall_shrink);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
Py_XDECREF(kwnames);
Py_DECREF(function);
if (new_frame == NULL) {
goto error;
@ -4582,22 +4594,24 @@ handle_eval_breaker:
/* Callable is not a normal Python function */
PyObject *res;
if (cframe.use_tracing) {
res = trace_call_function(tstate, function, stack_pointer-oparg, nargs, kwnames);
res = trace_call_function(
tstate, function, stack_pointer-total_args,
positional_args, call_shape.kwnames);
}
else {
res = PyObject_Vectorcall(function, stack_pointer-oparg,
nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
res = PyObject_Vectorcall(
function, stack_pointer-total_args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
call_shape.kwnames);
}
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(function);
Py_XDECREF(kwnames);
/* Clear the stack */
STACK_SHRINK(oparg);
for (int i = 0; i < oparg; i++) {
STACK_SHRINK(total_args);
for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]);
}
STACK_SHRINK(postcall_shrink);
RESET_STACK_ADJUST_FOR_CALLS;
STACK_SHRINK(call_shape.postcall_shrink);
PUSH(res);
if (res == NULL) {
goto error;
@ -4606,65 +4620,87 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(CALL_NO_KW_ADAPTIVE) {
TARGET(CALL_ADAPTIVE) {
SpecializedCacheEntry *cache = GET_CACHE();
oparg = cache->adaptive.original_oparg;
int named_args = cache->adaptive.original_oparg;
assert((named_args == 0 && call_shape.kwnames == NULL)
|| (named_args != 0 && named_args == PyTuple_GET_SIZE(call_shape.kwnames)));
if (cache->adaptive.counter == 0) {
next_instr--;
int nargs = oparg+extra_args;
if (_Py_Specialize_CallNoKw(
PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) {
int nargs = call_shape.total_args;
int err = _Py_Specialize_CallNoKw(
call_shape.callable, next_instr, nargs,
call_shape.kwnames, cache, BUILTINS());
if (err < 0) {
goto error;
}
DISPATCH();
}
else {
STAT_INC(CALL_NO_KW, deferred);
STAT_INC(CALL, deferred);
cache->adaptive.counter--;
kwnames = NULL;
oparg += extra_args;
nargs = oparg;
oparg = named_args;
goto call_function;
}
}
TARGET(CALL_NO_KW_PY_SIMPLE) {
TARGET(CALL_PY_EXACT_ARGS) {
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
int argcount = cache0->original_oparg + extra_args;
DEOPT_IF(argcount != cache0->index, CALL_NO_KW);
int argcount = call_shape.total_args;
DEOPT_IF(!PyFunction_Check(call_shape.callable), CALL);
_PyCallCache *cache1 = &caches[-1].call;
PyObject *callable = PEEK(argcount+1);
DEOPT_IF(!PyFunction_Check(callable), CALL_NO_KW);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != cache1->func_version, CALL_NO_KW);
/* PEP 523 */
DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
PyFunctionObject *func = (PyFunctionObject *)call_shape.callable;
DEOPT_IF(func->func_version != cache1->func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
DEOPT_IF(code->co_argcount != argcount, CALL);
InterpreterFrame *new_frame = _PyFrame_Push(tstate, func);
if (new_frame == NULL) {
RESET_STACK_ADJUST_FOR_CALLS;
goto error;
}
_PyFrame_InitializeSpecials(new_frame, func,
NULL, code->co_nlocalsplus);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i];
}
int deflen = cache1->defaults_len;
for (int i = 0; i < deflen; i++) {
PyObject *def = PyTuple_GET_ITEM(func->func_defaults, cache1->defaults_start+i);
Py_INCREF(def);
new_frame->localsplus[argcount+i] = def;
}
for (int i = argcount+deflen; i < code->co_nlocalsplus; i++) {
for (int i = argcount; i < code->co_nlocalsplus; i++) {
new_frame->localsplus[i] = NULL;
}
STACK_SHRINK(postcall_shrink);
RESET_STACK_ADJUST_FOR_CALLS;
STACK_SHRINK(call_shape.postcall_shrink);
Py_DECREF(func);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
frame = cframe.current_frame = new_frame;
goto start_frame;
}
TARGET(CALL_PY_WITH_DEFAULTS) {
SpecializedCacheEntry *caches = GET_CACHE();
int argcount = call_shape.total_args;
DEOPT_IF(!PyFunction_Check(call_shape.callable), CALL);
_PyCallCache *cache1 = &caches[-1].call;
PyFunctionObject *func = (PyFunctionObject *)call_shape.callable;
DEOPT_IF(func->func_version != cache1->func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL);
int minargs = cache1->min_args;
DEOPT_IF(argcount < minargs, CALL);
InterpreterFrame *new_frame = _PyFrame_Push(tstate, func);
if (new_frame == NULL) {
goto error;
}
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i];
}
int def_offset = cache1->defaults_len - code->co_argcount;
for (int i = argcount; i < code->co_argcount; i++) {
PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i + def_offset);
Py_INCREF(def);
new_frame->localsplus[i] = def;
}
for (int i = code->co_argcount; i < code->co_nlocalsplus; i++) {
new_frame->localsplus[i] = NULL;
}
STACK_SHRINK(call_shape.postcall_shrink);
Py_DECREF(func);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
@ -4674,35 +4710,75 @@ handle_eval_breaker:
TARGET(CALL_NO_KW_TYPE_1) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
assert(GET_CACHE()->adaptive.original_oparg == 1);
DEOPT_IF(call_shape.total_args != 1, CALL);
assert(call_shape.kwnames == NULL);
PyObject *obj = TOP();
PyObject *callable = SECOND();
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL_NO_KW);
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
PyObject *res = Py_NewRef(Py_TYPE(obj));
STACK_SHRINK(1);
Py_DECREF(callable);
Py_DECREF(obj);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
NOTRACE_DISPATCH();
}
TARGET(CALL_NO_KW_BUILTIN_CLASS_1) {
TARGET(CALL_NO_KW_STR_1) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
assert(cache0->original_oparg == 1);
PyObject *callable = SECOND();
DEOPT_IF(!PyType_Check(call_shape.callable), CALL);
PyTypeObject *tp = (PyTypeObject *)call_shape.callable;
DEOPT_IF(call_shape.total_args != 1, CALL);
DEOPT_IF(tp != &PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
assert(call_shape.kwnames == NULL);
PyObject *arg = TOP();
DEOPT_IF(!PyType_Check(callable), CALL_NO_KW);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != cache0->version, CALL_NO_KW);
STACK_SHRINK(1);
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, 1, NULL);
SET_TOP(res);
Py_DECREF(tp);
PyObject *res = PyObject_Str(arg);
Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(CALL_NO_KW_TUPLE_1) {
DEOPT_IF(!PyType_Check(call_shape.callable), CALL);
PyTypeObject *tp = (PyTypeObject *)call_shape.callable;
DEOPT_IF(call_shape.total_args != 1, CALL);
DEOPT_IF(tp != &PyTuple_Type, CALL);
STAT_INC(CALL, hit);
assert(call_shape.kwnames == NULL);
PyObject *arg = TOP();
PyObject *res = PySequence_Tuple(arg);
Py_DECREF(arg);
Py_DECREF(&PyTuple_Type);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(CALL_BUILTIN_CLASS) {
DEOPT_IF(!PyType_Check(call_shape.callable), CALL);
PyTypeObject *tp = (PyTypeObject *)call_shape.callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
int kwnames_len = GET_CACHE()->adaptive.original_oparg;
int nargs = call_shape.total_args - kwnames_len;
STACK_SHRINK(call_shape.total_args);
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, nargs, call_shape.kwnames);
/* Free the arguments. */
for (int i = 0; i < call_shape.total_args; i++) {
Py_DECREF(stack_pointer[i]);
}
Py_DECREF(tp);
STACK_SHRINK(call_shape.postcall_shrink-1);
SET_TOP(res);
if (res == NULL) {
goto error;
}
@ -4711,13 +4787,13 @@ handle_eval_breaker:
TARGET(CALL_NO_KW_BUILTIN_O) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
/* Builtin METH_O functions */
PyObject *callable = SECOND();
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(call_shape.kwnames == NULL);
DEOPT_IF(call_shape.total_args != 1, CALL);
PyObject *callable = call_shape.callable;
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
// This is slower but CPython promises to check all non-vectorcall
@ -4725,14 +4801,14 @@ handle_eval_breaker:
if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
goto error;
}
PyObject *arg = POP();
PyObject *arg = TOP();
PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCall(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the function object. */
Py_DECREF(arg);
Py_DECREF(callable);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
if (res == NULL) {
goto error;
@ -4742,32 +4818,31 @@ handle_eval_breaker:
TARGET(CALL_NO_KW_BUILTIN_FAST) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
/* Builtin METH_FASTCALL functions, without keywords */
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
int nargs = cache0->original_oparg;
PyObject **pfunc = &PEEK(nargs + 1);
PyObject *callable = *pfunc;
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
assert(call_shape.kwnames == NULL);
PyObject *callable = call_shape.callable;
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
CALL);
STAT_INC(CALL, hit);
int nargs = call_shape.total_args;
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
STACK_SHRINK(nargs);
/* res = func(self, args, nargs) */
PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
&PEEK(nargs),
stack_pointer,
nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the function object. */
while (stack_pointer > pfunc) {
PyObject *x = POP();
Py_DECREF(x);
/* Free the arguments. */
for (int i = 0; i < nargs; i++) {
Py_DECREF(stack_pointer[i]);
}
STACK_SHRINK(call_shape.postcall_shrink);
PUSH(res);
Py_DECREF(callable);
if (res == NULL) {
/* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
@ -4779,29 +4854,72 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
PyObject *callable = call_shape.callable;
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
int kwnames_len = GET_CACHE()->adaptive.original_oparg;
assert(
(call_shape.kwnames == NULL && kwnames_len == 0) ||
(call_shape.kwnames != NULL &&
PyTuple_GET_SIZE(call_shape.kwnames) == kwnames_len)
);
int nargs = call_shape.total_args - kwnames_len;
STACK_SHRINK(call_shape.total_args);
/* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
PyObject *res = cfunc(
PyCFunction_GET_SELF(callable),
stack_pointer,
nargs,
call_shape.kwnames
);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < call_shape.total_args; i++) {
Py_DECREF(stack_pointer[i]);
}
STACK_SHRINK(call_shape.postcall_shrink);
PUSH(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(CALL_NO_KW_LEN) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
assert(call_shape.kwnames == NULL);
/* len(o) */
SpecializedCacheEntry *caches = GET_CACHE();
assert(caches[0].adaptive.original_oparg == 1);
DEOPT_IF(call_shape.total_args != 1, CALL);
assert(caches[0].adaptive.original_oparg == 0);
_PyObjectCache *cache1 = &caches[-1].obj;
PyObject *callable = SECOND();
DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
PyObject *callable = call_shape.callable;
DEOPT_IF(callable != cache1->obj, CALL);
STAT_INC(CALL, hit);
Py_ssize_t len_i = PyObject_Length(TOP());
PyObject *arg = TOP();
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
PyObject *res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the function object. */
Py_DECREF(POP());
Py_DECREF(callable);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(arg);
if (res == NULL) {
goto error;
}
@ -4810,28 +4928,30 @@ handle_eval_breaker:
TARGET(CALL_NO_KW_ISINSTANCE) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
assert(call_shape.kwnames == NULL);
/* isinstance(o, o2) */
SpecializedCacheEntry *caches = GET_CACHE();
assert(caches[0].adaptive.original_oparg == 2);
assert(caches[0].adaptive.original_oparg == 0);
DEOPT_IF(call_shape.total_args != 2, CALL);
_PyObjectCache *cache1 = &caches[-1].obj;
PyObject *callable = THIRD();
DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
DEOPT_IF(call_shape.callable != cache1->obj, CALL);
STAT_INC(CALL, hit);
int retval = PyObject_IsInstance(SECOND(), TOP());
PyObject *cls = POP();
PyObject *inst = TOP();
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
goto error;
}
PyObject *res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the function object. */
Py_DECREF(POP());
Py_DECREF(POP());
Py_DECREF(callable);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(call_shape.callable);
if (res == NULL) {
goto error;
}
@ -4840,57 +4960,78 @@ handle_eval_breaker:
TARGET(CALL_NO_KW_LIST_APPEND) {
assert(cframe.use_tracing == 0);
assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
assert(GET_CACHE()->adaptive.original_oparg == 1);
DEOPT_IF(extra_args == 0, CALL_NO_KW);
assert(call_shape.kwnames == NULL);
SpecializedCacheEntry *caches = GET_CACHE();
_PyObjectCache *cache1 = &caches[-1].obj;
DEOPT_IF(call_shape.total_args != 2, CALL);
DEOPT_IF(call_shape.callable != cache1->obj, CALL);
PyObject *list = SECOND();
DEOPT_IF(!PyList_CheckExact(list), CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(extra_args == 1);
extra_args = 0;
assert(STACK_ADJUST_IS_RESET);
DEOPT_IF(!PyList_Check(list), CALL);
STAT_INC(CALL, hit);
PyObject *arg = TOP();
int err = PyList_Append(list, arg);
if (err) {
goto error;
}
PyObject *callable = THIRD();
Py_DECREF(arg);
Py_DECREF(list);
STACK_SHRINK(call_shape.postcall_shrink+1);
Py_INCREF(Py_None);
STACK_SHRINK(2);
SET_TOP(Py_None);
Py_DECREF(callable);
Py_DECREF(call_shape.callable);
NOTRACE_DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
assert(GET_CACHE()->adaptive.original_oparg == 1);
DEOPT_IF(extra_args == 0, CALL_NO_KW);
assert(extra_args == 1);
PyObject *callable = THIRD();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW);
DEOPT_IF(((PyMethodDescrObject *)callable)->d_method->ml_flags != METH_O, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(extra_args == 1);
extra_args = 0;
assert(STACK_ADJUST_IS_RESET);
PyCFunction cfunc = ((PyMethodDescrObject *)callable)->d_method->ml_meth;
assert(call_shape.kwnames == NULL);
DEOPT_IF(call_shape.total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
goto error;
}
PyObject *arg = POP();
PyObject *self = POP();
PyObject *arg = TOP();
PyObject *self = SECOND();
PyObject *res = cfunc(self, arg);
_Py_LeaveRecursiveCall(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
STACK_SHRINK(call_shape.postcall_shrink+1);
SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(call_shape.callable);
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
assert(call_shape.kwnames == NULL);
DEOPT_IF(call_shape.total_args != 1, CALL);
DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method;
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
goto error;
}
PyObject *self = TOP();
PyObject *res = cfunc(self, NULL);
_Py_LeaveRecursiveCall(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
Py_DECREF(call_shape.callable);
if (res == NULL) {
goto error;
}
@ -4898,32 +5039,26 @@ handle_eval_breaker:
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
assert(call_shape.kwnames == NULL);
/* Builtin METH_FASTCALL methods, without keywords */
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
DEOPT_IF(extra_args == 0, CALL_NO_KW);
assert(extra_args == 1);
int nargs = cache0->original_oparg;
PyObject *callable = PEEK(nargs + 2);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW);
PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(extra_args == 1);
extra_args = 0;
assert(STACK_ADJUST_IS_RESET);
DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
_PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
PyObject *self = PEEK(nargs+1);
PyObject *res = cfunc(self, &PEEK(nargs), nargs);
int nargs = call_shape.total_args-1;
STACK_SHRINK(nargs);
PyObject *self = TOP();
PyObject *res = cfunc(self, stack_pointer, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
STACK_SHRINK(nargs+1);
for (int i = 0; i <= nargs; i++) {
for (int i = 0; i < nargs; i++) {
Py_DECREF(stack_pointer[i]);
}
Py_DECREF(self);
STACK_SHRINK(call_shape.postcall_shrink);
SET_TOP(res);
Py_DECREF(callable);
Py_DECREF(call_shape.callable);
if (res == NULL) {
goto error;
}
@ -5283,7 +5418,7 @@ MISS_WITH_CACHE(LOAD_ATTR)
MISS_WITH_CACHE(STORE_ATTR)
MISS_WITH_CACHE(LOAD_GLOBAL)
MISS_WITH_CACHE(LOAD_METHOD)
MISS_WITH_CACHE(CALL_NO_KW)
MISS_WITH_CACHE(CALL)
MISS_WITH_CACHE(BINARY_OP)
MISS_WITH_CACHE(COMPARE_OP)
MISS_WITH_CACHE(BINARY_SUBSCR)
@ -7321,7 +7456,7 @@ format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
}
static void
format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevopcode, int prevopcode)
format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevprevopcode, int prevopcode)
{
if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) {
if (prevopcode == BEFORE_ASYNC_WITH) {
@ -7330,7 +7465,7 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop
"that does not implement __await__: %.100s",
type->tp_name);
}
else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_NO_KW && prevprevopcode == LOAD_CONST)) {
else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL && prevprevprevopcode == LOAD_CONST)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async with' received an object from __aexit__ "
"that does not implement __await__: %.100s",

65
Python/compile.c
View file

@ -1021,11 +1021,14 @@ stack_effect(int opcode, int oparg, int jump)
/* Functions and calls */
case PRECALL_METHOD:
return -1;
case CALL_NO_KW:
return -oparg;
case CALL_KW:
return -oparg-1;
case PRECALL_FUNCTION:
return -oparg;
case KW_NAMES:
return 0;
case CALL:
return 0;
case CALL_FUNCTION_EX:
return -1 - ((oparg & 0x01) != 0);
case MAKE_FUNCTION:
@ -1823,7 +1826,8 @@ compiler_call_exit_with_nones(struct compiler *c) {
ADDOP_LOAD_CONST(c, Py_None);
ADDOP_LOAD_CONST(c, Py_None);
ADDOP_LOAD_CONST(c, Py_None);
ADDOP_I(c, CALL_NO_KW, 3);
ADDOP_I(c, PRECALL_FUNCTION, 3);
ADDOP_I(c, CALL, 0);
return 1;
}
@ -2208,7 +2212,8 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos)
int old_end_col_offset = c->u->u_end_col_offset;
for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) {
SET_LOC(c, (expr_ty)asdl_seq_GET(decos, i));
ADDOP_I(c, CALL_NO_KW, 1);
ADDOP_I(c, PRECALL_FUNCTION, 1);
ADDOP_I(c, CALL, 0);
}
c->u->u_lineno = old_lineno;
c->u->u_end_lineno = old_end_lineno;
@ -3903,7 +3908,8 @@ compiler_assert(struct compiler *c, stmt_ty s)
ADDOP(c, LOAD_ASSERTION_ERROR);
if (s->v.Assert.msg) {
VISIT(c, expr, s->v.Assert.msg);
ADDOP_I(c, CALL_NO_KW, 1);
ADDOP_I(c, PRECALL_FUNCTION, 1);
ADDOP_I(c, CALL, 0);
}
ADDOP_I(c, RAISE_VARARGS, 1);
compiler_use_next_block(c, end);
@ -4723,15 +4729,16 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e)
VISIT_SEQ(c, expr, e->v.Call.args);
if (kwdsl) {
VISIT_SEQ(c, keyword, kwds);
ADDOP_I(c, PRECALL_METHOD, argsl + kwdsl);
if (!compiler_call_simple_kw_helper(c, kwds, kwdsl)) {
return 0;
};
ADDOP_I(c, PRECALL_METHOD, argsl + kwdsl+1);
ADDOP_I(c, CALL_KW, argsl + kwdsl);
ADDOP_I(c, CALL, kwdsl);
}
else {
ADDOP_I(c, PRECALL_METHOD, argsl);
ADDOP_I(c, CALL_NO_KW, argsl);
ADDOP_I(c, CALL, 0);
}
c->u->u_lineno = old_lineno;
return 1;
@ -4799,7 +4806,7 @@ compiler_joined_str(struct compiler *c, expr_ty e)
ADDOP_I(c, LIST_APPEND, 1);
}
ADDOP_I(c, PRECALL_METHOD, 1);
ADDOP_I(c, CALL_NO_KW, 1);
ADDOP_I(c, CALL, 0);
}
else {
VISIT_SEQ(c, expr, e->v.JoinedStr.values);
@ -4900,21 +4907,15 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be
}
/* Used by compiler_call_helper and maybe_optimize_method_call to emit
LOAD_CONST kw1
LOAD_CONST kw2
...
LOAD_CONST <tuple of kwnames>
before a CALL_(FUNCTION|METHOD)_KW.
Returns 1 on success, 0 on error.
*/
* KW_NAMES before CALL.
* Returns 1 on success, 0 on error.
*/
static int
compiler_call_simple_kw_helper(struct compiler *c,
asdl_keyword_seq *keywords,
Py_ssize_t nkwelts)
{
PyObject *names;
VISIT_SEQ(c, keyword, keywords);
names = PyTuple_New(nkwelts);
if (names == NULL) {
return 0;
@ -4924,7 +4925,12 @@ compiler_call_simple_kw_helper(struct compiler *c,
Py_INCREF(kw->arg);
PyTuple_SET_ITEM(names, i, kw->arg);
}
ADDOP_LOAD_CONST_NEW(c, names);
Py_ssize_t arg = compiler_add_const(c, names);
if (arg < 0) {
return 0;
}
Py_DECREF(names);
ADDOP_I(c, KW_NAMES, arg);
return 1;
}
@ -4968,14 +4974,17 @@ compiler_call_helper(struct compiler *c,
VISIT(c, expr, elt);
}
if (nkwelts) {
VISIT_SEQ(c, keyword, keywords);
ADDOP_I(c, PRECALL_FUNCTION, n + nelts + nkwelts);
if (!compiler_call_simple_kw_helper(c, keywords, nkwelts)) {
return 0;
};
ADDOP_I(c, CALL_KW, n + nelts + nkwelts);
ADDOP_I(c, CALL, nkwelts);
return 1;
}
else {
ADDOP_I(c, CALL_NO_KW, n + nelts);
ADDOP_I(c, PRECALL_FUNCTION, n + nelts);
ADDOP_I(c, CALL, 0);
return 1;
}
@ -5372,7 +5381,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
ADDOP(c, GET_ITER);
}
ADDOP_I(c, CALL_NO_KW, 1);
ADDOP_I(c, PRECALL_FUNCTION, 1);
ADDOP_I(c, CALL, 0);
if (is_async_generator && type != COMP_GENEXP) {
ADDOP(c, GET_AWAITABLE);
@ -6709,7 +6719,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc)
// rotated = pc_stores[:rotations]
// del pc_stores[:rotations]
// pc_stores[icontrol-istores:icontrol-istores] = rotated
// Do the same thing to the stack, using several
// Do the same thing to the stack, using several
// rotations:
while (rotations--) {
if (!pattern_helper_rotate(c, icontrol + 1)){
@ -8786,6 +8796,8 @@ optimize_basic_block(struct compiler *c, basicblock *bb, PyObject *consts)
}
i += swaptimize(bb, i);
break;
case KW_NAMES:
break;
default:
/* All HAS_CONST opcodes should be handled with LOAD_CONST */
assert (!HAS_CONST(inst->i_opcode));
@ -9097,7 +9109,8 @@ trim_unused_consts(struct compiler *c, struct assembler *a, PyObject *consts)
int max_const_index = 0;
for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) {
for (int i = 0; i < b->b_iused; i++) {
if (b->b_instr[i].i_opcode == LOAD_CONST &&
if ((b->b_instr[i].i_opcode == LOAD_CONST ||
b->b_instr[i].i_opcode == KW_NAMES) &&
b->b_instr[i].i_oparg > max_const_index) {
max_const_index = b->b_instr[i].i_oparg;
}

15
Python/frame.c
View file

@ -83,7 +83,7 @@ take_ownership(PyFrameObject *f, InterpreterFrame *frame)
}
void
_PyFrame_Clear(InterpreterFrame * frame)
_PyFrame_Clear(InterpreterFrame *frame)
{
/* It is the responsibility of the owning generator/coroutine
* to have cleared the enclosing generator, if any. */
@ -107,3 +107,16 @@ _PyFrame_Clear(InterpreterFrame * frame)
Py_DECREF(frame->f_func);
Py_DECREF(frame->f_code);
}
InterpreterFrame *
_PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func)
{
PyCodeObject *code = (PyCodeObject *)func->func_code;
size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
if (new_frame == NULL) {
return NULL;
}
_PyFrame_InitializeSpecials(new_frame, func, NULL, code->co_nlocalsplus);
return new_frame;
}

62
Python/opcode_targets.h generated
View file

@ -28,59 +28,59 @@ static void *opcode_targets[256] = {
&&TARGET_STORE_SUBSCR_ADAPTIVE,
&&TARGET_STORE_SUBSCR_LIST_INT,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_CALL_NO_KW_ADAPTIVE,
&&TARGET_CALL_ADAPTIVE,
&&TARGET_GET_LEN,
&&TARGET_MATCH_MAPPING,
&&TARGET_MATCH_SEQUENCE,
&&TARGET_MATCH_KEYS,
&&TARGET_CALL_NO_KW_BUILTIN_O,
&&TARGET_CALL_BUILTIN_CLASS,
&&TARGET_PUSH_EXC_INFO,
&&TARGET_CALL_NO_KW_BUILTIN_O,
&&TARGET_CALL_NO_KW_BUILTIN_FAST,
&&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
&&TARGET_CALL_NO_KW_LEN,
&&TARGET_CALL_NO_KW_ISINSTANCE,
&&TARGET_CALL_NO_KW_PY_SIMPLE,
&&TARGET_CALL_PY_EXACT_ARGS,
&&TARGET_CALL_PY_WITH_DEFAULTS,
&&TARGET_CALL_NO_KW_LIST_APPEND,
&&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
&&TARGET_CALL_NO_KW_STR_1,
&&TARGET_CALL_NO_KW_TUPLE_1,
&&TARGET_CALL_NO_KW_TYPE_1,
&&TARGET_CALL_NO_KW_BUILTIN_CLASS_1,
&&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_JUMP_ABSOLUTE_QUICK,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_WITH_EXCEPT_START,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
&&TARGET_BEFORE_ASYNC_WITH,
&&TARGET_BEFORE_WITH,
&&TARGET_END_ASYNC_FOR,
&&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_JUMP_ABSOLUTE_QUICK,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
&&TARGET_LOAD_METHOD_ADAPTIVE,
&&TARGET_LOAD_METHOD_CACHED,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_LOAD_METHOD_MODULE,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_GET_ITER,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_PRINT_EXPR,
&&TARGET_LOAD_BUILD_CLASS,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_METHOD_CACHED,
&&TARGET_GET_AWAITABLE,
&&TARGET_LOAD_ASSERTION_ERROR,
&&TARGET_RETURN_GENERATOR,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_LOAD_METHOD_MODULE,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_STORE_ATTR_SLOT,
&&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
@ -130,7 +130,7 @@ static void *opcode_targets[256] = {
&&TARGET_POP_JUMP_IF_NOT_NONE,
&&TARGET_POP_JUMP_IF_NONE,
&&TARGET_RAISE_VARARGS,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_MAKE_FUNCTION,
&&TARGET_BUILD_SLICE,
&&TARGET_JUMP_NO_INTERRUPT,
@ -139,20 +139,20 @@ static void *opcode_targets[256] = {
&&TARGET_LOAD_DEREF,
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_CALL_FUNCTION_EX,
&&_unknown_opcode,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_EXTENDED_ARG,
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
&&TARGET_MAP_ADD,
&&TARGET_LOAD_CLASSDEREF,
&&TARGET_COPY_FREE_VARS,
&&_unknown_opcode,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_RESUME,
&&TARGET_MATCH_CLASS,
&&_unknown_opcode,
&&TARGET_STORE_FAST__STORE_FAST,
&&_unknown_opcode,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
@ -166,12 +166,12 @@ static void *opcode_targets[256] = {
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_PRECALL_FUNCTION,
&&TARGET_PRECALL_METHOD,
&&TARGET_CALL_NO_KW,
&&TARGET_CALL_KW,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,

300
Python/specialize.c
View file

@ -124,7 +124,7 @@ _Py_GetSpecializationStats(void) {
err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
err += add_stat_dict(stats, STORE_ATTR, "store_attr");
err += add_stat_dict(stats, CALL_NO_KW, "call_no_kw");
err += add_stat_dict(stats, CALL, "call");
err += add_stat_dict(stats, BINARY_OP, "binary_op");
err += add_stat_dict(stats, COMPARE_OP, "compare_op");
if (err < 0) {
@ -251,7 +251,7 @@ static uint8_t adaptive_opcodes[256] = {
[LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
[BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
[STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
[CALL_NO_KW] = CALL_NO_KW_ADAPTIVE,
[CALL] = CALL_ADAPTIVE,
[STORE_ATTR] = STORE_ATTR_ADAPTIVE,
[BINARY_OP] = BINARY_OP_ADAPTIVE,
[COMPARE_OP] = COMPARE_OP_ADAPTIVE,
@ -264,7 +264,7 @@ static uint8_t cache_requirements[256] = {
[LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
[BINARY_SUBSCR] = 2, /* _PyAdaptiveEntry, _PyObjectCache */
[STORE_SUBSCR] = 0,
[CALL_NO_KW] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
[BINARY_OP] = 1, // _PyAdaptiveEntry
[COMPARE_OP] = 1, /* _PyAdaptiveEntry */
@ -512,8 +512,13 @@ initial_counter_value(void) {
#define SPEC_FAIL_CLASS 18
#define SPEC_FAIL_PYTHON_CLASS 19
#define SPEC_FAIL_C_METHOD_CALL 20
#define SPEC_FAIL_METHDESCR_NON_METHOD 21
#define SPEC_FAIL_METHOD_CALL_CLASS 22
#define SPEC_FAIL_BOUND_METHOD 21
#define SPEC_FAIL_CALL_STR 22
#define SPEC_FAIL_CLASS_NO_VECTORCALL 23
#define SPEC_FAIL_CLASS_MUTABLE 24
#define SPEC_FAIL_KWNAMES 25
#define SPEC_FAIL_METHOD_WRAPPER 26
#define SPEC_FAIL_OPERATOR_WRAPPER 27
/* COMPARE_OP */
#define SPEC_FAIL_STRING_COMPARE 13
@ -1337,122 +1342,40 @@ success:
static int
specialize_class_call(
PyObject *callable, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache)
int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
{
PyTypeObject *tp = _PyType_CAST(callable);
if (_Py_OPCODE(instr[-1]) == PRECALL_METHOD) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_METHOD_CALL_CLASS);
return -1;
}
if (tp->tp_new == PyBaseObject_Type.tp_new) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_PYTHON_CLASS);
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_PYTHON_CLASS);
return -1;
}
if (nargs == 1) {
if (tp == &PyType_Type) {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_TYPE_1, _Py_OPARG(*instr));
return 0;
}
if ((tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) && tp->tp_vectorcall != NULL) {
cache->adaptive.version = tp->tp_version_tag;
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_CLASS_1, _Py_OPARG(*instr));
return 0;
}
}
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_CLASS);
return -1;
}
static PyMethodDescrObject *_list_append = NULL;
_Py_IDENTIFIER(append);
static int
specialize_method_descriptor(
PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache)
{
int oparg = cache->adaptive.original_oparg;
if (nargs - oparg != 1) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_METHDESCR_NON_METHOD);
return -1;
}
if (_list_append == NULL) {
_list_append = (PyMethodDescrObject *)_PyType_LookupId(&PyList_Type, &PyId_append);
}
if (oparg == 1 && descr == _list_append) {
assert(_Py_OPCODE(instr[-1]) == PRECALL_METHOD);
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr));
return 0;
}
switch (descr->d_method->ml_flags &
(METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
METH_KEYWORDS | METH_METHOD)) {
case METH_O: {
if (oparg != 1) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return 1;
if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
if (nargs == 1 && kwnames == NULL) {
if (tp == &PyUnicode_Type) {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_STR_1, _Py_OPARG(*instr));
return 0;
}
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL: {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
_Py_OPARG(*instr));
else if (tp == &PyType_Type) {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_TYPE_1, _Py_OPARG(*instr));
return 0;
}
else if (tp == &PyTuple_Type) {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_TUPLE_1, _Py_OPARG(*instr));
return 0;
}
}
if (tp->tp_vectorcall != NULL) {
*instr = _Py_MAKECODEUNIT(CALL_BUILTIN_CLASS, _Py_OPARG(*instr));
return 0;
}
SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ?
SPEC_FAIL_CALL_STR : SPEC_FAIL_CLASS_NO_VECTORCALL);
return -1;
}
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OTHER);
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CLASS_MUTABLE);
return -1;
}
static int
specialize_py_call(
PyFunctionObject *func, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache)
{
_PyCallCache *cache1 = &cache[-1].call;
PyCodeObject *code = (PyCodeObject *)func->func_code;
int kind = function_kind(code);
if (kind != SIMPLE_FUNCTION) {
SPECIALIZATION_FAIL(CALL_NO_KW, kind);
return -1;
}
int argcount = code->co_argcount;
if (argcount > 0xffff) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults);
assert(defcount <= argcount);
int min_args = argcount-defcount;
if (nargs > argcount || nargs < min_args) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return -1;
}
assert(nargs <= argcount && nargs >= min_args);
int defstart = nargs - min_args;
int deflen = argcount - nargs;
assert(defstart >= 0 && deflen >= 0);
assert(deflen == 0 || func->func_defaults != NULL);
if (defstart > 0xffff || deflen > 0xffff) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
int version = _PyFunction_GetVersionForCurrentState(func);
if (version == 0) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
cache[0].adaptive.index = nargs;
cache1->func_version = version;
cache1->defaults_start = defstart;
cache1->defaults_len = deflen;
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_PY_SIMPLE, _Py_OPARG(*instr));
return 0;
}
#ifdef Py_STATS
static int
builtin_call_fail_kind(int ml_flags)
@ -1477,15 +1400,118 @@ builtin_call_fail_kind(int ml_flags)
}
#endif
static PyMethodDescrObject *_list_append = NULL;
_Py_IDENTIFIER(append);
static int
specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
SpecializedCacheEntry *cache, PyObject *builtins)
specialize_method_descriptor(
PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
{
_PyObjectCache *cache1 = &cache[-1].obj;
if (_Py_OPCODE(instr[-1]) == PRECALL_METHOD) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_C_METHOD_CALL);
if (kwnames) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
return -1;
}
if (_list_append == NULL) {
_list_append = (PyMethodDescrObject *)_PyType_LookupId(&PyList_Type, &PyId_append);
}
assert(_list_append != NULL);
if (nargs == 2 && descr == _list_append) {
assert(_Py_OPCODE(instr[-1]) == PRECALL_METHOD);
cache[-1].obj.obj = (PyObject *)_list_append;
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr));
return 0;
}
switch (descr->d_method->ml_flags &
(METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
METH_KEYWORDS | METH_METHOD)) {
case METH_NOARGS: {
if (nargs != 1) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return -1;
}
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
_Py_OPARG(*instr));
return 0;
}
case METH_O: {
if (nargs != 2) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL: {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
_Py_OPARG(*instr));
return 0;
}
}
SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags));
return -1;
}
static int
specialize_py_call(
PyFunctionObject *func, _Py_CODEUNIT *instr,
int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
{
_PyCallCache *cache1 = &cache[-1].call;
PyCodeObject *code = (PyCodeObject *)func->func_code;
int kind = function_kind(code);
if (kwnames) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
return -1;
}
if (kind != SIMPLE_FUNCTION) {
SPECIALIZATION_FAIL(CALL, kind);
return -1;
}
int argcount = code->co_argcount;
if (argcount > 0xffff) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults);
assert(defcount <= argcount);
int min_args = argcount-defcount;
if (nargs > argcount || nargs < min_args) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return -1;
}
assert(nargs <= argcount && nargs >= min_args);
assert(min_args >= 0 && defcount >= 0);
assert(defcount == 0 || func->func_defaults != NULL);
if (min_args > 0xffff || defcount > 0xffff) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
int version = _PyFunction_GetVersionForCurrentState(func);
if (version == 0) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
cache[0].adaptive.index = nargs;
cache1->func_version = version;
cache1->min_args = min_args;
cache1->defaults_len = defcount;
if (argcount == nargs) {
*instr = _Py_MAKECODEUNIT(CALL_PY_EXACT_ARGS, _Py_OPARG(*instr));
}
else {
*instr = _Py_MAKECODEUNIT(CALL_PY_WITH_DEFAULTS, _Py_OPARG(*instr));
}
return 0;
}
static int
specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins)
{
_PyObjectCache *cache1 = &cache[-1].obj;
if (PyCFunction_GET_FUNCTION(callable) == NULL) {
return 1;
}
@ -1493,8 +1519,12 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
(METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
METH_KEYWORDS | METH_METHOD)) {
case METH_O: {
if (kwnames) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
return -1;
}
if (nargs != 1) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return 1;
}
/* len(o) */
@ -1510,6 +1540,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
return 0;
}
case METH_FASTCALL: {
if (kwnames) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
return -1;
}
if (nargs == 2) {
/* isinstance(o1, o2) */
PyObject *builtin_isinstance = PyDict_GetItemString(
@ -1525,8 +1559,13 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL | METH_KEYWORDS: {
*instr = _Py_MAKECODEUNIT(CALL_BUILTIN_FAST_WITH_KEYWORDS,
_Py_OPARG(*instr));
return 0;
}
default:
SPECIALIZATION_FAIL(CALL_NO_KW,
SPECIALIZATION_FAIL(CALL,
builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
return 1;
}
@ -1549,6 +1588,15 @@ call_fail_kind(PyObject *callable)
else if (PyType_Check(callable)) {
return SPEC_FAIL_CLASS;
}
else if (Py_TYPE(callable) == &PyWrapperDescr_Type) {
return SPEC_FAIL_OPERATOR_WRAPPER;
}
else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) {
return SPEC_FAIL_METHOD_WRAPPER;
}
else if (Py_TYPE(callable) == &PyMethod_Type) {
return SPEC_FAIL_BOUND_METHOD;
}
return SPEC_FAIL_OTHER;
}
#endif
@ -1559,35 +1607,35 @@ call_fail_kind(PyObject *callable)
int
_Py_Specialize_CallNoKw(
PyObject *callable, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache,
PyObject *builtins)
int nargs, PyObject *kwnames,
SpecializedCacheEntry *cache, PyObject *builtins)
{
_PyAdaptiveEntry *cache0 = &cache->adaptive;
int fail;
if (PyCFunction_CheckExact(callable)) {
fail = specialize_c_call(callable, instr, nargs, cache, builtins);
fail = specialize_c_call(callable, instr, nargs, kwnames, cache, builtins);
}
else if (PyFunction_Check(callable)) {
fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, cache);
fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, kwnames, cache);
}
else if (PyType_Check(callable)) {
fail = specialize_class_call(callable, instr, nargs, cache);
fail = specialize_class_call(callable, instr, nargs, kwnames, cache);
}
else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
fail = specialize_method_descriptor(
(PyMethodDescrObject *)callable, instr, nargs, cache);
(PyMethodDescrObject *)callable, instr, nargs, kwnames, cache);
}
else {
SPECIALIZATION_FAIL(CALL_NO_KW, call_fail_kind(callable));
SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
fail = -1;
}
_PyAdaptiveEntry *cache0 = &cache->adaptive;
if (fail) {
STAT_INC(CALL_NO_KW, failure);
STAT_INC(CALL, failure);
assert(!PyErr_Occurred());
cache_backoff(cache0);
}
else {
STAT_INC(CALL_NO_KW, success);
STAT_INC(CALL, success);
assert(!PyErr_Occurred());
cache0->counter = initial_counter_value();
}