mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 05:58:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

bpo-44525: Split calls into PRECALL and CALL (GH-30011)

Browse source 
* Add 3 new opcodes for calls: PRECALL_METHOD, CALL_NO_KW, CALL_KW.

* Update specialization to handle new CALL opcodes.

* Specialize call to method descriptors.

* Remove old CALL opcodes: CALL_FUNCTION, CALL_METHOD, CALL_METHOD_KW, CALL_FUNCTION_KW.
This commit is contained in:
Mark Shannon 2021-12-14 18:22:44 +00:00 • committed by GitHub
parent d60457a667
commit 9f8f45144b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 395 additions and 236 deletions
Download patch file Download diff file Expand all files Collapse all files

216
Python/ceval.c
View file

@ -1673,6 +1673,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
CFrame cframe;
/* Variables used for making calls */
PyObject *kwnames;
int nargs;
/*
* It is only between a PRECALL_METHOD instruction and the following instruction,
* that these two values can be anything other than their defaults. */
int postcall_shrink = 1;
int extra_args = 0;
#define RESET_STACK_ADJUST_FOR_CALLS \
do { \
postcall_shrink = 1; \
extra_args = 0; \
} while (0)
#define STACK_ADJUST_IS_RESET \
(postcall_shrink == 1 && extra_args == 0)
/* WARNING: Because the CFrame lives on the C stack,
* but can be accessed from a heap allocated object (tstate)
* strict stack discipline must be maintained.
@ -1815,11 +1831,6 @@ check_eval_breaker:
switch (opcode) {
#endif
/* Variables used for making calls */
PyObject *kwnames;
int nargs;
int postcall_shrink;
/* BEWARE!
It is essential that any operation that fails must goto error
and that all operation that succeed call DISPATCH() ! */
@ -4319,7 +4330,7 @@ check_eval_breaker:
if (iter == NULL)
goto error;
PREDICT(FOR_ITER);
PREDICT(CALL_FUNCTION);
PREDICT(CALL_NO_KW);
DISPATCH();
}
@ -4661,7 +4672,7 @@ check_eval_breaker:
DISPATCH();
}
TARGET(CALL_METHOD) {
TARGET(PRECALL_METHOD) {
/* Designed to work in tamdem with LOAD_METHOD. */
/* `meth` is NULL when LOAD_METHOD thinks that it's not
a method call.
@ -4689,38 +4700,26 @@ check_eval_breaker:
make it accept the `self` as a first argument.
*/
int is_method = (PEEK(oparg + 2) != NULL);
oparg += is_method;
nargs = oparg;
kwnames = NULL;
extra_args = is_method;
postcall_shrink = 2-is_method;
goto call_function;
DISPATCH();
}
TARGET(CALL_METHOD_KW) {
/* Designed to work in tandem with LOAD_METHOD. Same as CALL_METHOD
but pops TOS to get a tuple of keyword names. */
TARGET(CALL_KW) {
kwnames = POP();
int is_method = (PEEK(oparg + 2) != NULL);
oparg += is_method;
oparg += extra_args;
extra_args = 0;
nargs = oparg - (int)PyTuple_GET_SIZE(kwnames);
postcall_shrink = 2-is_method;
goto call_function;
}
TARGET(CALL_FUNCTION_KW) {
kwnames = POP();
nargs = oparg - (int)PyTuple_GET_SIZE(kwnames);
postcall_shrink = 1;
goto call_function;
}
TARGET(CALL_FUNCTION) {
PREDICTED(CALL_FUNCTION);
STAT_INC(CALL_FUNCTION, unquickened);
TARGET(CALL_NO_KW) {
PyObject *function;
nargs = oparg;
PREDICTED(CALL_NO_KW);
STAT_INC(CALL_NO_KW, unquickened);
kwnames = NULL;
postcall_shrink = 1;
oparg += extra_args;
nargs = oparg;
call_function:
function = PEEK(oparg + 1);
if (Py_TYPE(function) == &PyMethod_Type) {
@ -4748,6 +4747,7 @@ check_eval_breaker:
stack_pointer, nargs, kwnames
);
STACK_SHRINK(postcall_shrink);
RESET_STACK_ADJUST_FOR_CALLS;
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
Py_XDECREF(kwnames);
@ -4780,6 +4780,7 @@ check_eval_breaker:
Py_DECREF(stack_pointer[i]);
}
STACK_SHRINK(postcall_shrink);
RESET_STACK_ADJUST_FOR_CALLS;
PUSH(res);
if (res == NULL) {
goto error;
@ -4788,43 +4789,46 @@ check_eval_breaker:
DISPATCH();
}
TARGET(CALL_FUNCTION_ADAPTIVE) {
TARGET(CALL_NO_KW_ADAPTIVE) {
SpecializedCacheEntry *cache = GET_CACHE();
nargs = cache->adaptive.original_oparg;
oparg = cache->adaptive.original_oparg;
if (cache->adaptive.counter == 0) {
next_instr--;
if (_Py_Specialize_CallFunction(
int nargs = oparg+extra_args;
if (_Py_Specialize_CallNoKw(
PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) {
goto error;
}
DISPATCH();
}
else {
STAT_INC(CALL_FUNCTION, deferred);
STAT_INC(CALL_NO_KW, deferred);
cache->adaptive.counter--;
oparg = nargs;
kwnames = NULL;
postcall_shrink = 1;
oparg += extra_args;
nargs = oparg;
goto call_function;
}
}
TARGET(CALL_FUNCTION_PY_SIMPLE) {
TARGET(CALL_NO_KW_PY_SIMPLE) {
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
int argcount = cache0->original_oparg;
int argcount = cache0->original_oparg + extra_args;
DEOPT_IF(argcount != cache0->index, CALL_NO_KW);
_PyCallCache *cache1 = &caches[-1].call;
PyObject *callable = PEEK(argcount+1);
DEOPT_IF(!PyFunction_Check(callable), CALL_FUNCTION);
DEOPT_IF(!PyFunction_Check(callable), CALL_NO_KW);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != cache1->func_version, CALL_FUNCTION);
DEOPT_IF(func->func_version != cache1->func_version, CALL_NO_KW);
/* PEP 523 */
DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_FUNCTION);
STAT_INC(CALL_FUNCTION, hit);
DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
PyCodeObject *code = (PyCodeObject *)func->func_code;
size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
if (new_frame == NULL) {
RESET_STACK_ADJUST_FOR_CALLS;
goto error;
}
_PyFrame_InitializeSpecials(new_frame, func,
@ -4842,7 +4846,8 @@ check_eval_breaker:
for (int i = argcount+deflen; i < code->co_nlocalsplus; i++) {
new_frame->localsplus[i] = NULL;
}
STACK_SHRINK(1);
STACK_SHRINK(postcall_shrink);
RESET_STACK_ADJUST_FOR_CALLS;
Py_DECREF(func);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
@ -4851,14 +4856,15 @@ check_eval_breaker:
goto start_frame;
}
TARGET(CALL_FUNCTION_BUILTIN_O) {
TARGET(CALL_NO_KW_BUILTIN_O) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
/* Builtin METH_O functions */
PyObject *callable = SECOND();
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_FUNCTION);
STAT_INC(CALL_FUNCTION, hit);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
// This is slower but CPython promises to check all non-vectorcall
@ -4881,18 +4887,19 @@ check_eval_breaker:
DISPATCH();
}
TARGET(CALL_FUNCTION_BUILTIN_FAST) {
TARGET(CALL_NO_KW_BUILTIN_FAST) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
/* Builtin METH_FASTCALL functions, without keywords */
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
int nargs = cache0->original_oparg;
PyObject **pfunc = &PEEK(nargs + 1);
PyObject *callable = *pfunc;
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
CALL_FUNCTION);
STAT_INC(CALL_FUNCTION, hit);
CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
/* res = func(self, args, nargs) */
@ -4919,16 +4926,17 @@ check_eval_breaker:
DISPATCH();
}
TARGET(CALL_FUNCTION_LEN) {
TARGET(CALL_NO_KW_LEN) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
/* len(o) */
SpecializedCacheEntry *caches = GET_CACHE();
assert(caches[0].adaptive.original_oparg == 1);
_PyObjectCache *cache1 = &caches[-1].obj;
PyObject *callable = SECOND();
DEOPT_IF(callable != cache1->obj, CALL_FUNCTION);
STAT_INC(CALL_FUNCTION, hit);
DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
Py_ssize_t len_i = PyObject_Length(TOP());
if (len_i < 0) {
@ -4947,16 +4955,17 @@ check_eval_breaker:
DISPATCH();
}
TARGET(CALL_FUNCTION_ISINSTANCE) {
TARGET(CALL_NO_KW_ISINSTANCE) {
assert(cframe.use_tracing == 0);
assert(STACK_ADJUST_IS_RESET);
/* isinstance(o, o2) */
SpecializedCacheEntry *caches = GET_CACHE();
assert(caches[0].adaptive.original_oparg == 2);
_PyObjectCache *cache1 = &caches[-1].obj;
PyObject *callable = THIRD();
DEOPT_IF(callable != cache1->obj, CALL_FUNCTION);
STAT_INC(CALL_FUNCTION, hit);
DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
int retval = PyObject_IsInstance(SECOND(), TOP());
if (retval < 0) {
@ -4976,6 +4985,97 @@ check_eval_breaker:
DISPATCH();
}
TARGET(CALL_NO_KW_LIST_APPEND) {
assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
assert(GET_CACHE()->adaptive.original_oparg == 1);
DEOPT_IF(extra_args == 0, CALL_NO_KW);
PyObject *list = SECOND();
DEOPT_IF(!PyList_CheckExact(list), CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(extra_args == 1);
extra_args = 0;
assert(STACK_ADJUST_IS_RESET);
PyObject *arg = TOP();
int err = PyList_Append(list, arg);
if (err) {
goto error;
}
PyObject *callable = THIRD();
Py_DECREF(arg);
Py_DECREF(list);
Py_INCREF(Py_None);
STACK_SHRINK(2);
SET_TOP(Py_None);
Py_DECREF(callable);
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
assert(GET_CACHE()->adaptive.original_oparg == 1);
DEOPT_IF(extra_args == 0, CALL_NO_KW);
assert(extra_args == 1);
PyObject *callable = THIRD();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW);
DEOPT_IF(((PyMethodDescrObject *)callable)->d_method->ml_flags != METH_O, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(extra_args == 1);
extra_args = 0;
assert(STACK_ADJUST_IS_RESET);
PyCFunction cfunc = ((PyMethodDescrObject *)callable)->d_method->ml_meth;
// This is slower but CPython promises to check all non-vectorcall
// function calls.
if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
goto error;
}
PyObject *arg = POP();
PyObject *self = POP();
PyObject *res = cfunc(self, arg);
_Py_LeaveRecursiveCall(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
/* Builtin METH_FASTCALL methods, without keywords */
SpecializedCacheEntry *caches = GET_CACHE();
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
DEOPT_IF(extra_args == 0, CALL_NO_KW);
assert(extra_args == 1);
int nargs = cache0->original_oparg;
PyObject *callable = PEEK(nargs + 2);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW);
PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL_NO_KW);
STAT_INC(CALL_NO_KW, hit);
assert(extra_args == 1);
extra_args = 0;
assert(STACK_ADJUST_IS_RESET);
_PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
PyObject *self = PEEK(nargs+1);
PyObject *res = cfunc(self, &PEEK(nargs), nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
STACK_SHRINK(nargs+1);
for (int i = 0; i <= nargs; i++) {
Py_DECREF(stack_pointer[i]);
}
SET_TOP(res);
Py_DECREF(callable);
if (res == NULL) {
goto error;
}
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
PREDICTED(CALL_FUNCTION_EX);
PyObject *func, *callargs, *kwargs = NULL, *result;
@ -5289,7 +5389,7 @@ MISS_WITH_CACHE(LOAD_ATTR)
MISS_WITH_CACHE(STORE_ATTR)
MISS_WITH_CACHE(LOAD_GLOBAL)
MISS_WITH_CACHE(LOAD_METHOD)
MISS_WITH_CACHE(CALL_FUNCTION)
MISS_WITH_CACHE(CALL_NO_KW)
MISS_WITH_CACHE(BINARY_OP)
MISS_WITH_CACHE(COMPARE_OP)
MISS_WITH_CACHE(BINARY_SUBSCR)
@ -7509,7 +7609,7 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop
"that does not implement __await__: %.100s",
type->tp_name);
}
else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_FUNCTION && prevprevopcode == DUP_TOP)) {
else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_NO_KW && prevprevopcode == DUP_TOP)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async with' received an object from __aexit__ "
"that does not implement __await__: %.100s",

31
Python/compile.c
View file

@ -1155,13 +1155,11 @@ stack_effect(int opcode, int oparg, int jump)
return -oparg;
/* Functions and calls */
case CALL_FUNCTION:
case PRECALL_METHOD:
return -1;
case CALL_NO_KW:
return -oparg;
case CALL_METHOD:
return -oparg-1;
case CALL_METHOD_KW:
return -oparg-2;
case CALL_FUNCTION_KW:
case CALL_KW:
return -oparg-1;
case CALL_FUNCTION_EX:
return -1 - ((oparg & 0x01) != 0);
@ -1817,7 +1815,7 @@ compiler_call_exit_with_nones(struct compiler *c) {
ADDOP_LOAD_CONST(c, Py_None);
ADDOP(c, DUP_TOP);
ADDOP(c, DUP_TOP);
ADDOP_I(c, CALL_FUNCTION, 3);
ADDOP_I(c, CALL_NO_KW, 3);
return 1;
}
@ -2166,7 +2164,7 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos)
int old_end_col_offset = c->u->u_end_col_offset;
for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) {
SET_LOC(c, (expr_ty)asdl_seq_GET(decos, i));
ADDOP_I(c, CALL_FUNCTION, 1);
ADDOP_I(c, CALL_NO_KW, 1);
}
c->u->u_lineno = old_lineno;
c->u->u_end_lineno = old_end_lineno;
@ -3885,7 +3883,7 @@ compiler_assert(struct compiler *c, stmt_ty s)
ADDOP(c, LOAD_ASSERTION_ERROR);
if (s->v.Assert.msg) {
VISIT(c, expr, s->v.Assert.msg);
ADDOP_I(c, CALL_FUNCTION, 1);
ADDOP_I(c, CALL_NO_KW, 1);
}
ADDOP_I(c, RAISE_VARARGS, 1);
compiler_use_next_block(c, end);
@ -4693,10 +4691,12 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e)
if (!compiler_call_simple_kw_helper(c, kwds, kwdsl)) {
return 0;
};
ADDOP_I(c, CALL_METHOD_KW, argsl + kwdsl);
ADDOP_I(c, PRECALL_METHOD, argsl + kwdsl+1);
ADDOP_I(c, CALL_KW, argsl + kwdsl);
}
else {
ADDOP_I(c, CALL_METHOD, argsl);
ADDOP_I(c, PRECALL_METHOD, argsl);
ADDOP_I(c, CALL_NO_KW, argsl);
}
c->u->u_lineno = old_lineno;
return 1;
@ -4763,7 +4763,8 @@ compiler_joined_str(struct compiler *c, expr_ty e)
VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i));
ADDOP_I(c, LIST_APPEND, 1);
}
ADDOP_I(c, CALL_METHOD, 1);
ADDOP_I(c, PRECALL_METHOD, 1);
ADDOP_I(c, CALL_NO_KW, 1);
}
else {
VISIT_SEQ(c, expr, e->v.JoinedStr.values);
@ -4935,11 +4936,11 @@ compiler_call_helper(struct compiler *c,
if (!compiler_call_simple_kw_helper(c, keywords, nkwelts)) {
return 0;
};
ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts);
ADDOP_I(c, CALL_KW, n + nelts + nkwelts);
return 1;
}
else {
ADDOP_I(c, CALL_FUNCTION, n + nelts);
ADDOP_I(c, CALL_NO_KW, n + nelts);
return 1;
}
@ -5336,7 +5337,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
ADDOP(c, GET_ITER);
}
ADDOP_I(c, CALL_FUNCTION, 1);
ADDOP_I(c, CALL_NO_KW, 1);
if (is_async_generator && type != COMP_GENEXP) {
ADDOP(c, GET_AWAITABLE);

54
Python/opcode_targets.h generated
View file

@ -39,34 +39,34 @@ static void *opcode_targets[256] = {
&&TARGET_POP_EXCEPT_AND_RERAISE,
&&TARGET_STORE_SUBSCR_LIST_INT,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_CALL_FUNCTION_ADAPTIVE,
&&TARGET_CALL_FUNCTION_BUILTIN_O,
&&TARGET_CALL_FUNCTION_BUILTIN_FAST,
&&TARGET_CALL_FUNCTION_LEN,
&&TARGET_CALL_FUNCTION_ISINSTANCE,
&&TARGET_CALL_FUNCTION_PY_SIMPLE,
&&TARGET_JUMP_ABSOLUTE_QUICK,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_CALL_NO_KW_ADAPTIVE,
&&TARGET_CALL_NO_KW_BUILTIN_O,
&&TARGET_CALL_NO_KW_BUILTIN_FAST,
&&TARGET_CALL_NO_KW_LEN,
&&TARGET_CALL_NO_KW_ISINSTANCE,
&&TARGET_CALL_NO_KW_PY_SIMPLE,
&&TARGET_CALL_NO_KW_LIST_APPEND,
&&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_WITH_EXCEPT_START,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
&&TARGET_BEFORE_ASYNC_WITH,
&&TARGET_BEFORE_WITH,
&&TARGET_END_ASYNC_FOR,
&&TARGET_JUMP_ABSOLUTE_QUICK,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_LOAD_METHOD_ADAPTIVE,
&&TARGET_LOAD_METHOD_CACHED,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_LOAD_METHOD_MODULE,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_GET_ITER,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_PRINT_EXPR,
@ -74,19 +74,19 @@ static void *opcode_targets[256] = {
&&TARGET_YIELD_FROM,
&&TARGET_GET_AWAITABLE,
&&TARGET_LOAD_ASSERTION_ERROR,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_LOAD_METHOD_MODULE,
&&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_STORE_ATTR_SLOT,
&&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&TARGET_SETUP_ANNOTATIONS,
&&TARGET_YIELD_VALUE,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_PREP_RERAISE_STAR,
&&TARGET_POP_EXCEPT,
&&TARGET_STORE_NAME,
@ -122,25 +122,25 @@ static void *opcode_targets[256] = {
&&TARGET_COPY,
&&TARGET_JUMP_IF_NOT_EXC_MATCH,
&&TARGET_BINARY_OP,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_LOAD_FAST,
&&TARGET_STORE_FAST,
&&TARGET_DELETE_FAST,
&&TARGET_JUMP_IF_NOT_EG_MATCH,
&&_unknown_opcode,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_GEN_START,
&&TARGET_RAISE_VARARGS,
&&TARGET_CALL_FUNCTION,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_MAKE_FUNCTION,
&&TARGET_BUILD_SLICE,
&&_unknown_opcode,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_MAKE_CELL,
&&TARGET_LOAD_CLOSURE,
&&TARGET_LOAD_DEREF,
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
&&_unknown_opcode,
&&TARGET_CALL_FUNCTION_KW,
&&_unknown_opcode,
&&TARGET_CALL_FUNCTION_EX,
&&_unknown_opcode,
&&TARGET_EXTENDED_ARG,
@ -160,16 +160,16 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_LOAD_METHOD,
&&TARGET_CALL_METHOD,
&&_unknown_opcode,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&TARGET_CALL_METHOD_KW,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_PRECALL_METHOD,
&&TARGET_CALL_NO_KW,
&&TARGET_CALL_KW,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,

99
Python/specialize.c
View file

@ -127,7 +127,7 @@ _Py_GetSpecializationStats(void) {
err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
err += add_stat_dict(stats, STORE_ATTR, "store_attr");
err += add_stat_dict(stats, CALL_FUNCTION, "call_function");
err += add_stat_dict(stats, CALL_NO_KW, "call_no_kw");
err += add_stat_dict(stats, BINARY_OP, "binary_op");
err += add_stat_dict(stats, COMPARE_OP, "compare_op");
if (err < 0) {
@ -186,7 +186,7 @@ _Py_PrintSpecializationStats(void)
print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr");
print_stats(out, &_specialization_stats[STORE_SUBSCR], "store_subscr");
print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr");
print_stats(out, &_specialization_stats[CALL_FUNCTION], "call_function");
print_stats(out, &_specialization_stats[CALL_NO_KW], "call_no_kw");
print_stats(out, &_specialization_stats[BINARY_OP], "binary_op");
print_stats(out, &_specialization_stats[COMPARE_OP], "compare_op");
if (out != stderr) {
@ -238,7 +238,7 @@ static uint8_t adaptive_opcodes[256] = {
[LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
[BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
[STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
[CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE,
[CALL_NO_KW] = CALL_NO_KW_ADAPTIVE,
[STORE_ATTR] = STORE_ATTR_ADAPTIVE,
[BINARY_OP] = BINARY_OP_ADAPTIVE,
[COMPARE_OP] = COMPARE_OP_ADAPTIVE,
@ -251,7 +251,7 @@ static uint8_t cache_requirements[256] = {
[LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
[BINARY_SUBSCR] = 2, /* _PyAdaptiveEntry, _PyObjectCache */
[STORE_SUBSCR] = 0,
[CALL_FUNCTION] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[CALL_NO_KW] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
[BINARY_OP] = 1, // _PyAdaptiveEntry
[COMPARE_OP] = 1, /* _PyAdaptiveEntry */
@ -491,6 +491,8 @@ initial_counter_value(void) {
#define SPEC_FAIL_PYCFUNCTION_NOARGS 16
#define SPEC_FAIL_BAD_CALL_FLAGS 17
#define SPEC_FAIL_CLASS 18
#define SPEC_FAIL_C_METHOD_CALL 19
#define SPEC_FAIL_METHDESCR_NON_METHOD 20
/* COMPARE_OP */
#define SPEC_FAIL_STRING_COMPARE 13
@ -1261,7 +1263,51 @@ specialize_class_call(
PyObject *callable, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache)
{
SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_CLASS);
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_CLASS);
return -1;
}
static PyMethodDescrObject *_list_append = NULL;
_Py_IDENTIFIER(append);
static int
specialize_method_descriptor(
PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache)
{
int oparg = cache->adaptive.original_oparg;
if (nargs - oparg != 1) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_METHDESCR_NON_METHOD);
return -1;
}
if (_list_append == NULL) {
_list_append = (PyMethodDescrObject *)_PyType_LookupId(&PyList_Type, &PyId_append);
}
if (oparg == 1 && descr == _list_append) {
assert(_Py_OPCODE(instr[-1]) == PRECALL_METHOD);
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr));
return 0;
}
switch (descr->d_method->ml_flags &
(METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
METH_KEYWORDS | METH_METHOD)) {
case METH_O: {
if (oparg != 1) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return 1;
}
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O,
_Py_OPARG(*instr));
return 0;
}
case METH_FASTCALL: {
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
_Py_OPARG(*instr));
return 0;
}
}
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OTHER);
return -1;
}
@ -1274,15 +1320,19 @@ specialize_py_call(
PyCodeObject *code = (PyCodeObject *)func->func_code;
int kind = function_kind(code);
if (kind != SIMPLE_FUNCTION) {
SPECIALIZATION_FAIL(CALL_FUNCTION, kind);
SPECIALIZATION_FAIL(CALL_NO_KW, kind);
return -1;
}
int argcount = code->co_argcount;
if (argcount > 0xffff) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults);
assert(defcount <= argcount);
int min_args = argcount-defcount;
if (nargs > argcount || nargs < min_args) {
SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
return -1;
}
assert(nargs <= argcount && nargs >= min_args);
@ -1291,18 +1341,19 @@ specialize_py_call(
assert(defstart >= 0 && deflen >= 0);
assert(deflen == 0 || func->func_defaults != NULL);
if (defstart > 0xffff || deflen > 0xffff) {
SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_RANGE);
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
int version = _PyFunction_GetVersionForCurrentState(func);
if (version == 0) {
SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_VERSIONS);
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
cache[0].adaptive.index = nargs;
cache1->func_version = version;
cache1->defaults_start = defstart;
cache1->defaults_len = deflen;
*instr = _Py_MAKECODEUNIT(CALL_FUNCTION_PY_SIMPLE, _Py_OPARG(*instr));
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_PY_SIMPLE, _Py_OPARG(*instr));
return 0;
}
@ -1335,6 +1386,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
SpecializedCacheEntry *cache, PyObject *builtins)
{
_PyObjectCache *cache1 = &cache[-1].obj;
if (_Py_OPCODE(instr[-1]) == PRECALL_METHOD) {
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_C_METHOD_CALL);
return -1;
}
if (PyCFunction_GET_FUNCTION(callable) == NULL) {
return 1;
}
@ -1343,18 +1398,18 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
METH_KEYWORDS | METH_METHOD)) {
case METH_O: {
if (nargs != 1) {
SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_RANGE);
SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
return 1;
}
/* len(o) */
PyObject *builtin_len = PyDict_GetItemString(builtins, "len");
if (callable == builtin_len) {
cache1->obj = builtin_len; // borrowed
*instr = _Py_MAKECODEUNIT(CALL_FUNCTION_LEN,
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_LEN,
_Py_OPARG(*instr));
return 0;
}
*instr = _Py_MAKECODEUNIT(CALL_FUNCTION_BUILTIN_O,
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_O,
_Py_OPARG(*instr));
return 0;
}
@ -1365,17 +1420,17 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
builtins, "isinstance");
if (callable == builtin_isinstance) {
cache1->obj = builtin_isinstance; // borrowed
*instr = _Py_MAKECODEUNIT(CALL_FUNCTION_ISINSTANCE,
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_ISINSTANCE,
_Py_OPARG(*instr));
return 0;
}
}
*instr = _Py_MAKECODEUNIT(CALL_FUNCTION_BUILTIN_FAST,
*instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_FAST,
_Py_OPARG(*instr));
return 0;
}
default:
SPECIALIZATION_FAIL(CALL_FUNCTION,
SPECIALIZATION_FAIL(CALL_NO_KW,
builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
return 1;
}
@ -1406,7 +1461,7 @@ call_fail_kind(PyObject *callable)
- Specialize calling classes.
*/
int
_Py_Specialize_CallFunction(
_Py_Specialize_CallNoKw(
PyObject *callable, _Py_CODEUNIT *instr,
int nargs, SpecializedCacheEntry *cache,
PyObject *builtins)
@ -1421,18 +1476,22 @@ _Py_Specialize_CallFunction(
else if (PyType_Check(callable)) {
fail = specialize_class_call(callable, instr, nargs, cache);
}
else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
fail = specialize_method_descriptor(
(PyMethodDescrObject *)callable, instr, nargs, cache);
}
else {
SPECIALIZATION_FAIL(CALL_FUNCTION, call_fail_kind(callable));
SPECIALIZATION_FAIL(CALL_NO_KW, call_fail_kind(callable));
fail = -1;
}
_PyAdaptiveEntry *cache0 = &cache->adaptive;
if (fail) {
STAT_INC(CALL_FUNCTION, specialization_failure);
STAT_INC(CALL_NO_KW, specialization_failure);
assert(!PyErr_Occurred());
cache_backoff(cache0);
}
else {
STAT_INC(CALL_FUNCTION, specialization_success);
STAT_INC(CALL_NO_KW, specialization_success);
assert(!PyErr_Occurred());
cache0->counter = initial_counter_value();
}