mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 14:07:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

gh-98831: Modernize CALL and family (#101508)

Browse source 
Includes a slight improvement to `DECREF_INPUTS()`.
This commit is contained in:
Guido van Rossum 2023-02-08 11:40:10 -08:00 • committed by GitHub
parent d9de079248
commit 616aec1ff1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 644 additions and 586 deletions
Download patch file Download diff file Expand all files Collapse all files

530
Python/bytecodes.c
View file

@ -2360,70 +2360,87 @@ dummy_func(
assert(oparg & 1); assert(oparg & 1);
} }
// stack effect: (__0, __array[oparg] -- )
inst(CALL_BOUND_METHOD_EXACT_ARGS) {
DEOPT_IF(is_method(stack_pointer, oparg), CALL);
PyObject *function = PEEK(oparg + 1);
DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)function)->im_self;
PEEK(oparg + 1) = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth);
Py_DECREF(function);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
inst(KW_NAMES, (--)) { inst(KW_NAMES, (--)) {
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts)); assert(oparg < PyTuple_GET_SIZE(consts));
kwnames = GETITEM(consts, oparg); kwnames = GETITEM(consts, oparg);
} }
// stack effect: (__0, __array[oparg] -- ) // Cache layout: counter/1, func_version/2, min_args/1
inst(CALL) { // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members!
family(call, INLINE_CACHE_ENTRIES_CALL) = {
CALL,
CALL_BOUND_METHOD_EXACT_ARGS,
CALL_PY_EXACT_ARGS,
CALL_PY_WITH_DEFAULTS,
CALL_NO_KW_TYPE_1,
CALL_NO_KW_STR_1,
CALL_NO_KW_TUPLE_1,
CALL_BUILTIN_CLASS,
CALL_NO_KW_BUILTIN_O,
CALL_NO_KW_BUILTIN_FAST,
CALL_BUILTIN_FAST_WITH_KEYWORDS,
CALL_NO_KW_LEN,
CALL_NO_KW_ISINSTANCE,
CALL_NO_KW_LIST_APPEND,
CALL_NO_KW_METHOD_DESCRIPTOR_O,
CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS,
CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
};
// On entry, the stack is either
// [NULL, callable, arg1, arg2, ...]
// or
// [method, self, arg1, arg2, ...]
// (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.)
// On exit, the stack is [result].
// When calling Python, inline the call using DISPATCH_INLINED().
inst(CALL, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr; _PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
next_instr--; next_instr--;
_Py_Specialize_Call(callable, next_instr, nargs, kwnames); _Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(CALL, deferred); STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter); DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
int total_args, is_meth; if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) {
is_meth = is_method(stack_pointer, oparg); is_meth = 1; // For consistenct; it's dead, though
PyObject *function = PEEK(oparg + 1); args--;
if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { total_args++;
PyObject *self = ((PyMethodObject *)function)->im_self; PyObject *self = ((PyMethodObject *)callable)->im_self;
PEEK(oparg+1) = Py_NewRef(self); args[0] = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func; method = ((PyMethodObject *)callable)->im_func;
PEEK(oparg+2) = Py_NewRef(meth); args[-1] = Py_NewRef(method);
Py_DECREF(function); Py_DECREF(callable);
is_meth = 1; callable = method;
} }
total_args = oparg + is_meth;
function = PEEK(total_args + 1);
int positional_args = total_args - KWNAMES_LEN(); int positional_args = total_args - KWNAMES_LEN();
// Check if the call can be inlined or not // Check if the call can be inlined or not
if (Py_TYPE(function) == &PyFunction_Type && if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL && tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall) ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{ {
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
STACK_SHRINK(total_args);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)function, locals, tstate, (PyFunctionObject *)callable, locals,
stack_pointer, positional_args, kwnames args, positional_args, kwnames
); );
kwnames = NULL; kwnames = NULL;
STACK_SHRINK(2-is_meth); // Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack, // The frame has stolen all the arguments from the stack,
// so there is no need to clean them up. // so there is no need to clean them up.
if (new_frame == NULL) { if (new_frame == NULL) {
@ -2433,190 +2450,180 @@ dummy_func(
DISPATCH_INLINED(new_frame); DISPATCH_INLINED(new_frame);
} }
/* Callable is not a normal Python function */ /* Callable is not a normal Python function */
PyObject *res;
if (cframe.use_tracing) { if (cframe.use_tracing) {
res = trace_call_function( res = trace_call_function(
tstate, function, stack_pointer-total_args, tstate, callable, args,
positional_args, kwnames); positional_args, kwnames);
} }
else { else {
res = PyObject_Vectorcall( res = PyObject_Vectorcall(
function, stack_pointer-total_args, callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames); kwnames);
} }
kwnames = NULL; kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(function); Py_DECREF(callable);
/* Clear the stack */
STACK_SHRINK(total_args);
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
STACK_SHRINK(2-is_meth); ERROR_IF(res == NULL, error);
PUSH(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) // Start out with [NULL, bound_method, arg1, arg2, ...]
inst(CALL_PY_EXACT_ARGS) { // Transform to [callable, self, arg1, arg2, ...]
// Then fall through to CALL_PY_EXACT_ARGS
inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, method, callable, unused[oparg] -- unused)) {
DEOPT_IF(method != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)callable)->im_self;
PEEK(oparg + 1) = Py_NewRef(self); // callable
PyObject *meth = ((PyMethodObject *)callable)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth); // method
Py_DECREF(callable);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, method, callable, args[oparg] -- unused)) {
assert(kwnames == NULL); assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL); DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = method != NULL;
int is_meth = is_method(stack_pointer, oparg); int argcount = oparg;
int argcount = oparg + is_meth; if (is_meth) {
PyObject *callable = PEEK(argcount + 1); callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL); DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable; PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code; PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != argcount, CALL); DEOPT_IF(code->co_argcount != argcount, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) { for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i]; new_frame->localsplus[i] = args[i];
} }
STACK_SHRINK(2-is_meth); // Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame); DISPATCH_INLINED(new_frame);
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, method, callable, args[oparg] -- unused)) {
inst(CALL_PY_WITH_DEFAULTS) {
assert(kwnames == NULL); assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL); DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = method != NULL;
int is_meth = is_method(stack_pointer, oparg); int argcount = oparg;
int argcount = oparg + is_meth; if (is_meth) {
PyObject *callable = PEEK(argcount + 1); callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL); DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable; PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code; PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL); DEOPT_IF(argcount > code->co_argcount, CALL);
int minargs = cache->min_args; DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(argcount < minargs, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) { for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i]; new_frame->localsplus[i] = args[i];
} }
for (int i = argcount; i < code->co_argcount; i++) { for (int i = argcount; i < code->co_argcount; i++) {
PyObject *def = PyTuple_GET_ITEM(func->func_defaults, PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args);
i - minargs);
new_frame->localsplus[i] = Py_NewRef(def); new_frame->localsplus[i] = Py_NewRef(def);
} }
STACK_SHRINK(2-is_meth); // Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame); DISPATCH_INLINED(new_frame);
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_TYPE_1) {
assert(kwnames == NULL); assert(kwnames == NULL);
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(oparg == 1); assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL); DEOPT_IF(null != NULL, CALL);
PyObject *obj = TOP(); PyObject *obj = args[0];
PyObject *callable = SECOND();
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); res = Py_NewRef(Py_TYPE(obj));
PyObject *res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(callable);
Py_DECREF(obj); Py_DECREF(obj);
STACK_SHRINK(2); Py_DECREF(&PyType_Type); // I.e., callable
SET_TOP(res);
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_STR_1) {
assert(kwnames == NULL); assert(kwnames == NULL);
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(oparg == 1); assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL); DEOPT_IF(null != NULL, CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = TOP(); PyObject *arg = args[0];
PyObject *res = PyObject_Str(arg); res = PyObject_Str(arg);
Py_DECREF(arg); Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type); Py_DECREF(&PyUnicode_Type); // I.e., callable
STACK_SHRINK(2); ERROR_IF(res == NULL, error);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_TUPLE_1) {
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg == 1); assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL); DEOPT_IF(null != NULL, CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = TOP(); PyObject *arg = args[0];
PyObject *res = PySequence_Tuple(arg); res = PySequence_Tuple(arg);
Py_DECREF(arg); Py_DECREF(arg);
Py_DECREF(&PyTuple_Type); Py_DECREF(&PyTuple_Type); // I.e., tuple
STACK_SHRINK(2); ERROR_IF(res == NULL, error);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
inst(CALL_BUILTIN_CLASS) { int is_meth = method != NULL;
int is_meth = is_method(stack_pointer, oparg); int total_args = oparg;
int total_args = oparg + is_meth; if (is_meth) {
callable = method;
args--;
total_args++;
}
int kwnames_len = KWNAMES_LEN(); int kwnames_len = KWNAMES_LEN();
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyType_Check(callable), CALL); DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable; PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL); DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
STACK_SHRINK(total_args); res = tp->tp_vectorcall((PyObject *)tp, args,
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, total_args - kwnames_len, kwnames);
total_args-kwnames_len, kwnames);
kwnames = NULL; kwnames = NULL;
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
Py_DECREF(tp); Py_DECREF(tp);
STACK_SHRINK(1-is_meth); ERROR_IF(res == NULL, error);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_BUILTIN_O) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */ /* Builtin METH_O functions */
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL); DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
@ -2626,81 +2633,74 @@ dummy_func(
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error; goto error;
} }
PyObject *arg = TOP(); PyObject *arg = args[0];
PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate); _Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg); Py_DECREF(arg);
Py_DECREF(callable); Py_DECREF(callable);
STACK_SHRINK(2-is_meth); ERROR_IF(res == NULL, error);
SET_TOP(res);
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_BUILTIN_FAST) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */ /* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
PyObject *callable = PEEK(total_args + 1); if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */ /* res = func(self, args, nargs) */
PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable), PyCFunction_GET_SELF(callable),
stack_pointer, args,
total_args); total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
/* Not deopting because this doesn't mean our optimization was /* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x, wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must 'invalid'). In those cases an exception is set, so we must
handle it. handle it.
*/ */
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */ /* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
PyObject *callable = PEEK(total_args + 1); if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL); (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */ /* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void)) (_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable); PyCFunction_GET_FUNCTION(callable);
PyObject *res = cfunc( res = cfunc(
PyCFunction_GET_SELF(callable), PyCFunction_GET_SELF(callable),
stack_pointer, args,
total_args - KWNAMES_LEN(), total_args - KWNAMES_LEN(),
kwnames kwnames
); );
@ -2709,117 +2709,109 @@ dummy_func(
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LEN, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_LEN) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(kwnames == NULL); assert(kwnames == NULL);
/* len(o) */ /* len(o) */
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL); DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.len, CALL); DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = TOP(); PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg); Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) { if (len_i < 0) {
goto error; goto error;
} }
PyObject *res = PyLong_FromSsize_t(len_i); res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
Py_DECREF(arg); Py_DECREF(arg);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) {
inst(CALL_NO_KW_ISINSTANCE) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(kwnames == NULL); assert(kwnames == NULL);
/* isinstance(o, o2) */ /* isinstance(o, o2) */
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
PyObject *callable = PEEK(total_args + 1); if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL); DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *cls = POP(); PyObject *cls = args[1];
PyObject *inst = TOP(); PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls); int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) { if (retval < 0) {
Py_DECREF(cls);
goto error; goto error;
} }
PyObject *res = PyBool_FromLong(retval); res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(inst); Py_DECREF(inst);
Py_DECREF(cls); Py_DECREF(cls);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
} }
// stack effect: (__0, __array[oparg] -- ) // This is secretly a super-instruction
inst(CALL_NO_KW_LIST_APPEND) { inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- unused)) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg == 1); assert(oparg == 1);
PyObject *callable = PEEK(3); assert(method != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.list_append, CALL); DEOPT_IF(method != interp->callable_cache.list_append, CALL);
PyObject *list = SECOND(); DEOPT_IF(!PyList_Check(self), CALL);
DEOPT_IF(!PyList_Check(list), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = POP(); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already
goto error;
} }
STACK_SHRINK(2); Py_DECREF(self);
Py_DECREF(list); Py_DECREF(method);
Py_DECREF(callable); STACK_SHRINK(3);
// CALL + POP_TOP // CALL + POP_TOP
JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
DISPATCH();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable = PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1); (PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(total_args != 2, CALL); DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL); DEOPT_IF(meth->ml_flags != METH_O, CALL);
PyObject *arg = TOP(); PyObject *arg = args[1];
PyObject *self = SECOND(); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth; PyCFunction cfunc = meth->ml_meth;
@ -2828,69 +2820,62 @@ dummy_func(
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error; goto error;
} }
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate); _Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self); Py_DECREF(self);
Py_DECREF(arg); Py_DECREF(arg);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { int is_meth = method != NULL;
int is_meth = is_method(stack_pointer, oparg); int total_args = oparg;
int total_args = oparg + is_meth; if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable = PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1); (PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = callable->d_common.d_type; PyTypeObject *d_type = callable->d_common.d_type;
PyObject *self = PEEK(total_args); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
int nargs = total_args-1; int nargs = total_args - 1;
STACK_SHRINK(nargs);
_PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames);
kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
kwnames = NULL; kwnames = NULL;
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < nargs; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg == 0 || oparg == 1); assert(oparg == 0 || oparg == 1);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL); DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
PyObject *self = TOP(); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
@ -2900,52 +2885,43 @@ dummy_func(
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error; goto error;
} }
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate); _Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self); Py_DECREF(self);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
// stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) {
inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable = PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1); (PyMethodDescrObject *)PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */ /* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
PyObject *self = PEEK(total_args); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
_PyCFunctionFast cfunc = _PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth; (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args-1; int nargs = total_args - 1;
STACK_SHRINK(nargs); res = cfunc(self, args + 1, nargs);
PyObject *res = cfunc(self, stack_pointer, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */ /* Clear the stack of the arguments. */
for (int i = 0; i < nargs; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { ERROR_IF(res == NULL, error);
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
} }
@ -3153,12 +3129,4 @@ dummy_func(
// Future families go below this point // // Future families go below this point //
family(call, INLINE_CACHE_ENTRIES_CALL) = {
CALL, CALL_PY_EXACT_ARGS,
CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS,
CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST,
CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN,
CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1,
CALL_NO_KW_TYPE_1 };
family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST };

6
Python/ceval.c
View file

@ -688,12 +688,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) {
} }
// GH-89279: Must be a macro to be sure it's inlined by MSVC.
#define is_method(stack_pointer, args) (PEEK((args)+2) != NULL)
#define KWNAMES_LEN() \
(kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))
/* Disable unused label warnings. They are handy for debugging, even /* Disable unused label warnings. They are handy for debugging, even
if computed gotos aren't used. */ if computed gotos aren't used. */

3
Python/ceval_macros.h
View file

@ -347,3 +347,6 @@ GETITEM(PyObject *v, Py_ssize_t i) {
} while (0); } while (0);
#define NAME_ERROR_MSG "name '%.200s' is not defined" #define NAME_ERROR_MSG "name '%.200s' is not defined"
#define KWNAMES_LEN() \
(kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames)))

558
Python/generated_cases.c.h generated
View file

@ -2994,19 +2994,6 @@
DISPATCH(); DISPATCH();
} }
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
DEOPT_IF(is_method(stack_pointer, oparg), CALL);
PyObject *function = PEEK(oparg + 1);
DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)function)->im_self;
PEEK(oparg + 1) = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth);
Py_DECREF(function);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
TARGET(KW_NAMES) { TARGET(KW_NAMES) {
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg < PyTuple_GET_SIZE(consts)); assert(oparg < PyTuple_GET_SIZE(consts));
@ -3016,48 +3003,55 @@
TARGET(CALL) { TARGET(CALL) {
PREDICTED(CALL); PREDICTED(CALL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size");
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr; _PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
next_instr--; next_instr--;
_Py_Specialize_Call(callable, next_instr, nargs, kwnames); _Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(CALL, deferred); STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter); DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
int total_args, is_meth; if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) {
is_meth = is_method(stack_pointer, oparg); is_meth = 1; // For consistenct; it's dead, though
PyObject *function = PEEK(oparg + 1); args--;
if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { total_args++;
PyObject *self = ((PyMethodObject *)function)->im_self; PyObject *self = ((PyMethodObject *)callable)->im_self;
PEEK(oparg+1) = Py_NewRef(self); args[0] = Py_NewRef(self);
PyObject *meth = ((PyMethodObject *)function)->im_func; method = ((PyMethodObject *)callable)->im_func;
PEEK(oparg+2) = Py_NewRef(meth); args[-1] = Py_NewRef(method);
Py_DECREF(function); Py_DECREF(callable);
is_meth = 1; callable = method;
} }
total_args = oparg + is_meth;
function = PEEK(total_args + 1);
int positional_args = total_args - KWNAMES_LEN(); int positional_args = total_args - KWNAMES_LEN();
// Check if the call can be inlined or not // Check if the call can be inlined or not
if (Py_TYPE(function) == &PyFunction_Type && if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL && tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall) ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{ {
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
STACK_SHRINK(total_args);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)function, locals, tstate, (PyFunctionObject *)callable, locals,
stack_pointer, positional_args, kwnames args, positional_args, kwnames
); );
kwnames = NULL; kwnames = NULL;
STACK_SHRINK(2-is_meth); // Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack, // The frame has stolen all the arguments from the stack,
// so there is no need to clean them up. // so there is no need to clean them up.
if (new_frame == NULL) { if (new_frame == NULL) {
@ -3067,189 +3061,234 @@
DISPATCH_INLINED(new_frame); DISPATCH_INLINED(new_frame);
} }
/* Callable is not a normal Python function */ /* Callable is not a normal Python function */
PyObject *res;
if (cframe.use_tracing) { if (cframe.use_tracing) {
res = trace_call_function( res = trace_call_function(
tstate, function, stack_pointer-total_args, tstate, callable, args,
positional_args, kwnames); positional_args, kwnames);
} }
else { else {
res = PyObject_Vectorcall( res = PyObject_Vectorcall(
function, stack_pointer-total_args, callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames); kwnames);
} }
kwnames = NULL; kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(function); Py_DECREF(callable);
/* Clear the stack */
STACK_SHRINK(total_args);
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
STACK_SHRINK(2-is_meth); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
PUSH(res); STACK_SHRINK(oparg);
if (res == NULL) { STACK_SHRINK(1);
goto error; POKE(1, res);
} JUMPBY(4);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
DEOPT_IF(method != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
STAT_INC(CALL, hit);
PyObject *self = ((PyMethodObject *)callable)->im_self;
PEEK(oparg + 1) = Py_NewRef(self); // callable
PyObject *meth = ((PyMethodObject *)callable)->im_func;
PEEK(oparg + 2) = Py_NewRef(meth); // method
Py_DECREF(callable);
GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS);
}
TARGET(CALL_PY_EXACT_ARGS) { TARGET(CALL_PY_EXACT_ARGS) {
PREDICTED(CALL_PY_EXACT_ARGS); PREDICTED(CALL_PY_EXACT_ARGS);
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
uint32_t func_version = read_u32(&next_instr[1].cache);
assert(kwnames == NULL); assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL); DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = method != NULL;
int is_meth = is_method(stack_pointer, oparg); int argcount = oparg;
int argcount = oparg + is_meth; if (is_meth) {
PyObject *callable = PEEK(argcount + 1); callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL); DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable; PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code; PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != argcount, CALL); DEOPT_IF(code->co_argcount != argcount, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) { for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i]; new_frame->localsplus[i] = args[i];
} }
STACK_SHRINK(2-is_meth); // Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame); DISPATCH_INLINED(new_frame);
} }
TARGET(CALL_PY_WITH_DEFAULTS) { TARGET(CALL_PY_WITH_DEFAULTS) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
uint32_t func_version = read_u32(&next_instr[1].cache);
uint16_t min_args = read_u16(&next_instr[3].cache);
assert(kwnames == NULL); assert(kwnames == NULL);
DEOPT_IF(tstate->interp->eval_frame, CALL); DEOPT_IF(tstate->interp->eval_frame, CALL);
_PyCallCache *cache = (_PyCallCache *)next_instr; int is_meth = method != NULL;
int is_meth = is_method(stack_pointer, oparg); int argcount = oparg;
int argcount = oparg + is_meth; if (is_meth) {
PyObject *callable = PEEK(argcount + 1); callable = method;
args--;
argcount++;
}
DEOPT_IF(!PyFunction_Check(callable), CALL); DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable; PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code; PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(argcount > code->co_argcount, CALL); DEOPT_IF(argcount > code->co_argcount, CALL);
int minargs = cache->min_args; DEOPT_IF(argcount < min_args, CALL);
DEOPT_IF(argcount < minargs, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount);
STACK_SHRINK(argcount);
for (int i = 0; i < argcount; i++) { for (int i = 0; i < argcount; i++) {
new_frame->localsplus[i] = stack_pointer[i]; new_frame->localsplus[i] = args[i];
} }
for (int i = argcount; i < code->co_argcount; i++) { for (int i = argcount; i < code->co_argcount; i++) {
PyObject *def = PyTuple_GET_ITEM(func->func_defaults, PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args);
i - minargs);
new_frame->localsplus[i] = Py_NewRef(def); new_frame->localsplus[i] = Py_NewRef(def);
} }
STACK_SHRINK(2-is_meth); // Manipulate stack and cache directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); JUMPBY(INLINE_CACHE_ENTRIES_CALL);
DISPATCH_INLINED(new_frame); DISPATCH_INLINED(new_frame);
} }
TARGET(CALL_NO_KW_TYPE_1) { TARGET(CALL_NO_KW_TYPE_1) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *null = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL); assert(kwnames == NULL);
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(oparg == 1); assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL); DEOPT_IF(null != NULL, CALL);
PyObject *obj = TOP(); PyObject *obj = args[0];
PyObject *callable = SECOND();
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); res = Py_NewRef(Py_TYPE(obj));
PyObject *res = Py_NewRef(Py_TYPE(obj));
Py_DECREF(callable);
Py_DECREF(obj); Py_DECREF(obj);
STACK_SHRINK(2); Py_DECREF(&PyType_Type); // I.e., callable
SET_TOP(res); STACK_SHRINK(oparg);
STACK_SHRINK(1);
POKE(1, res);
JUMPBY(4);
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_STR_1) { TARGET(CALL_NO_KW_STR_1) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *null = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL); assert(kwnames == NULL);
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(oparg == 1); assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL); DEOPT_IF(null != NULL, CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = TOP(); PyObject *arg = args[0];
PyObject *res = PyObject_Str(arg); res = PyObject_Str(arg);
Py_DECREF(arg); Py_DECREF(arg);
Py_DECREF(&PyUnicode_Type); Py_DECREF(&PyUnicode_Type); // I.e., callable
STACK_SHRINK(2); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
SET_TOP(res); STACK_SHRINK(oparg);
if (res == NULL) { STACK_SHRINK(1);
goto error; POKE(1, res);
} JUMPBY(4);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_TUPLE_1) { TARGET(CALL_NO_KW_TUPLE_1) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *null = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg == 1); assert(oparg == 1);
DEOPT_IF(is_method(stack_pointer, 1), CALL); DEOPT_IF(null != NULL, CALL);
PyObject *callable = PEEK(2);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = TOP(); PyObject *arg = args[0];
PyObject *res = PySequence_Tuple(arg); res = PySequence_Tuple(arg);
Py_DECREF(arg); Py_DECREF(arg);
Py_DECREF(&PyTuple_Type); Py_DECREF(&PyTuple_Type); // I.e., tuple
STACK_SHRINK(2); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
SET_TOP(res); STACK_SHRINK(oparg);
if (res == NULL) { STACK_SHRINK(1);
goto error; POKE(1, res);
} JUMPBY(4);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_BUILTIN_CLASS) { TARGET(CALL_BUILTIN_CLASS) {
int is_meth = is_method(stack_pointer, oparg); PyObject **args = &PEEK(oparg);
int total_args = oparg + is_meth; PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
int kwnames_len = KWNAMES_LEN(); int kwnames_len = KWNAMES_LEN();
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyType_Check(callable), CALL); DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable; PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL); DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
STACK_SHRINK(total_args); res = tp->tp_vectorcall((PyObject *)tp, args,
PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, total_args - kwnames_len, kwnames);
total_args-kwnames_len, kwnames);
kwnames = NULL; kwnames = NULL;
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
Py_DECREF(tp); Py_DECREF(tp);
STACK_SHRINK(1-is_meth); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
SET_TOP(res); STACK_SHRINK(oparg);
if (res == NULL) { STACK_SHRINK(1);
goto error; POKE(1, res);
} JUMPBY(4);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_BUILTIN_O) { TARGET(CALL_NO_KW_BUILTIN_O) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */ /* Builtin METH_O functions */
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL); DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
@ -3259,81 +3298,92 @@
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error; goto error;
} }
PyObject *arg = TOP(); PyObject *arg = args[0];
PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate); _Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg); Py_DECREF(arg);
Py_DECREF(callable); Py_DECREF(callable);
STACK_SHRINK(2-is_meth); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
SET_TOP(res); STACK_SHRINK(oparg);
if (res == NULL) { STACK_SHRINK(1);
goto error; POKE(1, res);
} JUMPBY(4);
JUMPBY(INLINE_CACHE_ENTRIES_CALL);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_BUILTIN_FAST) { TARGET(CALL_NO_KW_BUILTIN_FAST) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */ /* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
PyObject *callable = PEEK(total_args + 1); if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs) */ /* res = func(self, args, nargs) */
PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable), PyCFunction_GET_SELF(callable),
stack_pointer, args,
total_args); total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
/* Not deopting because this doesn't mean our optimization was /* Not deopting because this doesn't mean our optimization was
wrong. `res` can be NULL for valid reasons. Eg. getattr(x, wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
'invalid'). In those cases an exception is set, so we must 'invalid'). In those cases an exception is set, so we must
handle it. handle it.
*/ */
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */ /* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
PyObject *callable = PEEK(total_args + 1); if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
(METH_FASTCALL | METH_KEYWORDS), CALL); (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
STACK_SHRINK(total_args);
/* res = func(self, args, nargs, kwnames) */ /* res = func(self, args, nargs, kwnames) */
_PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void)) (_PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable); PyCFunction_GET_FUNCTION(callable);
PyObject *res = cfunc( res = cfunc(
PyCFunction_GET_SELF(callable), PyCFunction_GET_SELF(callable),
stack_pointer, args,
total_args - KWNAMES_LEN(), total_args - KWNAMES_LEN(),
kwnames kwnames
); );
@ -3342,99 +3392,112 @@
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < total_args; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
STACK_SHRINK(2-is_meth);
PUSH(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_LEN) { TARGET(CALL_NO_KW_LEN) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(kwnames == NULL); assert(kwnames == NULL);
/* len(o) */ /* len(o) */
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL); DEOPT_IF(total_args != 1, CALL);
PyObject *callable = PEEK(total_args + 1);
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.len, CALL); DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = TOP(); PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg); Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) { if (len_i < 0) {
goto error; goto error;
} }
PyObject *res = PyLong_FromSsize_t(len_i); res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
Py_DECREF(arg); Py_DECREF(arg);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_ISINSTANCE) { TARGET(CALL_NO_KW_ISINSTANCE) {
PyObject **args = &PEEK(oparg);
PyObject *callable = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(kwnames == NULL); assert(kwnames == NULL);
/* isinstance(o, o2) */ /* isinstance(o, o2) */
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
PyObject *callable = PEEK(total_args + 1); if (is_meth) {
callable = method;
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL); DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *cls = POP(); PyObject *cls = args[1];
PyObject *inst = TOP(); PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls); int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) { if (retval < 0) {
Py_DECREF(cls);
goto error; goto error;
} }
PyObject *res = PyBool_FromLong(retval); res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(inst); Py_DECREF(inst);
Py_DECREF(cls); Py_DECREF(cls);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_LIST_APPEND) { TARGET(CALL_NO_KW_LIST_APPEND) {
PyObject **args = &PEEK(oparg);
PyObject *self = PEEK(1 + oparg);
PyObject *method = PEEK(2 + oparg);
assert(cframe.use_tracing == 0); assert(cframe.use_tracing == 0);
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg == 1); assert(oparg == 1);
PyObject *callable = PEEK(3); assert(method != NULL);
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
DEOPT_IF(callable != interp->callable_cache.list_append, CALL); DEOPT_IF(method != interp->callable_cache.list_append, CALL);
PyObject *list = SECOND(); DEOPT_IF(!PyList_Check(self), CALL);
DEOPT_IF(!PyList_Check(list), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyObject *arg = POP(); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) {
if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already
goto error;
} }
STACK_SHRINK(2); Py_DECREF(self);
Py_DECREF(list); Py_DECREF(method);
Py_DECREF(callable); STACK_SHRINK(3);
// CALL + POP_TOP // CALL + POP_TOP
JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1);
assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); assert(_Py_OPCODE(next_instr[-1]) == POP_TOP);
@ -3442,17 +3505,24 @@
} }
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable = PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1); (PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(total_args != 2, CALL); DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL); DEOPT_IF(meth->ml_flags != METH_O, CALL);
PyObject *arg = TOP(); PyObject *arg = args[1];
PyObject *self = SECOND(); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth; PyCFunction cfunc = meth->ml_meth;
@ -3461,69 +3531,78 @@
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error; goto error;
} }
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate); _Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self); Py_DECREF(self);
Py_DECREF(arg); Py_DECREF(arg);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
int is_meth = is_method(stack_pointer, oparg); PyObject **args = &PEEK(oparg);
int total_args = oparg + is_meth; PyObject *method = PEEK(2 + oparg);
PyObject *res;
int is_meth = method != NULL;
int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable = PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1); (PyMethodDescrObject *)PEEK(total_args + 1);
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = callable->d_common.d_type; PyTypeObject *d_type = callable->d_common.d_type;
PyObject *self = PEEK(total_args); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
int nargs = total_args-1; int nargs = total_args - 1;
STACK_SHRINK(nargs);
_PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords cfunc =
(_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames);
kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
kwnames = NULL; kwnames = NULL;
/* Free the arguments. */ /* Free the arguments. */
for (int i = 0; i < nargs; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL); assert(kwnames == NULL);
assert(oparg == 0 || oparg == 1); assert(oparg == 0 || oparg == 1);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL); DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND();
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
PyObject *self = TOP(); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
@ -3533,52 +3612,55 @@
if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) {
goto error; goto error;
} }
PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate); _Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self); Py_DECREF(self);
STACK_SHRINK(oparg + 1);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }
TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
PyObject **args = &PEEK(oparg);
PyObject *method = PEEK(2 + oparg);
PyObject *res;
assert(kwnames == NULL); assert(kwnames == NULL);
int is_meth = is_method(stack_pointer, oparg); int is_meth = method != NULL;
int total_args = oparg + is_meth; int total_args = oparg;
if (is_meth) {
args--;
total_args++;
}
PyMethodDescrObject *callable = PyMethodDescrObject *callable =
(PyMethodDescrObject *)PEEK(total_args + 1); (PyMethodDescrObject *)PEEK(total_args + 1);
/* Builtin METH_FASTCALL methods, without keywords */ /* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = callable->d_method; PyMethodDef *meth = callable->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
PyObject *self = PEEK(total_args); PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL);
STAT_INC(CALL, hit); STAT_INC(CALL, hit);
_PyCFunctionFast cfunc = _PyCFunctionFast cfunc =
(_PyCFunctionFast)(void(*)(void))meth->ml_meth; (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args-1; int nargs = total_args - 1;
STACK_SHRINK(nargs); res = cfunc(self, args + 1, nargs);
PyObject *res = cfunc(self, stack_pointer, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */ /* Clear the stack of the arguments. */
for (int i = 0; i < nargs; i++) { for (int i = 0; i < total_args; i++) {
Py_DECREF(stack_pointer[i]); Py_DECREF(args[i]);
} }
Py_DECREF(self);
STACK_SHRINK(2-is_meth);
SET_TOP(res);
Py_DECREF(callable); Py_DECREF(callable);
if (res == NULL) { if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
goto error; STACK_SHRINK(oparg);
} STACK_SHRINK(1);
JUMPBY(INLINE_CACHE_ENTRIES_CALL); POKE(1, res);
JUMPBY(4);
CHECK_EVAL_BREAKER(); CHECK_EVAL_BREAKER();
DISPATCH(); DISPATCH();
} }

112
Python/opcode_metadata.h
View file

@ -286,44 +286,44 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) {
return 1; return 1;
case LOAD_ATTR_METHOD_LAZY_DICT: case LOAD_ATTR_METHOD_LAZY_DICT:
return 1; return 1;
case CALL_BOUND_METHOD_EXACT_ARGS:
return -1;
case KW_NAMES: case KW_NAMES:
return 0; return 0;
case CALL: case CALL:
return -1; return oparg + 2;
case CALL_BOUND_METHOD_EXACT_ARGS:
return oparg + 2;
case CALL_PY_EXACT_ARGS: case CALL_PY_EXACT_ARGS:
return -1; return oparg + 2;
case CALL_PY_WITH_DEFAULTS: case CALL_PY_WITH_DEFAULTS:
return -1; return oparg + 2;
case CALL_NO_KW_TYPE_1: case CALL_NO_KW_TYPE_1:
return -1; return oparg + 2;
case CALL_NO_KW_STR_1: case CALL_NO_KW_STR_1:
return -1; return oparg + 2;
case CALL_NO_KW_TUPLE_1: case CALL_NO_KW_TUPLE_1:
return -1; return oparg + 2;
case CALL_BUILTIN_CLASS: case CALL_BUILTIN_CLASS:
return -1; return oparg + 2;
case CALL_NO_KW_BUILTIN_O: case CALL_NO_KW_BUILTIN_O:
return -1; return oparg + 2;
case CALL_NO_KW_BUILTIN_FAST: case CALL_NO_KW_BUILTIN_FAST:
return -1; return oparg + 2;
case CALL_BUILTIN_FAST_WITH_KEYWORDS: case CALL_BUILTIN_FAST_WITH_KEYWORDS:
return -1; return oparg + 2;
case CALL_NO_KW_LEN: case CALL_NO_KW_LEN:
return -1; return oparg + 2;
case CALL_NO_KW_ISINSTANCE: case CALL_NO_KW_ISINSTANCE:
return -1; return oparg + 2;
case CALL_NO_KW_LIST_APPEND: case CALL_NO_KW_LIST_APPEND:
return -1; return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_O: case CALL_NO_KW_METHOD_DESCRIPTOR_O:
return -1; return oparg + 2;
case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
return -1; return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS:
return -1; return oparg + 2;
case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: case CALL_NO_KW_METHOD_DESCRIPTOR_FAST:
return -1; return oparg + 2;
case CALL_FUNCTION_EX: case CALL_FUNCTION_EX:
return ((oparg & 1) ? 1 : 0) + 3; return ((oparg & 1) ? 1 : 0) + 3;
case MAKE_FUNCTION: case MAKE_FUNCTION:
@ -634,44 +634,44 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) {
return ((oparg & 1) ? 1 : 0) + 1; return ((oparg & 1) ? 1 : 0) + 1;
case LOAD_ATTR_METHOD_LAZY_DICT: case LOAD_ATTR_METHOD_LAZY_DICT:
return ((oparg & 1) ? 1 : 0) + 1; return ((oparg & 1) ? 1 : 0) + 1;
case CALL_BOUND_METHOD_EXACT_ARGS:
return -1;
case KW_NAMES: case KW_NAMES:
return 0; return 0;
case CALL: case CALL:
return -1; return 1;
case CALL_BOUND_METHOD_EXACT_ARGS:
return 1;
case CALL_PY_EXACT_ARGS: case CALL_PY_EXACT_ARGS:
return -1; return 1;
case CALL_PY_WITH_DEFAULTS: case CALL_PY_WITH_DEFAULTS:
return -1; return 1;
case CALL_NO_KW_TYPE_1: case CALL_NO_KW_TYPE_1:
return -1; return 1;
case CALL_NO_KW_STR_1: case CALL_NO_KW_STR_1:
return -1; return 1;
case CALL_NO_KW_TUPLE_1: case CALL_NO_KW_TUPLE_1:
return -1; return 1;
case CALL_BUILTIN_CLASS: case CALL_BUILTIN_CLASS:
return -1; return 1;
case CALL_NO_KW_BUILTIN_O: case CALL_NO_KW_BUILTIN_O:
return -1; return 1;
case CALL_NO_KW_BUILTIN_FAST: case CALL_NO_KW_BUILTIN_FAST:
return -1; return 1;
case CALL_BUILTIN_FAST_WITH_KEYWORDS: case CALL_BUILTIN_FAST_WITH_KEYWORDS:
return -1; return 1;
case CALL_NO_KW_LEN: case CALL_NO_KW_LEN:
return -1; return 1;
case CALL_NO_KW_ISINSTANCE: case CALL_NO_KW_ISINSTANCE:
return -1; return 1;
case CALL_NO_KW_LIST_APPEND: case CALL_NO_KW_LIST_APPEND:
return -1; return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_O: case CALL_NO_KW_METHOD_DESCRIPTOR_O:
return -1; return 1;
case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS:
return -1; return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS:
return -1; return 1;
case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: case CALL_NO_KW_METHOD_DESCRIPTOR_FAST:
return -1; return 1;
case CALL_FUNCTION_EX: case CALL_FUNCTION_EX:
return 1; return 1;
case MAKE_FUNCTION: case MAKE_FUNCTION:
@ -846,25 +846,25 @@ struct opcode_metadata {
[LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 },
[CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 },
[CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB },
[RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },

21
Tools/cases_generator/generate_cases.py
View file

@ -391,9 +391,11 @@ class Instruction:
# Write the body, substituting a goto for ERROR_IF() and other stuff # Write the body, substituting a goto for ERROR_IF() and other stuff
assert dedent <= 0 assert dedent <= 0
extra = " " * -dedent extra = " " * -dedent
names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"})
for line in self.block_text: for line in self.block_text:
if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line):
space, cond, label = m.groups() space, cond, label = m.groups()
space = extra + space
# ERROR_IF() must pop the inputs from the stack. # ERROR_IF() must pop the inputs from the stack.
# The code block is responsible for DECREF()ing them. # The code block is responsible for DECREF()ing them.
# NOTE: If the label doesn't exist, just add it to ceval.c. # NOTE: If the label doesn't exist, just add it to ceval.c.
@ -412,16 +414,25 @@ class Instruction:
symbolic = "" symbolic = ""
if symbolic: if symbolic:
out.write_raw( out.write_raw(
f"{extra}{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n"
) )
else: else:
out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") out.write_raw(f"{space}if ({cond}) goto {label};\n")
elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line):
if not self.register: if not self.register:
space = m.group(1) space = extra + m.group(1)
for ieff in self.input_effects: for ieff in self.input_effects:
if ieff.name not in self.unmoved_names: if ieff.name in names_to_skip:
out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") continue
if ieff.size:
out.write_raw(
f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n"
)
out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n")
out.write_raw(f"{space}}}\n")
else:
decref = "XDECREF" if ieff.cond else "DECREF"
out.write_raw(f"{space}Py_{decref}({ieff.name});\n")
else: else:
out.write_raw(extra + line) out.write_raw(extra + line)