GH-98686: Get rid of "adaptive" and "quick" instructions (GH-99182)

This commit is contained in:
Brandt Bucher 2022-11-09 10:50:09 -08:00 committed by GitHub
parent 6e3cc72afe
commit c7f5708714
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 562 additions and 758 deletions

View file

@ -327,6 +327,15 @@ dummy_func(
}
inst(BINARY_SUBSCR, (container, sub -- res)) {
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_BinarySubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(BINARY_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
res = PyObject_GetItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
@ -364,25 +373,6 @@ dummy_func(
ERROR_IF(err, error);
}
// stack effect: (__0 -- )
inst(BINARY_SUBSCR_ADAPTIVE) {
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *sub = TOP();
PyObject *container = SECOND();
next_instr--;
if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(BINARY_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(BINARY_SUBSCR);
}
}
// stack effect: (__0 -- )
inst(BINARY_SUBSCR_LIST_INT) {
assert(cframe.use_tracing == 0);
@ -511,9 +501,17 @@ dummy_func(
}
inst(STORE_SUBSCR, (v, container, sub -- )) {
int err;
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
/* container[sub] = v */
err = PyObject_SetItem(container, sub, v);
int err = PyObject_SetItem(container, sub, v);
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
@ -521,25 +519,6 @@ dummy_func(
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
}
// stack effect: (__0, __1, __2 -- )
inst(STORE_SUBSCR_ADAPTIVE) {
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *sub = TOP();
PyObject *container = SECOND();
next_instr--;
if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(STORE_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(STORE_SUBSCR);
}
}
// stack effect: (__0, __1, __2 -- )
inst(STORE_SUBSCR_LIST_INT) {
assert(cframe.use_tracing == 0);
@ -1088,6 +1067,16 @@ dummy_func(
// stack effect: (__0 -- __array[oparg])
inst(UNPACK_SEQUENCE) {
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *seq = TOP();
next_instr--;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(UNPACK_SEQUENCE, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
PyObject *seq = POP();
PyObject **top = stack_pointer + oparg;
if (!unpack_iterable(tstate, seq, oparg, -1, top)) {
@ -1099,23 +1088,6 @@ dummy_func(
JUMPBY(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
}
// stack effect: (__0 -- __array[oparg])
inst(UNPACK_SEQUENCE_ADAPTIVE) {
assert(cframe.use_tracing == 0);
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *seq = TOP();
next_instr--;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(UNPACK_SEQUENCE, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(UNPACK_SEQUENCE);
}
}
// stack effect: (__0 -- __array[oparg])
inst(UNPACK_SEQUENCE_TWO_TUPLE) {
PyObject *seq = TOP();
@ -1173,6 +1145,19 @@ dummy_func(
// stack effect: (__0, __1 -- )
inst(STORE_ATTR) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg);
next_instr--;
if (_Py_Specialize_StoreAttr(owner, next_instr, name)) {
goto error;
}
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
PyObject *name = GETITEM(names, oparg);
PyObject *owner = TOP();
PyObject *v = SECOND();
@ -1289,6 +1274,16 @@ dummy_func(
// error: LOAD_GLOBAL has irregular stack effect
inst(LOAD_GLOBAL) {
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_GLOBAL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
int push_null = oparg & 1;
PEEK(0) = NULL;
PyObject *name = GETITEM(names, oparg>>1);
@ -1339,25 +1334,6 @@ dummy_func(
PUSH(v);
}
// error: LOAD_GLOBAL has irregular stack effect
inst(LOAD_GLOBAL_ADAPTIVE) {
assert(cframe.use_tracing == 0);
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(LOAD_GLOBAL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(LOAD_GLOBAL);
}
}
// error: LOAD_GLOBAL has irregular stack effect
inst(LOAD_GLOBAL_MODULE) {
assert(cframe.use_tracing == 0);
@ -1752,6 +1728,19 @@ dummy_func(
// error: LOAD_ATTR has irregular stack effect
inst(LOAD_ATTR) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadAttr(owner, next_instr, name)) {
goto error;
}
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
PyObject *name = GETITEM(names, oparg >> 1);
PyObject *owner = TOP();
if (oparg & 1) {
@ -1798,26 +1787,6 @@ dummy_func(
JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR);
}
// error: LOAD_ATTR has irregular stack effect
inst(LOAD_ATTR_ADAPTIVE) {
assert(cframe.use_tracing == 0);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(LOAD_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(LOAD_ATTR);
}
}
// error: LOAD_ATTR has irregular stack effect
inst(LOAD_ATTR_INSTANCE_VALUE) {
assert(cframe.use_tracing == 0);
@ -2009,7 +1978,7 @@ dummy_func(
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(names, oparg >> 1);
@ -2033,26 +2002,6 @@ dummy_func(
goto start_frame;
}
// stack effect: (__0, __1 -- )
inst(STORE_ATTR_ADAPTIVE) {
assert(cframe.use_tracing == 0);
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg);
next_instr--;
if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(STORE_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(STORE_ATTR);
}
}
// stack effect: (__0, __1 -- )
inst(STORE_ATTR_INSTANCE_VALUE) {
assert(cframe.use_tracing == 0);
@ -2093,7 +2042,7 @@ dummy_func(
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR);
DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
@ -2155,7 +2104,7 @@ dummy_func(
}
// stack effect: (__0 -- )
inst(COMPARE_OP) {
inst(COMPARE_OP_GENERIC) {
assert(oparg <= Py_GE);
PyObject *right = POP();
PyObject *left = TOP();
@ -2170,21 +2119,19 @@ dummy_func(
}
// stack effect: (__0 -- )
inst(COMPARE_OP_ADAPTIVE) {
assert(cframe.use_tracing == 0);
inst(COMPARE_OP) {
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *right = TOP();
PyObject *left = SECOND();
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(COMPARE_OP);
}
STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
GO_TO_INSTRUCTION(COMPARE_OP_GENERIC);
}
// stack effect: (__0 -- )
@ -2665,6 +2612,15 @@ dummy_func(
// stack effect: ( -- __0)
inst(FOR_ITER) {
_PyForIterCache *cache = (_PyForIterCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_ForIter(TOP(), next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(FOR_ITER, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
/* before: [iter]; after: [iter, iter()] *or* [] */
PyObject *iter = TOP();
PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
@ -2690,22 +2646,6 @@ dummy_func(
JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1);
}
// stack effect: ( -- __0)
inst(FOR_ITER_ADAPTIVE) {
assert(cframe.use_tracing == 0);
_PyForIterCache *cache = (_PyForIterCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
next_instr--;
_Py_Specialize_ForIter(TOP(), next_instr, oparg);
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(FOR_ITER, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(FOR_ITER);
}
}
// stack effect: ( -- __0)
inst(FOR_ITER_LIST) {
assert(cframe.use_tracing == 0);
@ -3018,6 +2958,18 @@ dummy_func(
// stack effect: (__0, __array[oparg] -- )
inst(CALL) {
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
next_instr--;
_Py_Specialize_Call(callable, next_instr, nargs, call_shape.kwnames);
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
int total_args, is_meth;
is_meth = is_method(stack_pointer, oparg);
PyObject *function = PEEK(oparg + 1);
@ -3091,28 +3043,6 @@ dummy_func(
CHECK_EVAL_BREAKER();
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_ADAPTIVE) {
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
next_instr--;
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
int err = _Py_Specialize_Call(callable, next_instr, nargs,
call_shape.kwnames);
if (err < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(CALL);
}
}
// stack effect: (__0, __array[oparg] -- )
inst(CALL_PY_EXACT_ARGS) {
assert(call_shape.kwnames == NULL);
@ -3809,7 +3739,7 @@ dummy_func(
}
// stack effect: (__0 -- )
inst(BINARY_OP) {
inst(BINARY_OP_GENERIC) {
PyObject *rhs = POP();
PyObject *lhs = TOP();
assert(0 <= oparg);
@ -3826,21 +3756,19 @@ dummy_func(
}
// stack effect: (__0 -- )
inst(BINARY_OP_ADAPTIVE) {
assert(cframe.use_tracing == 0);
inst(BINARY_OP) {
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
assert(cframe.use_tracing == 0);
PyObject *lhs = SECOND();
PyObject *rhs = TOP();
next_instr--;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
GO_TO_INSTRUCTION(BINARY_OP);
}
STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
GO_TO_INSTRUCTION(BINARY_OP_GENERIC);
}
// stack effect: ( -- )
@ -3854,28 +3782,10 @@ dummy_func(
// stack effect: ( -- )
inst(EXTENDED_ARG) {
assert(oparg);
oparg <<= 8;
oparg |= _Py_OPARG(*next_instr);
// We might be tracing. To avoid breaking tracing guarantees in
// quickened instructions, always deoptimize the next opcode:
opcode = _PyOpcode_Deopt[_Py_OPCODE(*next_instr)];
PRE_DISPATCH_GOTO();
// CPython hasn't traced the following instruction historically
// (DO_TRACING would clobber our extended oparg anyways), so just
// skip our usual cframe.use_tracing check before dispatch. Also,
// make sure the next instruction isn't a RESUME, since that needs
// to trace properly (and shouldn't have an extended arg anyways):
assert(opcode != RESUME);
DISPATCH_GOTO();
}
// stack effect: ( -- )
inst(EXTENDED_ARG_QUICK) {
assert(cframe.use_tracing == 0);
assert(oparg);
int oldoparg = oparg;
NEXTOPARG();
oparg |= oldoparg << 8;
opcode = _Py_OPCODE(*next_instr);
oparg = oparg << 8 | _Py_OPARG(*next_instr);
PRE_DISPATCH_GOTO();
DISPATCH_GOTO();
}
@ -3901,15 +3811,15 @@ dummy_func(
// Families go below this point //
family(binary_op) = {
BINARY_OP, BINARY_OP_ADAPTIVE, BINARY_OP_ADD_FLOAT,
BINARY_OP_ADD_INT, BINARY_OP_ADD_UNICODE, BINARY_OP_INPLACE_ADD_UNICODE,
BINARY_OP, BINARY_OP_ADD_FLOAT,
BINARY_OP_ADD_INT, BINARY_OP_ADD_UNICODE, BINARY_OP_GENERIC, BINARY_OP_INPLACE_ADD_UNICODE,
BINARY_OP_MULTIPLY_FLOAT, BINARY_OP_MULTIPLY_INT, BINARY_OP_SUBTRACT_FLOAT,
BINARY_OP_SUBTRACT_INT };
family(binary_subscr) = {
BINARY_SUBSCR, BINARY_SUBSCR_ADAPTIVE, BINARY_SUBSCR_DICT,
BINARY_SUBSCR, BINARY_SUBSCR_DICT,
BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, BINARY_SUBSCR_TUPLE_INT };
family(call) = {
CALL, CALL_ADAPTIVE, CALL_PY_EXACT_ARGS,
CALL, CALL_PY_EXACT_ARGS,
CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS,
CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST,
CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN,
@ -3917,14 +3827,13 @@ family(call) = {
CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1,
CALL_NO_KW_TYPE_1 };
family(compare_op) = {
COMPARE_OP, COMPARE_OP_ADAPTIVE, COMPARE_OP_FLOAT_JUMP,
COMPARE_OP, COMPARE_OP_FLOAT_JUMP, COMPARE_OP_GENERIC,
COMPARE_OP_INT_JUMP, COMPARE_OP_STR_JUMP };
family(extended_arg) = { EXTENDED_ARG, EXTENDED_ARG_QUICK };
family(for_iter) = {
FOR_ITER, FOR_ITER_ADAPTIVE, FOR_ITER_LIST,
FOR_ITER, FOR_ITER_LIST,
FOR_ITER_RANGE };
family(load_attr) = {
LOAD_ATTR, LOAD_ATTR_ADAPTIVE, LOAD_ATTR_CLASS,
LOAD_ATTR, LOAD_ATTR_CLASS,
LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_MODULE,
LOAD_ATTR_PROPERTY, LOAD_ATTR_SLOT, LOAD_ATTR_WITH_HINT,
LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_METHOD_NO_DICT, LOAD_ATTR_METHOD_WITH_DICT,
@ -3932,15 +3841,15 @@ family(load_attr) = {
family(load_const) = { LOAD_CONST, LOAD_CONST__LOAD_FAST };
family(load_fast) = { LOAD_FAST, LOAD_FAST__LOAD_CONST, LOAD_FAST__LOAD_FAST };
family(load_global) = {
LOAD_GLOBAL, LOAD_GLOBAL_ADAPTIVE, LOAD_GLOBAL_BUILTIN,
LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN,
LOAD_GLOBAL_MODULE };
family(store_attr) = {
STORE_ATTR, STORE_ATTR_ADAPTIVE, STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR, STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT };
family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST };
family(store_subscr) = {
STORE_SUBSCR, STORE_SUBSCR_ADAPTIVE, STORE_SUBSCR_DICT,
STORE_SUBSCR, STORE_SUBSCR_DICT,
STORE_SUBSCR_LIST_INT };
family(unpack_sequence) = {
UNPACK_SEQUENCE, UNPACK_SEQUENCE_ADAPTIVE, UNPACK_SEQUENCE_LIST,
UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST,
UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE };