GH-128914: Remove all but one conditional stack effects (GH-129226)

* Remove all 'if (0)' and 'if (1)' conditional stack effects

* Use array instead of conditional for BUILD_SLICE args

* Refactor LOAD_GLOBAL to use a common conditional uop

* Remove conditional stack effects from LOAD_ATTR specializations

* Replace conditional stack effects in LOAD_ATTR with a 0 or 1 sized array.

* Remove conditional stack effects from CALL_FUNCTION_EX
This commit is contained in:
Mark Shannon 2025-01-27 16:24:48 +00:00 committed by GitHub
parent 8ec76d9034
commit 75b4962157
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 518 additions and 758 deletions

View file

@ -1953,7 +1953,6 @@
case _LOAD_GLOBAL: {
_PyStackRef *res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
res = &stack_pointer[0];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
@ -1961,9 +1960,17 @@
_PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (PyStackRef_IsNull(*res)) JUMP_TO_ERROR();
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _PUSH_NULL_CONDITIONAL: {
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
null = PyStackRef_NULL;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2029,8 +2036,6 @@
case _LOAD_GLOBAL_MODULE_FROM_KEYS: {
PyDictKeysObject *globals_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
@ -2052,10 +2057,8 @@
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2063,8 +2066,6 @@
case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: {
PyDictKeysObject *builtins_keys;
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
@ -2086,10 +2087,8 @@
res = PyStackRef_FromPyObjectSteal(res_o);
#endif
STAT_INC(LOAD_GLOBAL, hit);
null = PyStackRef_NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2608,9 +2607,10 @@
case _LOAD_ATTR: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self_or_null = PyStackRef_NULL;
_PyStackRef *self_or_null;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
self_or_null = &stack_pointer[0];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
PyObject *attr_o;
if (oparg & 1) {
@ -2625,7 +2625,7 @@
meth | self | arg1 | ... | argN
*/
assert(attr_o != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
self_or_null[0] = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
@ -2636,7 +2636,7 @@
*/
PyStackRef_CLOSE(owner);
if (attr_o == NULL) JUMP_TO_ERROR();
self_or_null = PyStackRef_NULL;
self_or_null[0] = PyStackRef_NULL;
}
}
else {
@ -2646,13 +2646,10 @@
stack_pointer = _PyFrame_GetStackPointer(frame);
PyStackRef_CLOSE(owner);
if (attr_o == NULL) JUMP_TO_ERROR();
/* We need to define self_or_null on all paths */
self_or_null = PyStackRef_NULL;
}
attr = PyStackRef_FromPyObjectSteal(attr_o);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
stack_pointer += (oparg&1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2704,11 +2701,9 @@
break;
}
case _LOAD_ATTR_INSTANCE_VALUE_0: {
case _LOAD_ATTR_INSTANCE_VALUE: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -2729,48 +2724,11 @@
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_INSTANCE_VALUE_1: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t offset = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) {
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
/* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */
case _CHECK_ATTR_MODULE_PUSH_KEYS: {
_PyStackRef owner;
PyDictKeysObject *mod_keys;
@ -2799,8 +2757,6 @@
PyDictKeysObject *mod_keys;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
owner = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
@ -2828,12 +2784,8 @@
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2860,7 +2812,6 @@
PyDictObject *dict;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
dict = (PyDictObject *)stack_pointer[-1].bits;
owner = stack_pointer[-2];
@ -2916,20 +2867,16 @@
STAT_INC(LOAD_ATTR, hit);
attr = PyStackRef_FromPyObjectNew(attr_o);
UNLOCK_OBJECT(dict);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-2] = attr;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_SLOT_0: {
case _LOAD_ATTR_SLOT: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -2949,47 +2896,11 @@
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_SLOT_1: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyObject **addr = (PyObject **)((char *)owner_o + index);
PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr);
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr);
if (!increfed) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
#else
attr = PyStackRef_FromPyObjectNew(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
/* _LOAD_ATTR_SLOT is split on (oparg & 1) */
case _CHECK_ATTR_CLASS: {
_PyStackRef owner;
owner = stack_pointer[-1];
@ -3007,43 +2918,19 @@
break;
}
case _LOAD_ATTR_CLASS_0: {
case _LOAD_ATTR_CLASS: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_CLASS_1: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = PyStackRef_FromPyObjectNew(descr);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
/* _LOAD_ATTR_CLASS is split on (oparg & 1) */
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyStackRef owner;
_PyInterpreterFrame *new_frame;
@ -4059,7 +3946,7 @@
case _LOAD_ATTR_METHOD_WITH_VALUES: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
_PyStackRef self;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
@ -4080,7 +3967,7 @@
case _LOAD_ATTR_METHOD_NO_DICT: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
_PyStackRef self;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
@ -4146,7 +4033,7 @@
case _LOAD_ATTR_METHOD_LAZY_DICT: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef self = PyStackRef_NULL;
_PyStackRef self;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND0();
@ -5644,15 +5531,14 @@
/* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */
case _MAKE_CALLARGS_A_TUPLE: {
_PyStackRef kwargs_in = PyStackRef_NULL;
_PyStackRef kwargs_in;
_PyStackRef callargs;
_PyStackRef func;
_PyStackRef tuple;
_PyStackRef kwargs_out = PyStackRef_NULL;
oparg = CURRENT_OPARG();
if (oparg & 1) { kwargs_in = stack_pointer[-(oparg & 1)]; }
callargs = stack_pointer[-1 - (oparg & 1)];
func = stack_pointer[-3 - (oparg & 1)];
_PyStackRef kwargs_out;
kwargs_in = stack_pointer[-1];
callargs = stack_pointer[-2];
func = stack_pointer[-4];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (PyTuple_CheckExact(callargs_o)) {
tuple = callargs;
@ -5675,8 +5561,8 @@
PyStackRef_CLOSE(callargs);
tuple = PyStackRef_FromPyObjectSteal(tuple_o);
}
stack_pointer[-1 - (oparg & 1)] = tuple;
if (oparg & 1) stack_pointer[-(oparg & 1)] = kwargs_out;
stack_pointer[-2] = tuple;
stack_pointer[-1] = kwargs_out;
break;
}
@ -5753,25 +5639,21 @@
}
case _BUILD_SLICE: {
_PyStackRef step = PyStackRef_NULL;
_PyStackRef stop;
_PyStackRef start;
_PyStackRef *args;
_PyStackRef slice;
oparg = CURRENT_OPARG();
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
PyObject *start_o = PyStackRef_AsPyObjectBorrow(start);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop);
PyObject *step_o = PyStackRef_AsPyObjectBorrow(step);
args = &stack_pointer[-oparg];
PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]);
PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]);
PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL;
PyObject *slice_o = PySlice_New(start_o, stop_o, step_o);
PyStackRef_CLOSE(start);
PyStackRef_CLOSE(stop);
PyStackRef_XCLOSE(step);
for (int _i = oparg; --_i >= 0;) {
PyStackRef_CLOSE(args[_i]);
}
if (slice_o == NULL) JUMP_TO_ERROR();
slice = PyStackRef_FromPyObjectSteal(slice_o);
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
stack_pointer[-oparg] = slice;
stack_pointer += 1 - oparg;
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -6099,32 +5981,6 @@
break;
}
case _LOAD_CONST_INLINE_WITH_NULL: {
_PyStackRef value;
_PyStackRef null;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
value = PyStackRef_FromPyObjectNew(ptr);
null = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer[1] = null;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_CONST_INLINE_BORROW_WITH_NULL: {
_PyStackRef value;
_PyStackRef null;
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
value = PyStackRef_FromPyObjectImmortal(ptr);
null = PyStackRef_NULL;
stack_pointer[0] = value;
stack_pointer[1] = null;
stack_pointer += 2;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND0();
assert(PyStackRef_FunctionCheck(frame->f_funcobj));
@ -6138,8 +5994,6 @@
case _LOAD_GLOBAL_MODULE: {
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
@ -6150,18 +6004,14 @@
}
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
null = PyStackRef_NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_GLOBAL_BUILTINS: {
_PyStackRef res;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyDictObject *dict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
@ -6172,10 +6022,8 @@
}
Py_INCREF(res_o);
res = PyStackRef_FromPyObjectSteal(res_o);
null = PyStackRef_NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -6183,8 +6031,6 @@
case _LOAD_ATTR_MODULE: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -6200,12 +6046,8 @@
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}