mirror of
https://github.com/python/cpython.git
synced 2025-08-23 10:16:01 +00:00
GH-130296: Avoid stack transients in four instructions. (GH-130310)
* Combine _GUARD_GLOBALS_VERSION_PUSH_KEYS and _LOAD_GLOBAL_MODULE_FROM_KEYS into _LOAD_GLOBAL_MODULE * Combine _GUARD_BUILTINS_VERSION_PUSH_KEYS and _LOAD_GLOBAL_BUILTINS_FROM_KEYS into _LOAD_GLOBAL_BUILTINS * Combine _CHECK_ATTR_MODULE_PUSH_KEYS and _LOAD_ATTR_MODULE_FROM_KEYS into _LOAD_ATTR_MODULE * Remove stack transient in LOAD_ATTR_WITH_HINT
This commit is contained in:
Mark Shannon
GitHub
parent
ab11c09705
commit
54965f3fb2
15 changed files with 615 additions and 1005 deletions
249
Python/executor_cases.c.h
generated
249
Python/executor_cases.c.h
generated
|
|
@ -2267,9 +2267,10 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _GUARD_GLOBALS_VERSION_PUSH_KEYS: {
|
||||
PyDictKeysObject *globals_keys;
|
||||
case _LOAD_GLOBAL_MODULE: {
|
||||
_PyStackRef res;
|
||||
uint16_t version = (uint16_t)CURRENT_OPERAND0();
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND1();
|
||||
PyDictObject *dict = (PyDictObject *)GLOBALS();
|
||||
if (!PyDict_CheckExact(dict)) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
|
|
@ -2280,17 +2281,34 @@
|
|||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
globals_keys = keys;
|
||||
assert(DK_IS_UNICODE(globals_keys));
|
||||
stack_pointer[0].bits = (uintptr_t)globals_keys;
|
||||
assert(DK_IS_UNICODE(keys));
|
||||
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
|
||||
assert(index < DK_SIZE(keys));
|
||||
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
|
||||
if (res_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
#if Py_GIL_DISABLED
|
||||
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
|
||||
if (!increfed) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
#else
|
||||
res = PyStackRef_FromPyObjectNew(res_o);
|
||||
#endif
|
||||
STAT_INC(LOAD_GLOBAL, hit);
|
||||
stack_pointer[0] = res;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _GUARD_BUILTINS_VERSION_PUSH_KEYS: {
|
||||
PyDictKeysObject *builtins_keys;
|
||||
case _LOAD_GLOBAL_BUILTINS: {
|
||||
_PyStackRef res;
|
||||
uint16_t version = (uint16_t)CURRENT_OPERAND0();
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND1();
|
||||
PyDictObject *dict = (PyDictObject *)BUILTINS();
|
||||
if (!PyDict_CheckExact(dict)) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
|
|
@ -2301,23 +2319,9 @@
|
|||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
builtins_keys = keys;
|
||||
assert(DK_IS_UNICODE(builtins_keys));
|
||||
stack_pointer[0].bits = (uintptr_t)builtins_keys;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_GLOBAL_MODULE_FROM_KEYS: {
|
||||
PyDictKeysObject *globals_keys;
|
||||
_PyStackRef res;
|
||||
globals_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND0();
|
||||
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(globals_keys);
|
||||
assert(DK_IS_UNICODE(keys));
|
||||
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
|
||||
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (res_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -2329,38 +2333,7 @@
|
|||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
#else
|
||||
Py_INCREF(res_o);
|
||||
res = PyStackRef_FromPyObjectSteal(res_o);
|
||||
#endif
|
||||
STAT_INC(LOAD_GLOBAL, hit);
|
||||
stack_pointer[0] = res;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_GLOBAL_BUILTINS_FROM_KEYS: {
|
||||
PyDictKeysObject *builtins_keys;
|
||||
_PyStackRef res;
|
||||
builtins_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND0();
|
||||
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(builtins_keys);
|
||||
PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (res_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
#if Py_GIL_DISABLED
|
||||
int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res);
|
||||
if (!increfed) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
#else
|
||||
Py_INCREF(res_o);
|
||||
res = PyStackRef_FromPyObjectSteal(res_o);
|
||||
res = PyStackRef_FromPyObjectNew(res_o);
|
||||
#endif
|
||||
STAT_INC(LOAD_GLOBAL, hit);
|
||||
stack_pointer[0] = res;
|
||||
|
|
@ -3143,11 +3116,12 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _CHECK_ATTR_MODULE_PUSH_KEYS: {
|
||||
case _LOAD_ATTR_MODULE: {
|
||||
_PyStackRef owner;
|
||||
PyDictKeysObject *mod_keys;
|
||||
_PyStackRef attr;
|
||||
owner = stack_pointer[-1];
|
||||
uint32_t dict_version = (uint32_t)CURRENT_OPERAND0();
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND1();
|
||||
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
|
||||
if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
|
|
@ -3160,27 +3134,10 @@
|
|||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
mod_keys = keys;
|
||||
stack_pointer[0].bits = (uintptr_t)mod_keys;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_ATTR_MODULE_FROM_KEYS: {
|
||||
PyDictKeysObject *mod_keys;
|
||||
_PyStackRef owner;
|
||||
_PyStackRef attr;
|
||||
mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
|
||||
owner = stack_pointer[-2];
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND0();
|
||||
assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
|
||||
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
|
||||
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
|
||||
assert(keys->dk_kind == DICT_KEYS_UNICODE);
|
||||
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries));
|
||||
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index;
|
||||
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
|
||||
// Clear mod_keys from stack in case we need to deopt
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (attr_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -3194,8 +3151,7 @@
|
|||
}
|
||||
}
|
||||
#else
|
||||
Py_INCREF(attr_o);
|
||||
attr = PyStackRef_FromPyObjectSteal(attr_o);
|
||||
attr = PyStackRef_FromPyObjectNew(attr_o);
|
||||
#endif
|
||||
STAT_INC(LOAD_ATTR, hit);
|
||||
stack_pointer[-1] = attr;
|
||||
|
|
@ -3205,37 +3161,22 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _CHECK_ATTR_WITH_HINT: {
|
||||
_PyStackRef owner;
|
||||
PyDictObject *dict;
|
||||
owner = stack_pointer[-1];
|
||||
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
|
||||
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
|
||||
PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o);
|
||||
if (dict_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
assert(PyDict_CheckExact((PyObject *)dict_o));
|
||||
dict = dict_o;
|
||||
stack_pointer[0].bits = (uintptr_t)dict;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_ATTR_WITH_HINT: {
|
||||
PyDictObject *dict;
|
||||
_PyStackRef owner;
|
||||
_PyStackRef attr;
|
||||
oparg = CURRENT_OPARG();
|
||||
dict = (PyDictObject *)stack_pointer[-1].bits;
|
||||
owner = stack_pointer[-2];
|
||||
owner = stack_pointer[-1];
|
||||
uint16_t hint = (uint16_t)CURRENT_OPERAND0();
|
||||
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
|
||||
assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
|
||||
PyDictObject *dict = _PyObject_GetManagedDict(owner_o);
|
||||
if (dict == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
assert(PyDict_CheckExact((PyObject *)dict));
|
||||
PyObject *attr_o;
|
||||
if (!LOCK_OBJECT(dict)) {
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (true) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -3243,8 +3184,6 @@
|
|||
}
|
||||
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
|
||||
UNLOCK_OBJECT(dict);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (true) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -3253,8 +3192,6 @@
|
|||
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
|
||||
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
|
||||
UNLOCK_OBJECT(dict);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (true) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -3263,8 +3200,6 @@
|
|||
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
|
||||
if (ep->me_key != name) {
|
||||
UNLOCK_OBJECT(dict);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (true) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -3273,8 +3208,6 @@
|
|||
attr_o = ep->me_value;
|
||||
if (attr_o == NULL) {
|
||||
UNLOCK_OBJECT(dict);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
if (true) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
|
|
@ -3283,15 +3216,10 @@
|
|||
STAT_INC(LOAD_ATTR, hit);
|
||||
attr = PyStackRef_FromPyObjectNew(attr_o);
|
||||
UNLOCK_OBJECT(dict);
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
_PyStackRef tmp = owner;
|
||||
owner = attr;
|
||||
stack_pointer[-1] = owner;
|
||||
PyStackRef_CLOSE(tmp);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
stack_pointer[-1] = attr;
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
PyStackRef_CLOSE(owner);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -3395,7 +3323,7 @@
|
|||
break;
|
||||
}
|
||||
|
||||
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has unused cache entries */
|
||||
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */
|
||||
|
||||
case _GUARD_DORV_NO_DICT: {
|
||||
_PyStackRef owner;
|
||||
|
|
@ -6833,6 +6761,23 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _POP_TOP_LOAD_CONST_INLINE: {
|
||||
_PyStackRef pop;
|
||||
_PyStackRef value;
|
||||
pop = stack_pointer[-1];
|
||||
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
|
||||
stack_pointer += -1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
PyStackRef_CLOSE(pop);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
value = PyStackRef_FromPyObjectNew(ptr);
|
||||
stack_pointer[0] = value;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_CONST_INLINE_BORROW: {
|
||||
_PyStackRef value;
|
||||
PyObject *ptr = (PyObject *)CURRENT_OPERAND0();
|
||||
|
|
@ -6871,70 +6816,6 @@
|
|||
break;
|
||||
}
|
||||
|
||||
case _LOAD_GLOBAL_MODULE: {
|
||||
_PyStackRef res;
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND0();
|
||||
PyDictObject *dict = (PyDictObject *)GLOBALS();
|
||||
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
|
||||
PyObject *res_o = entries[index].me_value;
|
||||
if (res_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
Py_INCREF(res_o);
|
||||
res = PyStackRef_FromPyObjectSteal(res_o);
|
||||
stack_pointer[0] = res;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_GLOBAL_BUILTINS: {
|
||||
_PyStackRef res;
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND0();
|
||||
PyDictObject *dict = (PyDictObject *)BUILTINS();
|
||||
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
|
||||
PyObject *res_o = entries[index].me_value;
|
||||
if (res_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
Py_INCREF(res_o);
|
||||
res = PyStackRef_FromPyObjectSteal(res_o);
|
||||
stack_pointer[0] = res;
|
||||
stack_pointer += 1;
|
||||
assert(WITHIN_STACK_BOUNDS());
|
||||
break;
|
||||
}
|
||||
|
||||
case _LOAD_ATTR_MODULE: {
|
||||
_PyStackRef owner;
|
||||
_PyStackRef attr;
|
||||
owner = stack_pointer[-1];
|
||||
uint16_t index = (uint16_t)CURRENT_OPERAND0();
|
||||
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
|
||||
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
|
||||
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
|
||||
assert(index < dict->ma_keys->dk_nentries);
|
||||
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
|
||||
PyObject *attr_o = ep->me_value;
|
||||
if (attr_o == NULL) {
|
||||
UOP_STAT_INC(uopcode, miss);
|
||||
JUMP_TO_JUMP_TARGET();
|
||||
}
|
||||
STAT_INC(LOAD_ATTR, hit);
|
||||
Py_INCREF(attr_o);
|
||||
attr = PyStackRef_FromPyObjectSteal(attr_o);
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
_PyStackRef tmp = owner;
|
||||
owner = attr;
|
||||
stack_pointer[-1] = owner;
|
||||
PyStackRef_CLOSE(tmp);
|
||||
stack_pointer = _PyFrame_GetStackPointer(frame);
|
||||
stack_pointer[-1] = attr;
|
||||
break;
|
||||
}
|
||||
|
||||
case _START_EXECUTOR: {
|
||||
PyObject *executor = (PyObject *)CURRENT_OPERAND0();
|
||||
_PyFrame_SetStackPointer(frame, stack_pointer);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue