mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-26 19:55:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-115999: Specialize loading attributes from modules in free-threaded builds (#127711)

Browse source 
We use the same approach that was used for specialization of LOAD_GLOBAL in free-threaded builds:

_CHECK_ATTR_MODULE is renamed to _CHECK_ATTR_MODULE_PUSH_KEYS; it pushes the keys object for the following _LOAD_ATTR_MODULE_FROM_KEYS (nee _LOAD_ATTR_MODULE). This arrangement avoids having to recheck the keys version.

_LOAD_ATTR_MODULE is renamed to _LOAD_ATTR_MODULE_FROM_KEYS; it loads the value from the keys object pushed by the preceding _CHECK_ATTR_MODULE_PUSH_KEYS at the cached index.
This commit is contained in:
mpage 2024-12-13 10:17:16 -08:00 • committed by GitHub
parent 292067fbc9
commit 2de048ce79
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 435 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

55
Python/bytecodes.c
View file

@ -2070,7 +2070,7 @@ dummy_func(
};
specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
@ -2079,7 +2079,7 @@ dummy_func(
}
OPCODE_DEFERRED_INC(LOAD_ATTR);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) {
@ -2158,33 +2158,43 @@ dummy_func(
_LOAD_ATTR_INSTANCE_VALUE +
unused/5; // Skip over rest of cache
op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) {
op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys: PyDictKeysObject *)) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != dict_version);
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version);
mod_keys = keys;
}
op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
PyObject *attr_o = ep->me_value;
op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys: PyDictKeysObject * -- attr, null if (oparg & 1))) {
assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
DEAD(mod_keys);
// Clear mod_keys from stack in case we need to deopt
POP_DEAD_INPUTS();
DEOPT_IF(attr_o == NULL);
STAT_INC(LOAD_ATTR, hit);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
if (!increfed) {
DEOPT_IF(true);
}
#else
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
DECREF_INPUTS();
PyStackRef_CLOSE(owner);
}
macro(LOAD_ATTR_MODULE) =
unused/1 +
_CHECK_ATTR_MODULE +
_LOAD_ATTR_MODULE +
_CHECK_ATTR_MODULE_PUSH_KEYS +
_LOAD_ATTR_MODULE_FROM_KEYS +
unused/5;
op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
@ -4963,6 +4973,21 @@ dummy_func(
null = PyStackRef_NULL;
}
tier2 op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
PyObject *attr_o = ep->me_value;
DEOPT_IF(attr_o == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
DECREF_INPUTS();
}
/* Internal -- for testing executors */
op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) {
_PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt);

70
Python/executor_cases.c.h generated
View file

@ -2641,8 +2641,9 @@
/* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */
case _CHECK_ATTR_MODULE: {
case _CHECK_ATTR_MODULE_PUSH_KEYS: {
_PyStackRef owner;
PyDictKeysObject *mod_keys;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
@ -2652,33 +2653,51 @@
}
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
if (dict->ma_keys->dk_version != dict_version) {
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
mod_keys = keys;
stack_pointer[0].bits = (uintptr_t)mod_keys;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_MODULE: {
case _LOAD_ATTR_MODULE_FROM_KEYS: {
PyDictKeysObject *mod_keys;
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
mod_keys = (PyDictKeysObject *)stack_pointer[-1].bits;
owner = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
PyObject *attr_o = ep->me_value;
assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
// Clear mod_keys from stack in case we need to deopt
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
if (!increfed) {
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
#else
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
@ -5928,6 +5947,35 @@
break;
}
case _LOAD_ATTR_MODULE: {
_PyStackRef owner;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND0();
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
PyObject *attr_o = ep->me_value;
if (attr_o == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INTERNAL_INCREMENT_OPT_COUNTER: {
_PyStackRef opt;
opt = stack_pointer[-1];

33
Python/generated_cases.c.h generated
View file

@ -5200,7 +5200,7 @@
owner = stack_pointer[-1];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
@ -5211,7 +5211,7 @@
}
OPCODE_DEFERRED_INC(LOAD_ATTR);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
#endif /* ENABLE_SPECIALIZATION_FT */
}
/* Skip 8 cache entries */
// _LOAD_ATTR
@ -5553,10 +5553,11 @@
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
PyDictKeysObject *mod_keys;
_PyStackRef attr;
_PyStackRef null = PyStackRef_NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_MODULE
// _CHECK_ATTR_MODULE_PUSH_KEYS
{
owner = stack_pointer[-1];
uint32_t dict_version = read_u32(&this_instr[2].cache);
@ -5564,21 +5565,29 @@
DEOPT_IF(Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro, LOAD_ATTR);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR);
PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys);
DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version, LOAD_ATTR);
mod_keys = keys;
}
// _LOAD_ATTR_MODULE
// _LOAD_ATTR_MODULE_FROM_KEYS
{
uint16_t index = read_u16(&this_instr[4].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
PyObject *attr_o = ep->me_value;
assert(mod_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries));
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index;
PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value);
// Clear mod_keys from stack in case we need to deopt
DEOPT_IF(attr_o == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
#ifdef Py_GIL_DISABLED
int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr);
if (!increfed) {
DEOPT_IF(true, LOAD_ATTR);
}
#else
Py_INCREF(attr_o);
attr = PyStackRef_FromPyObjectSteal(attr_o);
#endif
STAT_INC(LOAD_ATTR, hit);
null = PyStackRef_NULL;
PyStackRef_CLOSE(owner);
}

2
Python/optimizer_analysis.c
View file

@ -95,7 +95,7 @@ type_watcher_callback(PyTypeObject* type)
static PyObject *
convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj)
{
assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE);
assert(inst->opcode == _LOAD_GLOBAL_MODULE || inst->opcode == _LOAD_GLOBAL_BUILTINS || inst->opcode == _LOAD_ATTR_MODULE_FROM_KEYS);
assert(PyDict_CheckExact(obj));
PyDictObject *dict = (PyDictObject *)obj;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);

10
Python/optimizer_bytecodes.c
View file

@ -492,8 +492,9 @@ dummy_func(void) {
(void)owner;
}
op(_CHECK_ATTR_MODULE, (dict_version/2, owner -- owner)) {
op(_CHECK_ATTR_MODULE_PUSH_KEYS, (dict_version/2, owner -- owner, mod_keys)) {
(void)dict_version;
mod_keys = sym_new_not_null(ctx);
if (sym_is_const(owner)) {
PyObject *cnst = sym_get_const(owner);
if (PyModule_CheckExact(cnst)) {
@ -515,12 +516,12 @@ dummy_func(void) {
self_or_null = sym_new_unknown(ctx);
}
op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) {
op(_LOAD_ATTR_MODULE_FROM_KEYS, (index/1, owner, mod_keys -- attr, null if (oparg & 1))) {
(void)index;
null = sym_new_null(ctx);
attr = NULL;
if (this_instr[-1].opcode == _NOP) {
// Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched.
// Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched.
assert(sym_is_const(owner));
PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
assert(PyModule_CheckExact(mod));
@ -530,6 +531,9 @@ dummy_func(void) {
this_instr[-1].opcode = _POP_TOP;
attr = sym_new_const(ctx, res);
}
else {
this_instr->opcode = _LOAD_ATTR_MODULE;
}
}
if (attr == NULL) {
/* No conversion made. We don't know what `attr` is. */

47
Python/optimizer_cases.c.h generated
View file

@ -1134,61 +1134,74 @@
break;
}
case _CHECK_ATTR_MODULE: {
case _CHECK_ATTR_MODULE_PUSH_KEYS: {
_Py_UopsSymbol *owner;
_Py_UopsSymbol *mod_keys;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)this_instr->operand0;
(void)dict_version;
mod_keys = sym_new_not_null(ctx);
if (sym_is_const(owner)) {
PyObject *cnst = sym_get_const(owner);
if (PyModule_CheckExact(cnst)) {
PyModuleObject *mod = (PyModuleObject *)cnst;
PyObject *dict = mod->md_dict;
stack_pointer[0] = mod_keys;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
uint64_t watched_mutations = get_mutations(dict);
if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
PyDict_Watch(GLOBALS_WATCHER_ID, dict);
_Py_BloomFilter_Add(dependencies, dict);
this_instr->opcode = _NOP;
}
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
}
stack_pointer[0] = mod_keys;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
case _LOAD_ATTR_MODULE: {
case _LOAD_ATTR_MODULE_FROM_KEYS: {
_Py_UopsSymbol *owner;
_Py_UopsSymbol *attr;
_Py_UopsSymbol *null = NULL;
owner = stack_pointer[-1];
owner = stack_pointer[-2];
uint16_t index = (uint16_t)this_instr->operand0;
(void)index;
null = sym_new_null(ctx);
attr = NULL;
if (this_instr[-1].opcode == _NOP) {
// Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched.
// Preceding _CHECK_ATTR_MODULE_PUSH_KEYS was removed: mod is const and dict is watched.
assert(sym_is_const(owner));
PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner);
assert(PyModule_CheckExact(mod));
PyObject *dict = mod->md_dict;
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
stack_pointer[-2] = attr;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
PyObject *res = convert_global_to_const(this_instr, dict);
if (res != NULL) {
this_instr[-1].opcode = _POP_TOP;
attr = sym_new_const(ctx, res);
}
stack_pointer += -(oparg & 1);
else {
this_instr->opcode = _LOAD_ATTR_MODULE;
}
stack_pointer += 1 - (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
}
if (attr == NULL) {
/* No conversion made. We don't know what `attr` is. */
attr = sym_new_not_null(ctx);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
stack_pointer[-2] = attr;
if (oparg & 1) stack_pointer[-1] = null;
stack_pointer += -1 + (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
@ -2528,6 +2541,18 @@
break;
}
case _LOAD_ATTR_MODULE: {
_Py_UopsSymbol *attr;
_Py_UopsSymbol *null = NULL;
attr = sym_new_not_null(ctx);
null = sym_new_null(ctx);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
assert(WITHIN_STACK_BOUNDS());
break;
}
case _INTERNAL_INCREMENT_OPT_COUNTER: {
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());

92
Python/specialize.c
View file

@ -738,22 +738,16 @@ unspecialize(_Py_CODEUNIT *instr)
}
static int function_kind(PyCodeObject *code);
#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
#endif
static uint32_t type_get_version(PyTypeObject *t, int opcode);
static int
specialize_module_load_attr(
PyObject *owner, _Py_CODEUNIT *instr, PyObject *name
) {
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyModuleObject *m = (PyModuleObject *)owner;
assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
PyDictObject *dict = (PyDictObject *)m->md_dict;
if (dict == NULL) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);
return -1;
}
if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_STRING);
return -1;
@ -773,19 +767,35 @@ specialize_module_load_attr(
SPEC_FAIL_OUT_OF_RANGE);
return -1;
}
uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(
_PyInterpreterState_GET(), dict->ma_keys);
uint32_t keys_version = _PyDict_GetKeysVersionForCurrentState(
_PyInterpreterState_GET(), dict);
if (keys_version == 0) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
write_u32(cache->version, keys_version);
cache->index = (uint16_t)index;
instr->op.code = LOAD_ATTR_MODULE;
specialize(instr, LOAD_ATTR_MODULE);
return 0;
}
static int
specialize_module_load_attr(
PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
PyModuleObject *m = (PyModuleObject *)owner;
assert((Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
PyDictObject *dict = (PyDictObject *)m->md_dict;
if (dict == NULL) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NO_DICT);
return -1;
}
int result;
Py_BEGIN_CRITICAL_SECTION(dict);
result = specialize_module_load_attr_lock_held(dict, instr, name);
Py_END_CRITICAL_SECTION();
return result;
}
/* Attribute specialization */
@ -968,7 +978,7 @@ specialize_dict_access(
}
write_u32(cache->version, type->tp_version_tag);
cache->index = (uint16_t)offset;
instr->op.code = values_op;
specialize(instr, values_op);
}
else {
PyDictObject *dict = _PyObject_GetManagedDict(owner);
@ -992,11 +1002,12 @@ specialize_dict_access(
}
cache->index = (uint16_t)index;
write_u32(cache->version, type->tp_version_tag);
instr->op.code = hint_op;
specialize(instr, hint_op);
}
return 1;
}
#ifndef Py_GIL_DISABLED
static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name,
PyObject* descr, DescriptorClassification kind, bool is_method);
static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name);
@ -1093,7 +1104,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
write_u32(lm_cache->type_version, type->tp_version_tag);
/* borrowed */
write_obj(lm_cache->descr, fget);
instr->op.code = LOAD_ATTR_PROPERTY;
specialize(instr, LOAD_ATTR_PROPERTY);
return 0;
}
case OBJECT_SLOT:
@ -1117,7 +1128,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
assert(offset > 0);
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
instr->op.code = LOAD_ATTR_SLOT;
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
case DUNDER_CLASS:
@ -1126,7 +1137,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
assert(offset == (uint16_t)offset);
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
instr->op.code = LOAD_ATTR_SLOT;
specialize(instr, LOAD_ATTR_SLOT);
return 0;
}
case OTHER_SLOT:
@ -1162,7 +1173,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
/* borrowed */
write_obj(lm_cache->descr, descr);
write_u32(lm_cache->type_version, type->tp_version_tag);
instr->op.code = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN;
specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
return 0;
}
case BUILTIN_CLASSMETHOD:
@ -1186,6 +1197,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
if (shadow) {
goto try_instance;
}
set_counter((_Py_BackoffCounter*)instr + 1, adaptive_counter_cooldown());
return 0;
}
Py_UNREACHABLE();
@ -1197,14 +1209,14 @@ try_instance:
}
return -1;
}
#endif // Py_GIL_DISABLED
void
_Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
assert(ENABLE_SPECIALIZATION);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
PyTypeObject *type = Py_TYPE(owner);
bool fail;
@ -1219,22 +1231,24 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam
fail = specialize_module_load_attr(owner, instr, name);
}
else if (PyType_Check(owner)) {
#ifdef Py_GIL_DISABLED
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
fail = true;
#else
fail = specialize_class_load_attr(owner, instr, name);
#endif
}
else {
#ifdef Py_GIL_DISABLED
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
fail = true;
#else
fail = specialize_instance_load_attr(owner, instr, name);
#endif
}
if (fail) {
STAT_INC(LOAD_ATTR, failure);
assert(!PyErr_Occurred());
instr->op.code = LOAD_ATTR;
cache->counter = adaptive_counter_backoff(cache->counter);
}
else {
STAT_INC(LOAD_ATTR, success);
assert(!PyErr_Occurred());
cache->counter = adaptive_counter_cooldown();
unspecialize(instr);
}
}
@ -1339,6 +1353,7 @@ success:
cache->counter = adaptive_counter_cooldown();
}
#ifndef Py_GIL_DISABLED
#ifdef Py_STATS
static int
@ -1422,10 +1437,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
write_obj(cache->descr, descr);
if (metaclass_check) {
write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag);
instr->op.code = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK;
specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK);
}
else {
instr->op.code = LOAD_ATTR_CLASS;
specialize(instr, LOAD_ATTR_CLASS);
}
return 0;
#ifdef Py_STATS
@ -1461,7 +1476,7 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
return 0;
}
write_u32(cache->keys_version, keys_version);
instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
}
else {
Py_ssize_t dictoffset;
@ -1476,7 +1491,7 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
}
}
if (dictoffset == 0) {
instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
specialize(instr, is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
}
else if (is_method) {
PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
@ -1490,7 +1505,7 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
dictoffset -= MANAGED_DICT_OFFSET;
assert(((uint16_t)dictoffset) == dictoffset);
cache->dict_offset = (uint16_t)dictoffset;
instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT;
specialize(instr, LOAD_ATTR_METHOD_LAZY_DICT);
}
else {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
@ -1516,6 +1531,9 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
return 1;
}
#endif // Py_GIL_DISABLED
static void
specialize_load_global_lock_held(
PyObject *globals, PyObject *builtins,
@ -1661,6 +1679,7 @@ function_kind(PyCodeObject *code) {
return SIMPLE_FUNCTION;
}
#ifndef Py_GIL_DISABLED
/* Returning false indicates a failure. */
static bool
function_check_args(PyObject *o, int expected_argcount, int opcode)
@ -1693,6 +1712,7 @@ function_get_version(PyObject *o, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
/* Returning 0 indicates a failure. */
static uint32_t