mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-22 09:45:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

bpo-46841: Use inline caching for attribute accesses (GH-31640)

Browse source
This commit is contained in:
Brandt Bucher 2022-03-03 15:31:00 -08:00 • committed by GitHub
parent 65b92ccdec
commit 05a8bc1c94
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 292 additions and 270 deletions
Download patch file Download diff file Expand all files Collapse all files

120
Python/specialize.c
View file

@ -58,12 +58,9 @@ static uint8_t adaptive_opcodes[256] = {
/* The number of cache entries required for a "family" of instructions. */
static uint8_t cache_requirements[256] = {
[LOAD_ATTR] = 1, // _PyAdaptiveEntry
[LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
[STORE_SUBSCR] = 0,
[CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[STORE_ATTR] = 1, // _PyAdaptiveEntry
};
Py_ssize_t _Py_QuickenedCount = 0;
@ -641,11 +638,10 @@ initial_counter_value(void) {
static int
specialize_module_load_attr(
PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
_PyAdaptiveEntry *cache0, int opcode,
int opcode_module)
specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
PyObject *name, int opcode, int opcode_module)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyModuleObject *m = (PyModuleObject *)owner;
PyObject *value = NULL;
assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
@ -676,8 +672,8 @@ specialize_module_load_attr(
SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
return -1;
}
cache0->version = keys_version;
cache0->index = (uint16_t)index;
write_u32(cache->version, keys_version);
cache->index = (uint16_t)index;
*instr = _Py_MAKECODEUNIT(opcode_module, _Py_OPARG(*instr));
return 0;
}
@ -765,7 +761,6 @@ static int
specialize_dict_access(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name,
_PyAdaptiveEntry *cache0,
int base_op, int values_op, int hint_op)
{
assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
@ -775,6 +770,7 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyObject **dictptr = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = (PyDictObject *)*dictptr;
if (dict == NULL) {
@ -787,8 +783,8 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache0->version = type->tp_version_tag;
cache0->index = (uint16_t)index;
write_u32(cache->version, type->tp_version_tag);
cache->index = (uint16_t)index;
*instr = _Py_MAKECODEUNIT(values_op, _Py_OPARG(*instr));
}
else {
@ -804,20 +800,22 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache0->index = (uint16_t)hint;
cache0->version = type->tp_version_tag;
cache->index = (uint16_t)hint;
write_u32(cache->version, type->tp_version_tag);
*instr = _Py_MAKECODEUNIT(hint_op, _Py_OPARG(*instr));
}
return 1;
}
int
_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache)
_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
_PyAdaptiveEntry *cache0 = &cache->adaptive;
assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] ==
INLINE_CACHE_ENTRIES_LOAD_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
if (PyModule_CheckExact(owner)) {
int err = specialize_module_load_attr(owner, instr, name, cache0,
LOAD_ATTR, LOAD_ATTR_MODULE);
int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
LOAD_ATTR_MODULE);
if (err) {
goto fail;
}
@ -856,8 +854,8 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp
}
assert(dmem->type == T_OBJECT_EX);
assert(offset > 0);
cache0->index = (uint16_t)offset;
cache0->version = type->tp_version_tag;
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
*instr = _Py_MAKECODEUNIT(LOAD_ATTR_SLOT, _Py_OPARG(*instr));
goto success;
}
@ -865,8 +863,8 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp
{
Py_ssize_t offset = offsetof(PyObject, ob_type);
assert(offset == (uint16_t)offset);
cache0->index = (uint16_t)offset;
cache0->version = type->tp_version_tag;
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
*instr = _Py_MAKECODEUNIT(LOAD_ATTR_SLOT, _Py_OPARG(*instr));
goto success;
}
@ -887,41 +885,33 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, Sp
break;
}
int err = specialize_dict_access(
owner, instr, type, kind, name, cache0,
owner, instr, type, kind, name,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT
);
if (err < 0) {
return -1;
}
if (err) {
if (_Py_OPCODE(instr[0]) == LOAD_ATTR_INSTANCE_VALUE) {
// Note: instr[-1] exists because there's something on the stack,
// and instr[-2] exists because there's at least a RESUME as well.
if (_Py_OPCODE(instr[-1]) == LOAD_FAST) {
instr[-1] = _Py_MAKECODEUNIT(LOAD_FAST__LOAD_ATTR_INSTANCE_VALUE, _Py_OPARG(instr[-1]));
if (_Py_OPCODE(instr[-2]) == LOAD_FAST__LOAD_FAST) {
instr[-2] = _Py_MAKECODEUNIT(LOAD_FAST, _Py_OPARG(instr[-2]));
}
}
}
goto success;
}
fail:
STAT_INC(LOAD_ATTR, failure);
assert(!PyErr_Occurred());
cache_backoff(cache0);
cache->counter = ADAPTIVE_CACHE_BACKOFF;
return 0;
success:
STAT_INC(LOAD_ATTR, success);
assert(!PyErr_Occurred());
cache0->counter = initial_counter_value();
cache->counter = initial_counter_value();
return 0;
}
int
_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache)
_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
_PyAdaptiveEntry *cache0 = &cache->adaptive;
assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] ==
INLINE_CACHE_ENTRIES_STORE_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
if (PyModule_CheckExact(owner)) {
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
@ -954,8 +944,8 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S
}
assert(dmem->type == T_OBJECT_EX);
assert(offset > 0);
cache0->index = (uint16_t)offset;
cache0->version = type->tp_version_tag;
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
*instr = _Py_MAKECODEUNIT(STORE_ATTR_SLOT, _Py_OPARG(*instr));
goto success;
}
@ -978,7 +968,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S
}
int err = specialize_dict_access(
owner, instr, type, kind, name, cache0,
owner, instr, type, kind, name,
STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT
);
if (err < 0) {
@ -990,12 +980,12 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S
fail:
STAT_INC(STORE_ATTR, failure);
assert(!PyErr_Occurred());
cache_backoff(cache0);
cache->counter = ADAPTIVE_CACHE_BACKOFF;
return 0;
success:
STAT_INC(STORE_ATTR, success);
assert(!PyErr_Occurred());
cache0->counter = initial_counter_value();
cache->counter = initial_counter_value();
return 0;
}
@ -1037,18 +1027,18 @@ load_method_fail_kind(DescriptorClassification kind)
#endif
static int
specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
_PyAttrCache *cache1, _PyObjectCache *cache2)
specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
PyObject *name)
{
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyObject *descr = NULL;
DescriptorClassification kind = 0;
kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0);
switch (kind) {
case METHOD:
case NON_DESCRIPTOR:
cache1->tp_version = ((PyTypeObject *)owner)->tp_version_tag;
cache2->obj = descr;
write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
write_obj(cache->descr, descr);
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_CLASS, _Py_OPARG(*instr));
return 0;
#ifdef Py_STATS
@ -1078,16 +1068,18 @@ typedef enum {
// can cause a significant drop in cache hits. A possible test is
// python.exe -m test_typing test_re test_dis test_zlib.
int
_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache)
_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
_PyAdaptiveEntry *cache0 = &cache->adaptive;
_PyAttrCache *cache1 = &cache[-1].attr;
_PyObjectCache *cache2 = &cache[-2].obj;
assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] ==
INLINE_CACHE_ENTRIES_LOAD_METHOD);
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
if (PyModule_CheckExact(owner)) {
int err = specialize_module_load_attr(owner, instr, name, cache0,
LOAD_METHOD, LOAD_METHOD_MODULE);
assert(INLINE_CACHE_ENTRIES_LOAD_ATTR <=
INLINE_CACHE_ENTRIES_LOAD_METHOD);
int err = specialize_module_load_attr(owner, instr, name, LOAD_METHOD,
LOAD_METHOD_MODULE);
if (err) {
goto fail;
}
@ -1099,7 +1091,7 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
}
}
if (PyType_Check(owner)) {
int err = specialize_class_load_method(owner, instr, name, cache1, cache2);
int err = specialize_class_load_method(owner, instr, name);
if (err) {
goto fail;
}
@ -1157,7 +1149,7 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS);
goto fail;
}
cache1->dk_version = keys_version;
write_u32(cache->keys_version, keys_version);
}
switch(dictkind) {
case NO_DICT:
@ -1167,12 +1159,12 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_VALUES, _Py_OPARG(*instr));
break;
case MANAGED_DICT:
*(int16_t *)&cache0->index = (int16_t)MANAGED_DICT_OFFSET;
*(int16_t *)&cache->dict_offset = (int16_t)MANAGED_DICT_OFFSET;
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_DICT, _Py_OPARG(*instr));
break;
case OFFSET_DICT:
assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX);
cache0->index = (uint16_t)owner_cls->tp_dictoffset;
cache->dict_offset = (uint16_t)owner_cls->tp_dictoffset;
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_DICT, _Py_OPARG(*instr));
break;
}
@ -1190,18 +1182,18 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
* PyType_Modified usages in typeobject.c). The MCACHE has been
* working since Python 2.6 and it's battle-tested.
*/
cache1->tp_version = owner_cls->tp_version_tag;
cache2->obj = descr;
write_u32(cache->type_version, owner_cls->tp_version_tag);
write_obj(cache->descr, descr);
// Fall through.
success:
STAT_INC(LOAD_METHOD, success);
assert(!PyErr_Occurred());
cache0->counter = initial_counter_value();
cache->counter = initial_counter_value();
return 0;
fail:
STAT_INC(LOAD_METHOD, failure);
assert(!PyErr_Occurred());
cache_backoff(cache0);
cache->counter = ADAPTIVE_CACHE_BACKOFF;
return 0;
}
@ -1238,7 +1230,7 @@ _Py_Specialize_LoadGlobal(
goto fail;
}
cache->index = (uint16_t)index;
write32(&cache->module_keys_version, keys_version);
write_u32(cache->module_keys_version, keys_version);
*instr = _Py_MAKECODEUNIT(LOAD_GLOBAL_MODULE, _Py_OPARG(*instr));
goto success;
}
@ -1273,7 +1265,7 @@ _Py_Specialize_LoadGlobal(
goto fail;
}
cache->index = (uint16_t)index;
write32(&cache->module_keys_version, globals_version);
write_u32(cache->module_keys_version, globals_version);
cache->builtin_keys_version = (uint16_t)builtins_version;
*instr = _Py_MAKECODEUNIT(LOAD_GLOBAL_BUILTIN, _Py_OPARG(*instr));
goto success;
@ -1393,7 +1385,7 @@ _Py_Specialize_BinarySubscr(
goto fail;
}
assert(cls->tp_version_tag != 0);
write32(&cache->type_version, cls->tp_version_tag);
write_u32(cache->type_version, cls->tp_version_tag);
int version = _PyFunction_GetVersionForCurrentState(func);
if (version == 0 || version != (uint16_t)version) {
SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS);