mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-07 19:35:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-115999: Specialize STORE_ATTR in free-threaded builds. (gh-127838)

Browse source 
* Add `_PyDictKeys_StringLookupSplit` which does locking on dict keys and
  use in place of `_PyDictKeys_StringLookup`.

* Change `_PyObject_TryGetInstanceAttribute` to use that function
  in the case of split keys.

* Add `unicodekeys_lookup_split` helper which allows code sharing
  between `_Py_dict_lookup` and `_PyDictKeys_StringLookupSplit`.

* Fix locking for `STORE_ATTR_INSTANCE_VALUE`.  Create
  `_GUARD_TYPE_VERSION_AND_LOCK` uop so that object stays locked and
  `tp_version_tag` cannot change.

* Pass `tp_version_tag` to `specialize_dict_access()`, ensuring
  the version we store on the cache is the correct one (in case of
  it changing during the specalize analysis).

* Split `analyze_descriptor` into `analyze_descriptor_load` and
  `analyze_descriptor_store` since those don't share much logic.
  Add `descriptor_is_class` helper function.

* In `specialize_dict_access`, double check `_PyObject_GetManagedDict()`
  in case we race and dict was materialized before the lock.

* Avoid borrowed references in `_Py_Specialize_StoreAttr()`.

* Use `specialize()` and `unspecialize()` helpers.

* Add unit tests to ensure specializing happens as expected in FT builds.

* Add unit tests to attempt to trigger data races (useful for running under TSAN).

* Add `has_split_table` function to `_testinternalcapi`.
This commit is contained in:
Neil Schemenauer 2024-12-19 10:21:17 -08:00 • committed by GitHub
parent d2f1d917e8
commit 1b15c89a17
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 716 additions and 297 deletions
Download patch file Download diff file Expand all files Collapse all files

271
Python/specialize.c
View file

@ -741,8 +741,8 @@ static int function_kind(PyCodeObject *code);
#ifndef Py_GIL_DISABLED
static bool function_check_args(PyObject *o, int expected_argcount, int opcode);
static uint32_t function_get_version(PyObject *o, int opcode);
#endif
static uint32_t type_get_version(PyTypeObject *t, int opcode);
#endif
static int
specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name)
@ -881,71 +881,142 @@ classify_descriptor(PyObject *descriptor, bool has_getattr)
return NON_DESCRIPTOR;
}
static DescriptorClassification
analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
static bool
descriptor_is_class(PyObject *descriptor, PyObject *name)
{
return ((PyUnicode_CompareWithASCIIString(name, "__class__") == 0) &&
(descriptor == _PyType_Lookup(&PyBaseObject_Type, name)));
}
#ifndef Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) {
bool has_getattr = false;
if (store) {
if (type->tp_setattro != PyObject_GenericSetAttr) {
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
has_getattr = false;
}
else if (getattro_slot == _Py_slot_tp_getattr_hook ||
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__));
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__));
has_getattr = getattr != NULL;
Py_XDECREF(getattr);
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
Py_DECREF(getattribute);
*descr = NULL;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
Fall through to usual descriptor analysis.
Usual attribute lookup should only be allowed at runtime
if we can guarantee that there is no way an exception can be
raised. This means some specializations, e.g. specializing
for property() isn't safe.
*/
Py_XDECREF(getattribute);
}
else {
getattrofunc getattro_slot = type->tp_getattro;
if (getattro_slot == PyObject_GenericGetAttr) {
/* Normal attribute lookup; */
has_getattr = false;
}
else if (getattro_slot == _Py_slot_tp_getattr_hook ||
getattro_slot == _Py_slot_tp_getattro) {
/* One or both of __getattribute__ or __getattr__ may have been
overridden See typeobject.c for why these functions are special. */
PyObject *getattribute = _PyType_Lookup(type,
&_Py_ID(__getattribute__));
PyInterpreterState *interp = _PyInterpreterState_GET();
bool has_custom_getattribute = getattribute != NULL &&
getattribute != interp->callable_cache.object__getattribute__;
has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL;
if (has_custom_getattribute) {
if (getattro_slot == _Py_slot_tp_getattro &&
!has_getattr &&
Py_IS_TYPE(getattribute, &PyFunction_Type)) {
*descr = getattribute;
return GETATTRIBUTE_IS_PYTHON_FUNCTION;
}
/* Potentially both __getattr__ and __getattribute__ are set.
Too complicated */
*descr = NULL;
return GETSET_OVERRIDDEN;
}
/* Potentially has __getattr__ but no custom __getattribute__.
Fall through to usual descriptor analysis.
Usual attribute lookup should only be allowed at runtime
if we can guarantee that there is no way an exception can be
raised. This means some specializations, e.g. specializing
for property() isn't safe.
*/
}
else {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
*descr = NULL;
return GETSET_OVERRIDDEN;
}
PyObject *descriptor = _PyType_Lookup(type, name);
PyObject *descriptor = _PyType_LookupRef(type, name);
*descr = descriptor;
if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
return DUNDER_CLASS;
}
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, has_getattr);
}
#endif //!Py_GIL_DISABLED
static DescriptorClassification
analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version)
{
if (type->tp_setattro != PyObject_GenericSetAttr) {
*descr = NULL;
return GETSET_OVERRIDDEN;
}
PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, tp_version);
*descr = descriptor;
if (descriptor_is_class(descriptor, name)) {
return DUNDER_CLASS;
}
return classify_descriptor(descriptor, false);
}
static int
specialize_dict_access_inline(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int values_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);
return 0;
}
assert(index >= 0);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache->index = (uint16_t)offset;
write_u32(cache->version, tp_version);
specialize(instr, values_op);
return 1;
}
static int
specialize_dict_access_hint(
PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int hint_op)
{
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
// We found an instance with a __dict__.
if (_PyDict_HasSplitTable(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
return 0;
}
Py_ssize_t index = _PyDict_LookupIndex(dict, name);
if (index != (uint16_t)index) {
SPECIALIZATION_FAIL(base_op,
index == DKIX_EMPTY ?
SPEC_FAIL_ATTR_NOT_IN_DICT :
SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache->index = (uint16_t)index;
write_u32(cache->version, tp_version);
specialize(instr, hint_op);
return 1;
}
static int
specialize_dict_access(
PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
DescriptorClassification kind, PyObject *name,
DescriptorClassification kind, PyObject *name, unsigned int tp_version,
int base_op, int values_op, int hint_op)
{
assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
@ -956,29 +1027,25 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&
_PyObject_InlineValues(owner)->valid &&
FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner)->valid) &&
!(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL))
{
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
assert (index != DKIX_ERROR);
if (index == DKIX_EMPTY) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_IN_KEYS);
return 0;
int res;
Py_BEGIN_CRITICAL_SECTION(owner);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
// managed dict, not materialized, inline values valid
res = specialize_dict_access_inline(owner, instr, type, kind, name,
tp_version, base_op, values_op);
}
assert(index >= 0);
char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index];
Py_ssize_t offset = value_addr - (char *)owner;
if (offset != (uint16_t)offset) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
return 0;
else {
// lost race and dict was created, fail specialization
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OTHER);
res = 0;
}
write_u32(cache->version, type->tp_version_tag);
cache->index = (uint16_t)offset;
specialize(instr, values_op);
Py_END_CRITICAL_SECTION();
return res;
}
else {
PyDictObject *dict = _PyObject_GetManagedDict(owner);
@ -986,25 +1053,14 @@ specialize_dict_access(
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
}
// We found an instance with a __dict__.
if (dict->ma_values) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT);
return 0;
}
Py_ssize_t index =
_PyDict_LookupIndex(dict, name);
if (index != (uint16_t)index) {
SPECIALIZATION_FAIL(base_op,
index == DKIX_EMPTY ?
SPEC_FAIL_ATTR_NOT_IN_DICT :
SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
cache->index = (uint16_t)index;
write_u32(cache->version, type->tp_version_tag);
specialize(instr, hint_op);
int res;
Py_BEGIN_CRITICAL_SECTION(dict);
// materialized managed dict
res = specialize_dict_access_hint(dict, instr, type, kind, name,
tp_version, base_op, hint_op);
Py_END_CRITICAL_SECTION();
return res;
}
return 1;
}
#ifndef Py_GIL_DISABLED
@ -1050,7 +1106,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
PyTypeObject *type = Py_TYPE(owner);
bool shadow = instance_has_key(owner, name);
PyObject *descr = NULL;
DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
DescriptorClassification kind = analyze_descriptor_load(type, name, &descr);
Py_XDECREF(descr); // turn strong ref into a borrowed ref
assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
if (type_get_version(type, LOAD_ATTR) == 0) {
return -1;
@ -1204,8 +1261,8 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na
}
Py_UNREACHABLE();
try_instance:
if (specialize_dict_access(owner, instr, type, kind, name, LOAD_ATTR,
LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag,
LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT))
{
return 0;
}
@ -1259,8 +1316,9 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
{
PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st);
assert(ENABLE_SPECIALIZATION);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
PyObject *descr = NULL;
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
if (!_PyType_IsReady(type)) {
@ -1274,11 +1332,12 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
goto fail;
}
PyObject *descr;
DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
if (type_get_version(type, STORE_ATTR) == 0) {
unsigned int tp_version = 0;
DescriptorClassification kind = analyze_descriptor_store(type, name, &descr, &tp_version);
if (tp_version == 0) {
goto fail;
}
assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
switch(kind) {
case OVERRIDING:
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
@ -1309,8 +1368,8 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT);
assert(offset > 0);
cache->index = (uint16_t)offset;
write_u32(cache->version, type->tp_version_tag);
instr->op.code = STORE_ATTR_SLOT;
write_u32(cache->version, tp_version);
specialize(instr, STORE_ATTR_SLOT);
goto success;
}
case DUNDER_CLASS:
@ -1337,22 +1396,19 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na
SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_CLASS_ATTR_SIMPLE);
goto fail;
case ABSENT:
if (specialize_dict_access(owner, instr, type, kind, name, STORE_ATTR,
STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT))
{
if (specialize_dict_access(owner, instr, type, kind, name, tp_version,
STORE_ATTR, STORE_ATTR_INSTANCE_VALUE,
STORE_ATTR_WITH_HINT)) {
goto success;
}
}
fail:
STAT_INC(STORE_ATTR, failure);
assert(!PyErr_Occurred());
instr->op.code = STORE_ATTR;
cache->counter = adaptive_counter_backoff(cache->counter);
Py_XDECREF(descr);
unspecialize(instr);
return;
success:
STAT_INC(STORE_ATTR, success);
assert(!PyErr_Occurred());
cache->counter = adaptive_counter_cooldown();
Py_XDECREF(descr);
return;
}
#ifndef Py_GIL_DISABLED
@ -1421,7 +1477,8 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
}
PyObject *descr = NULL;
DescriptorClassification kind = 0;
kind = analyze_descriptor(cls, name, &descr, 0);
kind = analyze_descriptor_load(cls, name, &descr);
Py_XDECREF(descr); // turn strong ref into a borrowed ref
if (type_get_version(cls, LOAD_ATTR) == 0) {
return -1;
}
@ -1714,7 +1771,6 @@ function_get_version(PyObject *o, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
/* Returning 0 indicates a failure. */
static uint32_t
@ -1727,6 +1783,7 @@ type_get_version(PyTypeObject *t, int opcode)
}
return version;
}
#endif // Py_GIL_DISABLED
void
_Py_Specialize_BinarySubscr(