gh-115999: Enable specialization of CALL instructions in free-threaded builds (#127123)

The CALL family of instructions were mostly thread-safe already and only required a small number of changes, which are documented below.

A few changes were needed to make CALL_ALLOC_AND_ENTER_INIT thread-safe:

Added _PyType_LookupRefAndVersion, which returns the type version corresponding to the returned ref.

Added _PyType_CacheInitForSpecialization, which takes an init method and the corresponding type version and only populates the specialization cache if the current type version matches the supplied version. This prevents potentially caching a stale value in free-threaded builds if we race with an update to __init__.

Only cache __init__ functions that are deferred in free-threaded builds. This ensures that the reference to __init__ that is stored in the specialization cache is valid if the type version guard in _CHECK_AND_ALLOCATE_OBJECT passes.
Fix a bug in _CREATE_INIT_FRAME where the frame is pushed to the stack on failure.

A few other miscellaneous changes were also needed:

Use {LOCK,UNLOCK}_OBJECT in LIST_APPEND. This ensures that the list's per-object lock is held while we are appending to it.

Add missing co_tlbc for _Py_InitCleanup.

Stop/start the world around setting the eval frame hook. This allows us to read interp->eval_frame non-atomically and preserves the behavior of _CHECK_PEP_523 documented below.
This commit is contained in:
mpage 2024-12-03 11:20:20 -08:00 committed by GitHub
parent fc5a0dc224
commit dabcecfd6d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 220 additions and 92 deletions

View file

@ -5528,9 +5528,12 @@ _PyTypes_AfterFork(void)
}
/* Internal API to look for a name through the MRO.
This returns a borrowed reference, and doesn't set an exception! */
This returns a strong reference, and doesn't set an exception!
If nonzero, version is set to the value of type->tp_version at the time of
the lookup.
*/
PyObject *
_PyType_LookupRef(PyTypeObject *type, PyObject *name)
_PyType_LookupRefAndVersion(PyTypeObject *type, PyObject *name, unsigned int *version)
{
PyObject *res;
int error;
@ -5553,6 +5556,9 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name)
// If the sequence is still valid then we're done
if (value == NULL || _Py_TryIncref(value)) {
if (_PySeqLock_EndRead(&entry->sequence, sequence)) {
if (version != NULL) {
*version = entry_version;
}
return value;
}
Py_XDECREF(value);
@ -5574,6 +5580,9 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name)
OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name));
OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name));
Py_XINCREF(entry->value);
if (version != NULL) {
*version = entry->version;
}
return entry->value;
}
#endif
@ -5587,12 +5596,12 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name)
// anyone else can modify our mro or mutate the type.
int has_version = 0;
int version = 0;
unsigned int assigned_version = 0;
BEGIN_TYPE_LOCK();
res = find_name_in_mro(type, name, &error);
if (MCACHE_CACHEABLE_NAME(name)) {
has_version = assign_version_tag(interp, type);
version = type->tp_version_tag;
assigned_version = type->tp_version_tag;
}
END_TYPE_LOCK();
@ -5609,28 +5618,67 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name)
if (error == -1) {
PyErr_Clear();
}
if (version != NULL) {
// 0 is not a valid version
*version = 0;
}
return NULL;
}
if (has_version) {
#if Py_GIL_DISABLED
update_cache_gil_disabled(entry, name, version, res);
update_cache_gil_disabled(entry, name, assigned_version, res);
#else
PyObject *old_value = update_cache(entry, name, version, res);
PyObject *old_value = update_cache(entry, name, assigned_version, res);
Py_DECREF(old_value);
#endif
}
if (version != NULL) {
// 0 is not a valid version
*version = has_version ? assigned_version : 0;
}
return res;
}
/* Internal API to look for a name through the MRO.
This returns a strong reference, and doesn't set an exception!
*/
PyObject *
_PyType_LookupRef(PyTypeObject *type, PyObject *name)
{
return _PyType_LookupRefAndVersion(type, name, NULL);
}
/* Internal API to look for a name through the MRO.
This returns a borrowed reference, and doesn't set an exception! */
PyObject *
_PyType_Lookup(PyTypeObject *type, PyObject *name)
{
PyObject *res = _PyType_LookupRef(type, name);
PyObject *res = _PyType_LookupRefAndVersion(type, name, NULL);
Py_XDECREF(res);
return res;
}
int
_PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init,
unsigned int tp_version)
{
if (!init || !tp_version) {
return 0;
}
int can_cache;
BEGIN_TYPE_LOCK();
can_cache = ((PyTypeObject*)type)->tp_version_tag == tp_version;
#ifdef Py_GIL_DISABLED
can_cache = can_cache && _PyObject_HasDeferredRefcount(init);
#endif
if (can_cache) {
FT_ATOMIC_STORE_PTR_RELEASE(type->_spec_cache.init, init);
}
END_TYPE_LOCK();
return can_cache;
}
static void
set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags)
{