mirror of
https://github.com/python/cpython.git
synced 2025-09-26 18:29:57 +00:00
gh-110481: Implement biased reference counting (gh-110764)
This commit is contained in:
Sam Gross
GitHub
parent
05f2f0ac92
commit
6dfb8fe023
29 changed files with 511 additions and 52 deletions
191
Include/object.h
191
Include/object.h
|
|
@ -106,9 +106,26 @@ check by comparing the reference count field to the immortality reference count.
|
|||
#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2)
|
||||
#endif
|
||||
|
||||
// Py_NOGIL builds indicate immortal objects using `ob_ref_local`, which is
|
||||
// always 32-bits.
|
||||
#ifdef Py_NOGIL
|
||||
#define _Py_IMMORTAL_REFCNT_LOCAL UINT32_MAX
|
||||
#endif
|
||||
|
||||
// Make all internal uses of PyObject_HEAD_INIT immortal while preserving the
|
||||
// C-API expectation that the refcnt will be set to 1.
|
||||
#ifdef Py_BUILD_CORE
|
||||
#if defined(Py_NOGIL)
|
||||
#define PyObject_HEAD_INIT(type) \
|
||||
{ \
|
||||
0, \
|
||||
0, \
|
||||
0, \
|
||||
0, \
|
||||
_Py_IMMORTAL_REFCNT_LOCAL, \
|
||||
0, \
|
||||
(type), \
|
||||
},
|
||||
#elif defined(Py_BUILD_CORE)
|
||||
#define PyObject_HEAD_INIT(type) \
|
||||
{ \
|
||||
{ _Py_IMMORTAL_REFCNT }, \
|
||||
|
|
@ -142,6 +159,7 @@ check by comparing the reference count field to the immortality reference count.
|
|||
* by hand. Similarly every pointer to a variable-size Python object can,
|
||||
* in addition, be cast to PyVarObject*.
|
||||
*/
|
||||
#ifndef Py_NOGIL
|
||||
struct _object {
|
||||
#if (defined(__GNUC__) || defined(__clang__)) \
|
||||
&& !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L)
|
||||
|
|
@ -166,6 +184,36 @@ struct _object {
|
|||
|
||||
PyTypeObject *ob_type;
|
||||
};
|
||||
#else
|
||||
// Objects that are not owned by any thread use a thread id (tid) of zero.
|
||||
// This includes both immortal objects and objects whose reference count
|
||||
// fields have been merged.
|
||||
#define _Py_UNOWNED_TID 0
|
||||
|
||||
// The shared reference count uses the two least-significant bits to store
|
||||
// flags. The remaining bits are used to store the reference count.
|
||||
#define _Py_REF_SHARED_SHIFT 2
|
||||
#define _Py_REF_SHARED_FLAG_MASK 0x3
|
||||
|
||||
// The shared flags are initialized to zero.
|
||||
#define _Py_REF_SHARED_INIT 0x0
|
||||
#define _Py_REF_MAYBE_WEAKREF 0x1
|
||||
#define _Py_REF_QUEUED 0x2
|
||||
#define _Py_REF_MERGED 0x3
|
||||
|
||||
// Create a shared field from a refcnt and desired flags
|
||||
#define _Py_REF_SHARED(refcnt, flags) (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags))
|
||||
|
||||
struct _object {
|
||||
uintptr_t ob_tid; // thread id (or zero)
|
||||
uint16_t _padding;
|
||||
uint8_t ob_mutex; // per-object lock
|
||||
uint8_t ob_gc_bits; // gc-related state
|
||||
uint32_t ob_ref_local; // local reference count
|
||||
Py_ssize_t ob_ref_shared; // shared (atomic) reference count
|
||||
PyTypeObject *ob_type;
|
||||
};
|
||||
#endif
|
||||
|
||||
/* Cast argument to PyObject* type. */
|
||||
#define _PyObject_CAST(op) _Py_CAST(PyObject*, (op))
|
||||
|
|
@ -183,9 +231,56 @@ typedef struct {
|
|||
PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y);
|
||||
#define Py_Is(x, y) ((x) == (y))
|
||||
|
||||
#ifndef Py_LIMITED_API
|
||||
static inline uintptr_t
|
||||
_Py_ThreadId(void)
|
||||
{
|
||||
uintptr_t tid;
|
||||
#if defined(_MSC_VER) && defined(_M_X64)
|
||||
tid = __readgsqword(48);
|
||||
#elif defined(_MSC_VER) && defined(_M_IX86)
|
||||
tid = __readfsdword(24);
|
||||
#elif defined(_MSC_VER) && defined(_M_ARM64)
|
||||
tid = __getReg(18);
|
||||
#elif defined(__i386__)
|
||||
__asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS
|
||||
#elif defined(__MACH__) && defined(__x86_64__)
|
||||
__asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
__asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid));
|
||||
#elif defined(__aarch64__) && defined(__APPLE__)
|
||||
__asm__ ("mrs %0, tpidrro_el0" : "=r" (tid));
|
||||
#elif defined(__aarch64__)
|
||||
__asm__ ("mrs %0, tpidr_el0" : "=r" (tid));
|
||||
#else
|
||||
# error "define _Py_ThreadId for this platform"
|
||||
#endif
|
||||
return tid;
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(Py_NOGIL) && !defined(Py_LIMITED_API)
|
||||
static inline Py_ALWAYS_INLINE int
|
||||
_Py_IsOwnedByCurrentThread(PyObject *ob)
|
||||
{
|
||||
return ob->ob_tid == _Py_ThreadId();
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline Py_ssize_t Py_REFCNT(PyObject *ob) {
|
||||
#if !defined(Py_NOGIL)
|
||||
return ob->ob_refcnt;
|
||||
#else
|
||||
uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local);
|
||||
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
|
||||
return _Py_IMMORTAL_REFCNT;
|
||||
}
|
||||
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared);
|
||||
return _Py_STATIC_CAST(Py_ssize_t, local) +
|
||||
Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT);
|
||||
#endif
|
||||
}
|
||||
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
|
||||
# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob))
|
||||
|
|
@ -216,7 +311,9 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) {
|
|||
|
||||
static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op)
|
||||
{
|
||||
#if SIZEOF_VOID_P > 4
|
||||
#if defined(Py_NOGIL)
|
||||
return op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL;
|
||||
#elif SIZEOF_VOID_P > 4
|
||||
return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0;
|
||||
#else
|
||||
return op->ob_refcnt == _Py_IMMORTAL_REFCNT;
|
||||
|
|
@ -240,7 +337,24 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) {
|
|||
if (_Py_IsImmortal(ob)) {
|
||||
return;
|
||||
}
|
||||
#if !defined(Py_NOGIL)
|
||||
ob->ob_refcnt = refcnt;
|
||||
#else
|
||||
if (_Py_IsOwnedByCurrentThread(ob)) {
|
||||
// Set local refcount to desired refcount and shared refcount to zero,
|
||||
// but preserve the shared refcount flags.
|
||||
assert(refcnt < UINT32_MAX);
|
||||
ob->ob_ref_local = _Py_STATIC_CAST(uint32_t, refcnt);
|
||||
ob->ob_ref_shared &= _Py_REF_SHARED_FLAG_MASK;
|
||||
}
|
||||
else {
|
||||
// Set local refcount to zero and shared refcount to desired refcount.
|
||||
// Mark the object as merged.
|
||||
ob->ob_tid = _Py_UNOWNED_TID;
|
||||
ob->ob_ref_local = 0;
|
||||
ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
|
||||
# define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt))
|
||||
|
|
@ -618,7 +732,19 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
|
|||
#else
|
||||
// Non-limited C API and limited C API for Python 3.9 and older access
|
||||
// directly PyObject.ob_refcnt.
|
||||
#if SIZEOF_VOID_P > 4
|
||||
#if defined(Py_NOGIL)
|
||||
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
|
||||
uint32_t new_local = local + 1;
|
||||
if (new_local == 0) {
|
||||
return;
|
||||
}
|
||||
if (_Py_IsOwnedByCurrentThread(op)) {
|
||||
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local);
|
||||
}
|
||||
else {
|
||||
_Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT));
|
||||
}
|
||||
#elif SIZEOF_VOID_P > 4
|
||||
// Portable saturated add, branching on the carry flag and set low bits
|
||||
PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN];
|
||||
PY_UINT32_T new_refcnt = cur_refcnt + 1;
|
||||
|
|
@ -643,6 +769,19 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
|
|||
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
|
||||
#endif
|
||||
|
||||
|
||||
#if !defined(Py_LIMITED_API) && defined(Py_NOGIL)
|
||||
// Implements Py_DECREF on objects not owned by the current thread.
|
||||
PyAPI_FUNC(void) _Py_DecRefShared(PyObject *);
|
||||
PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int);
|
||||
|
||||
// Called from Py_DECREF by the owning thread when the local refcount reaches
|
||||
// zero. The call will deallocate the object if the shared refcount is also
|
||||
// zero. Otherwise, the thread gives up ownership and merges the reference
|
||||
// count fields.
|
||||
PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *);
|
||||
#endif
|
||||
|
||||
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
|
||||
// Stable ABI implements Py_DECREF() as a function call on limited C API
|
||||
// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was
|
||||
|
|
@ -657,6 +796,52 @@ static inline void Py_DECREF(PyObject *op) {
|
|||
}
|
||||
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
|
||||
|
||||
#elif defined(Py_NOGIL) && defined(Py_REF_DEBUG)
|
||||
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
|
||||
{
|
||||
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
|
||||
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
|
||||
return;
|
||||
}
|
||||
_Py_DECREF_STAT_INC();
|
||||
_Py_DECREF_DecRefTotal();
|
||||
if (_Py_IsOwnedByCurrentThread(op)) {
|
||||
if (local == 0) {
|
||||
_Py_NegativeRefcount(filename, lineno, op);
|
||||
}
|
||||
local--;
|
||||
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
|
||||
if (local == 0) {
|
||||
_Py_MergeZeroLocalRefcount(op);
|
||||
}
|
||||
}
|
||||
else {
|
||||
_Py_DecRefSharedDebug(op, filename, lineno);
|
||||
}
|
||||
}
|
||||
#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
|
||||
|
||||
#elif defined(Py_NOGIL)
|
||||
static inline void Py_DECREF(PyObject *op)
|
||||
{
|
||||
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
|
||||
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
|
||||
return;
|
||||
}
|
||||
_Py_DECREF_STAT_INC();
|
||||
if (_Py_IsOwnedByCurrentThread(op)) {
|
||||
local--;
|
||||
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
|
||||
if (local == 0) {
|
||||
_Py_MergeZeroLocalRefcount(op);
|
||||
}
|
||||
}
|
||||
else {
|
||||
_Py_DecRefShared(op);
|
||||
}
|
||||
}
|
||||
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
|
||||
|
||||
#elif defined(Py_REF_DEBUG)
|
||||
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue