mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 14:07:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

GH-127705: Use _PyStackRefs in the default build. (GH-127875)

Browse source
This commit is contained in:
Mark Shannon 2025-03-10 14:06:56 +00:00 • committed by GitHub
parent 7cc99a54b7
commit 2bef8ea8ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 688 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

386
Include/internal/pycore_stackref.h
View file

@ -60,8 +60,6 @@ extern "C" {
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
typedef union _PyStackRef {
uint64_t index;
} _PyStackRef;
@ -153,6 +151,16 @@ _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
}
#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
static inline void
PyStackRef_XCLOSE(_PyStackRef ref)
{
if (PyStackRef_IsNull(ref)) {
return;
}
PyObject *obj = _Py_stackref_close(ref);
Py_DECREF(obj);
}
static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
@ -162,7 +170,36 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
}
#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref)
extern void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct);
static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
return ref;
}
#define PyStackRef_CLEAR(REF) \
do { \
_PyStackRef *_tmp_op_ptr = &(REF); \
_PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
*_tmp_op_ptr = PyStackRef_NULL; \
PyStackRef_XCLOSE(_tmp_old_op); \
} while (0)
static inline _PyStackRef
_PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber)
{
assert(!_Py_IsImmortal(obj));
return _Py_stackref_create(obj, filename, linenumber);
}
#define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__)
static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
return true;
}
#else
@ -171,12 +208,13 @@ typedef union _PyStackRef {
} _PyStackRef;
#ifdef Py_GIL_DISABLED
#define Py_TAG_DEFERRED (1)
#define Py_TAG_PTR ((uintptr_t)0)
#define Py_TAG_BITS ((uintptr_t)1)
#ifdef Py_GIL_DISABLED
static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
@ -184,6 +222,11 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
// Checks that mask out the deferred bit in the free threading build.
#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None)
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
{
@ -220,6 +263,17 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj)
}
# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
static inline _PyStackRef
PyStackRef_FromPyObjectStealMortal(PyObject *obj)
{
assert(obj != NULL);
assert(!_Py_IsImmortal(obj));
// Make sure we don't take an already tagged value.
assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
return (_PyStackRef){ .bits = (uintptr_t)obj };
}
static inline _PyStackRef
PyStackRef_FromPyObjectNew(PyObject *obj)
{
@ -255,6 +309,13 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj)
} \
} while (0)
static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
(void)destruct;
PyStackRef_CLOSE(ref);
}
static inline _PyStackRef
PyStackRef_DUP(_PyStackRef stackref)
{
@ -269,6 +330,18 @@ PyStackRef_DUP(_PyStackRef stackref)
return stackref;
}
static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
return true;
}
static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
return ref;
}
// Convert a possibly deferred reference to a strong reference.
static inline _PyStackRef
PyStackRef_AsStrongReference(_PyStackRef stackref)
@ -276,55 +349,13 @@ PyStackRef_AsStrongReference(_PyStackRef stackref)
return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref));
}
#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) PyStackRef_CLOSE(stackref)
#else // Py_GIL_DISABLED
// With GIL
static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
#define PyStackRef_IsNull(stackref) ((stackref).bits == 0)
#define PyStackRef_True ((_PyStackRef){.bits = (uintptr_t)&_Py_TrueStruct })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
#define PyStackRef_AsPyObjectBorrow(stackref) ((PyObject *)(stackref).bits)
#define PyStackRef_AsPyObjectSteal(stackref) PyStackRef_AsPyObjectBorrow(stackref)
#define PyStackRef_FromPyObjectSteal(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
#define PyStackRef_FromPyObjectNew(obj) ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) })
#define PyStackRef_FromPyObjectImmortal(obj) ((_PyStackRef){ .bits = (uintptr_t)(obj) })
#define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref))
#define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
#define PyStackRef_CLOSE_SPECIALIZED(stackref, dealloc) _Py_DECREF_SPECIALIZED(PyStackRef_AsPyObjectBorrow(stackref), dealloc)
#endif // Py_GIL_DISABLED
// Check if a stackref is exactly the same as another stackref, including the
// the deferred bit. This can only be used safely if you know that the deferred
// bits of `a` and `b` match.
#define PyStackRef_IsExactly(a, b) \
(assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits)
// Checks that mask out the deferred bit in the free threading build.
#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None)
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
#endif
// Converts a PyStackRef back to a PyObject *, converting the
// stackref to a new reference.
#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))
#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref))
#define PyStackRef_XCLOSE(stackref) \
do { \
_PyStackRef _tmp = (stackref); \
if (!PyStackRef_IsNull(_tmp)) { \
PyStackRef_CLOSE(_tmp); \
} \
} while (0);
#define PyStackRef_CLEAR(op) \
do { \
@ -336,15 +367,250 @@ static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
} \
} while (0)
#define PyStackRef_XCLOSE(stackref) \
do { \
_PyStackRef _tmp = (stackref); \
if (!PyStackRef_IsNull(_tmp)) { \
PyStackRef_CLOSE(_tmp); \
} \
} while (0);
#define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew
#else // Py_GIL_DISABLED
// With GIL
/* References to immortal objects always have their tag bit set to Py_TAG_REFCNT
* as they can (must) have their reclamation deferred */
#define Py_TAG_BITS 1
#define Py_TAG_REFCNT 1
#if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT
# error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT"
#endif
#define BITS_TO_PTR(REF) ((PyObject *)((REF).bits))
#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_BITS)))
#define PyStackRef_NULL_BITS Py_TAG_REFCNT
static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS };
#define PyStackRef_IsNull(ref) ((ref).bits == PyStackRef_NULL_BITS)
#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT })
#define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT))
#define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT))
#define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT))
#ifdef Py_DEBUG
static inline void PyStackRef_CheckValid(_PyStackRef ref) {
assert(ref.bits != 0);
int tag = ref.bits & Py_TAG_BITS;
PyObject *obj = BITS_TO_PTR_MASKED(ref);
switch (tag) {
case 0:
/* Can be immortal if object was made immortal after reference came into existence */
assert(!_Py_IsStaticImmortal(obj));
break;
case Py_TAG_REFCNT:
assert(obj == NULL || _Py_IsImmortal(obj));
break;
default:
assert(0);
}
}
#else
#define PyStackRef_CheckValid(REF) ((void)0)
#endif
#ifdef _WIN32
#define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_BITS) == 0)
#define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED
#else
/* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */
static inline int
PyStackRef_RefcountOnObject(_PyStackRef ref)
{
return (ref.bits & Py_TAG_BITS) == 0;
}
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
{
return BITS_TO_PTR_MASKED(ref);
}
#endif
static inline PyObject *
PyStackRef_AsPyObjectSteal(_PyStackRef ref)
{
if (PyStackRef_RefcountOnObject(ref)) {
return BITS_TO_PTR(ref);
}
else {
return Py_NewRef(BITS_TO_PTR_MASKED(ref));
}
}
static inline _PyStackRef
PyStackRef_FromPyObjectSteal(PyObject *obj)
{
assert(obj != NULL);
#if SIZEOF_VOID_P > 4
unsigned int tag = obj->ob_flags & Py_TAG_BITS;
#else
unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0;
#endif
_PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
PyStackRef_CheckValid(ref);
return ref;
}
static inline _PyStackRef
PyStackRef_FromPyObjectStealMortal(PyObject *obj)
{
assert(obj != NULL);
assert(!_Py_IsImmortal(obj));
_PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) });
PyStackRef_CheckValid(ref);
return ref;
}
// Check if a stackref is exactly the same as another stackref, including the
// the deferred bit. This can only be used safely if you know that the deferred
// bits of `a` and `b` match.
#define PyStackRef_IsExactly(a, b) \
(assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits)
static inline _PyStackRef
_PyStackRef_FromPyObjectNew(PyObject *obj)
{
assert(obj != NULL);
if (_Py_IsImmortal(obj)) {
return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT};
}
Py_INCREF_MORTAL(obj);
_PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
PyStackRef_CheckValid(ref);
return ref;
}
#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))
static inline _PyStackRef
_PyStackRef_FromPyObjectNewMortal(PyObject *obj)
{
assert(obj != NULL);
Py_INCREF_MORTAL(obj);
_PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
PyStackRef_CheckValid(ref);
return ref;
}
#define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj))
/* Create a new reference from an object with an embedded reference count */
static inline _PyStackRef
PyStackRef_FromPyObjectImmortal(PyObject *obj)
{
assert(_Py_IsImmortal(obj));
return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT};
}
/* WARNING: This macro evaluates its argument more than once */
#ifdef _WIN32
#define PyStackRef_DUP(REF) \
(PyStackRef_RefcountOnObject(REF) ? (Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))
#else
static inline _PyStackRef
PyStackRef_DUP(_PyStackRef ref)
{
assert(!PyStackRef_IsNull(ref));
if (PyStackRef_RefcountOnObject(ref)) {
Py_INCREF_MORTAL(BITS_TO_PTR(ref));
}
return ref;
}
#endif
static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
return (ref.bits & Py_TAG_BITS) == 0 || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref));
}
static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
if (PyStackRef_IsHeapSafe(ref)) {
return ref;
}
PyObject *obj = BITS_TO_PTR_MASKED(ref);
Py_INCREF(obj);
ref.bits = (uintptr_t)obj;
PyStackRef_CheckValid(ref);
return ref;
}
#ifdef _WIN32
#define PyStackRef_CLOSE(REF) \
do { \
_PyStackRef _temp = (REF); \
if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \
} while (0)
#else
static inline void
PyStackRef_CLOSE(_PyStackRef ref)
{
assert(!PyStackRef_IsNull(ref));
if (PyStackRef_RefcountOnObject(ref)) {
Py_DECREF_MORTAL(BITS_TO_PTR(ref));
}
}
#endif
static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
assert(!PyStackRef_IsNull(ref));
if (PyStackRef_RefcountOnObject(ref)) {
Py_DECREF_MORTAL_SPECIALIZED(BITS_TO_PTR(ref), destruct);
}
}
#ifdef _WIN32
#define PyStackRef_XCLOSE PyStackRef_CLOSE
#else
static inline void
PyStackRef_XCLOSE(_PyStackRef ref)
{
assert(ref.bits != 0);
if (PyStackRef_RefcountOnObject(ref)) {
assert(!PyStackRef_IsNull(ref));
Py_DECREF_MORTAL(BITS_TO_PTR(ref));
}
}
#endif
#define PyStackRef_CLEAR(REF) \
do { \
_PyStackRef *_tmp_op_ptr = &(REF); \
_PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
*_tmp_op_ptr = PyStackRef_NULL; \
PyStackRef_XCLOSE(_tmp_old_op); \
} while (0)
#endif // Py_GIL_DISABLED
// Note: this is a macro because MSVC (Windows) has trouble inlining it.
#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT)))
#endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref))
// Converts a PyStackRef back to a PyObject *, converting the
// stackref to a new reference.
#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))
// StackRef type checks