mirror of
https://github.com/python/cpython.git
synced 2025-10-14 18:59:46 +00:00
gh-117139: Convert the evaluation stack to stack refs (#118450)
This PR sets up tagged pointers for CPython. The general idea is to create a separate struct _PyStackRef for everything on the evaluation stack to store the bits. This forces the C compiler to warn us if we try to cast things or pull things out of the struct directly. Only for free threading: We tag the low bit if something is deferred - that means we skip incref and decref operations on it. This behavior may change in the future if Mark's plans to defer all objects in the interpreter loop pans out. This implies a strict stack reference discipline is required. ALL incref and decref operations on stackrefs must use the stackref variants. It is unsafe to untag something then do normal incref/decref ops on it. The new incref and decref variants are called dup and close. They mimic a "handle" API operating on these stackrefs. Please read Include/internal/pycore_stackref.h for more information! --------- Co-authored-by: Mark Shannon <9448417+markshannon@users.noreply.github.com>
This commit is contained in:
Ken Jin
GitHub
parent
d611c4c8e9
commit
22b0de2755
35 changed files with 5228 additions and 3758 deletions
|
|
@ -8,187 +8,264 @@ extern "C" {
|
|||
# error "this header requires Py_BUILD_CORE define"
|
||||
#endif
|
||||
|
||||
#include "pycore_object_deferred.h"
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/*
|
||||
This file introduces a new API for handling references on the stack, called
|
||||
_PyStackRef. This API is inspired by HPy.
|
||||
|
||||
There are 3 main operations, that convert _PyStackRef to PyObject* and
|
||||
vice versa:
|
||||
|
||||
1. Borrow (discouraged)
|
||||
2. Steal
|
||||
3. New
|
||||
|
||||
Borrow means that the reference is converted without any change in ownership.
|
||||
This is discouraged because it makes verification much harder. It also makes
|
||||
unboxed integers harder in the future.
|
||||
|
||||
Steal means that ownership is transferred to something else. The total
|
||||
number of references to the object stays the same.
|
||||
|
||||
New creates a new reference from the old reference. The old reference
|
||||
is still valid.
|
||||
|
||||
With these 3 API, a strict stack discipline must be maintained. All
|
||||
_PyStackRef must be operated on by the new reference operations:
|
||||
|
||||
1. DUP
|
||||
2. CLOSE
|
||||
|
||||
DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old
|
||||
reference. The old reference remains unchanged.
|
||||
|
||||
CLOSE is roughly equivalent to Py_DECREF. It destroys a reference.
|
||||
|
||||
Note that it is unsafe to borrow a _PyStackRef and then do normal
|
||||
CPython refcounting operations on it!
|
||||
*/
|
||||
|
||||
typedef union {
|
||||
uintptr_t bits;
|
||||
} _PyStackRef;
|
||||
|
||||
static const _PyStackRef Py_STACKREF_NULL = { .bits = 0 };
|
||||
|
||||
#define Py_TAG_DEFERRED (1)
|
||||
|
||||
// Gets a PyObject * from a _PyStackRef
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
static inline PyObject *
|
||||
PyStackRef_Get(_PyStackRef tagged)
|
||||
{
|
||||
PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG_DEFERRED)));
|
||||
return cleared;
|
||||
}
|
||||
#define Py_TAG_PTR (0)
|
||||
#define Py_TAG_BITS (1)
|
||||
|
||||
#ifdef Py_GIL_DISABLED
|
||||
static const _PyStackRef PyStackRef_NULL = { .bits = 0 | Py_TAG_DEFERRED};
|
||||
#else
|
||||
# define PyStackRef_Get(tagged) ((PyObject *)((tagged).bits))
|
||||
static const _PyStackRef PyStackRef_NULL = { .bits = 0 };
|
||||
#endif
|
||||
|
||||
// Converts a PyObject * to a PyStackRef, stealing the reference.
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
static inline _PyStackRef
|
||||
_PyStackRef_StealRef(PyObject *obj)
|
||||
{
|
||||
// Make sure we don't take an already tagged value.
|
||||
assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
|
||||
return ((_PyStackRef){.bits = ((uintptr_t)(obj))});
|
||||
}
|
||||
# define PyStackRef_StealRef(obj) _PyStackRef_StealRef(_PyObject_CAST(obj))
|
||||
#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
|
||||
|
||||
|
||||
#ifdef Py_GIL_DISABLED
|
||||
# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
|
||||
#else
|
||||
# define PyStackRef_StealRef(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))})
|
||||
# define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) })
|
||||
#endif
|
||||
|
||||
#ifdef Py_GIL_DISABLED
|
||||
# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
|
||||
#else
|
||||
# define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) })
|
||||
#endif
|
||||
|
||||
#ifdef Py_GIL_DISABLED
|
||||
# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
|
||||
#else
|
||||
# define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) })
|
||||
#endif
|
||||
|
||||
|
||||
static inline int
|
||||
PyStackRef_Is(_PyStackRef a, _PyStackRef b) {
|
||||
return a.bits == b.bits;
|
||||
}
|
||||
|
||||
static inline int
|
||||
PyStackRef_IsDeferred(_PyStackRef ref)
|
||||
{
|
||||
return ((ref.bits & Py_TAG_BITS) == Py_TAG_DEFERRED);
|
||||
}
|
||||
|
||||
// Gets a PyObject * from a _PyStackRef
|
||||
static inline PyObject *
|
||||
PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
|
||||
{
|
||||
#ifdef Py_GIL_DISABLED
|
||||
PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS)));
|
||||
return cleared;
|
||||
#else
|
||||
return ((PyObject *)(stackref).bits);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Converts a PyStackRef back to a PyObject *, stealing the
|
||||
// PyStackRef.
|
||||
static inline PyObject *
|
||||
PyStackRef_AsPyObjectSteal(_PyStackRef stackref)
|
||||
{
|
||||
#ifdef Py_GIL_DISABLED
|
||||
if (!PyStackRef_IsNull(stackref) && PyStackRef_IsDeferred(stackref)) {
|
||||
return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
}
|
||||
return PyStackRef_AsPyObjectBorrow(stackref);
|
||||
#else
|
||||
return PyStackRef_AsPyObjectBorrow(stackref);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Converts a PyStackRef back to a PyObject *, converting the
|
||||
// stackref to a new reference.
|
||||
static inline PyObject *
|
||||
PyStackRef_AsPyObjectNew(_PyStackRef stackref)
|
||||
{
|
||||
return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
}
|
||||
|
||||
static inline PyTypeObject *
|
||||
PyStackRef_TYPE(_PyStackRef stackref)
|
||||
{
|
||||
return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
}
|
||||
|
||||
// Converts a PyObject * to a PyStackRef, stealing the reference
|
||||
static inline _PyStackRef
|
||||
_PyStackRef_FromPyObjectSteal(PyObject *obj)
|
||||
{
|
||||
#ifdef Py_GIL_DISABLED
|
||||
// Make sure we don't take an already tagged value.
|
||||
assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
|
||||
int tag = (obj == NULL || _Py_IsImmortal(obj)) ? (Py_TAG_DEFERRED) : Py_TAG_PTR;
|
||||
return ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
|
||||
#else
|
||||
return ((_PyStackRef){.bits = ((uintptr_t)(obj))});
|
||||
#endif
|
||||
}
|
||||
|
||||
#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
|
||||
|
||||
|
||||
// Converts a PyObject * to a PyStackRef, with a new reference
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
static inline _PyStackRef
|
||||
_PyStackRef_NewRefDeferred(PyObject *obj)
|
||||
PyStackRef_FromPyObjectNew(PyObject *obj)
|
||||
{
|
||||
#ifdef Py_GIL_DISABLED
|
||||
// Make sure we don't take an already tagged value.
|
||||
assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
|
||||
assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
|
||||
assert(obj != NULL);
|
||||
if (_PyObject_HasDeferredRefcount(obj)) {
|
||||
// TODO (gh-117139): Add deferred objects later.
|
||||
if (_Py_IsImmortal(obj)) {
|
||||
return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
|
||||
}
|
||||
else {
|
||||
return (_PyStackRef){ .bits = (uintptr_t)Py_NewRef(obj) };
|
||||
return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
|
||||
}
|
||||
}
|
||||
# define PyStackRef_NewRefDeferred(obj) _PyStackRef_NewRefDeferred(_PyObject_CAST(obj))
|
||||
#else
|
||||
# define PyStackRef_NewRefDeferred(obj) PyStackRef_NewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))}))
|
||||
return ((_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) });
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))
|
||||
|
||||
// Same as PyStackRef_FromPyObjectNew but only for immortal objects.
|
||||
static inline _PyStackRef
|
||||
_PyStackRef_XNewRefDeferred(PyObject *obj)
|
||||
PyStackRef_FromPyObjectImmortal(PyObject *obj)
|
||||
{
|
||||
#ifdef Py_GIL_DISABLED
|
||||
// Make sure we don't take an already tagged value.
|
||||
assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0);
|
||||
if (obj == NULL) {
|
||||
return Py_STACKREF_NULL;
|
||||
}
|
||||
return _PyStackRef_NewRefDeferred(obj);
|
||||
}
|
||||
# define PyStackRef_XNewRefDeferred(obj) _PyStackRef_XNewRefDeferred(_PyObject_CAST(obj))
|
||||
assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
|
||||
assert(obj != NULL);
|
||||
assert(_Py_IsImmortal(obj));
|
||||
return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
|
||||
#else
|
||||
# define PyStackRef_XNewRefDeferred(obj) PyStackRef_XNewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))}))
|
||||
assert(_Py_IsImmortal(obj));
|
||||
return ((_PyStackRef){ .bits = (uintptr_t)(obj) });
|
||||
#endif
|
||||
|
||||
// Converts a PyStackRef back to a PyObject *.
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
static inline PyObject *
|
||||
PyStackRef_StealObject(_PyStackRef tagged)
|
||||
{
|
||||
if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
|
||||
assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged)));
|
||||
return Py_NewRef(PyStackRef_Get(tagged));
|
||||
}
|
||||
return PyStackRef_Get(tagged);
|
||||
}
|
||||
#else
|
||||
# define PyStackRef_StealObject(tagged) PyStackRef_Get(tagged)
|
||||
#endif
|
||||
|
||||
static inline void
|
||||
_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length)
|
||||
{
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
dst[i] = PyStackRef_Get(src[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
_Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length)
|
||||
{
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
dst[i] = PyStackRef_StealObject(src[i]);
|
||||
}
|
||||
}
|
||||
#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj))
|
||||
|
||||
|
||||
#define PyStackRef_XSETREF(dst, src) \
|
||||
do { \
|
||||
_PyStackRef *_tmp_dst_ptr = &(dst); \
|
||||
_PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
|
||||
*_tmp_dst_ptr = (src); \
|
||||
PyStackRef_XDECREF(_tmp_old_dst); \
|
||||
} while (0)
|
||||
|
||||
#define PyStackRef_SETREF(dst, src) \
|
||||
do { \
|
||||
_PyStackRef *_tmp_dst_ptr = &(dst); \
|
||||
_PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
|
||||
*_tmp_dst_ptr = (src); \
|
||||
PyStackRef_DECREF(_tmp_old_dst); \
|
||||
} while (0)
|
||||
|
||||
#define PyStackRef_CLEAR(op) \
|
||||
do { \
|
||||
_PyStackRef *_tmp_op_ptr = &(op); \
|
||||
_PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
|
||||
if (_tmp_old_op.bits != Py_STACKREF_NULL.bits) { \
|
||||
*_tmp_op_ptr = Py_STACKREF_NULL; \
|
||||
PyStackRef_DECREF(_tmp_old_op); \
|
||||
if (!PyStackRef_IsNull(_tmp_old_op)) { \
|
||||
*_tmp_op_ptr = PyStackRef_NULL; \
|
||||
PyStackRef_CLOSE(_tmp_old_op); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
static inline void
|
||||
PyStackRef_DECREF(_PyStackRef tagged)
|
||||
PyStackRef_CLOSE(_PyStackRef stackref)
|
||||
{
|
||||
if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
|
||||
#ifdef Py_GIL_DISABLED
|
||||
if (PyStackRef_IsDeferred(stackref)) {
|
||||
// No assert for being immortal or deferred here.
|
||||
// The GC unsets deferred objects right before clearing.
|
||||
return;
|
||||
}
|
||||
Py_DECREF(PyStackRef_Get(tagged));
|
||||
}
|
||||
Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
#else
|
||||
# define PyStackRef_DECREF(op) Py_DECREF(PyStackRef_Get(op))
|
||||
Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
#endif
|
||||
|
||||
#if defined(Py_GIL_DISABLED)
|
||||
static inline void
|
||||
PyStackRef_INCREF(_PyStackRef tagged)
|
||||
{
|
||||
if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) {
|
||||
assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged)));
|
||||
return;
|
||||
}
|
||||
Py_INCREF(PyStackRef_Get(tagged));
|
||||
}
|
||||
#else
|
||||
# define PyStackRef_INCREF(op) Py_INCREF(PyStackRef_Get(op))
|
||||
#endif
|
||||
|
||||
static inline void
|
||||
PyStackRef_XDECREF(_PyStackRef op)
|
||||
#define PyStackRef_XCLOSE(stackref) \
|
||||
do { \
|
||||
_PyStackRef _tmp = (stackref); \
|
||||
if (!PyStackRef_IsNull(_tmp)) { \
|
||||
PyStackRef_CLOSE(_tmp); \
|
||||
} \
|
||||
} while (0);
|
||||
|
||||
|
||||
static inline _PyStackRef
|
||||
PyStackRef_DUP(_PyStackRef stackref)
|
||||
{
|
||||
if (op.bits != Py_STACKREF_NULL.bits) {
|
||||
PyStackRef_DECREF(op);
|
||||
#ifdef Py_GIL_DISABLED
|
||||
if (PyStackRef_IsDeferred(stackref)) {
|
||||
assert(PyStackRef_IsNull(stackref) ||
|
||||
_Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)));
|
||||
return stackref;
|
||||
}
|
||||
Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
return stackref;
|
||||
#else
|
||||
Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
|
||||
return stackref;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline _PyStackRef
|
||||
PyStackRef_NewRef(_PyStackRef obj)
|
||||
PyStackRef_XDUP(_PyStackRef stackref)
|
||||
{
|
||||
PyStackRef_INCREF(obj);
|
||||
return obj;
|
||||
if (!PyStackRef_IsNull(stackref)) {
|
||||
return PyStackRef_DUP(stackref);
|
||||
}
|
||||
return stackref;
|
||||
}
|
||||
|
||||
static inline _PyStackRef
|
||||
PyStackRef_XNewRef(_PyStackRef obj)
|
||||
|
||||
static inline void
|
||||
_PyObjectStack_FromStackRefStack(PyObject **dst, const _PyStackRef *src, size_t length)
|
||||
{
|
||||
if (obj.bits == Py_STACKREF_NULL.bits) {
|
||||
return obj;
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
dst[i] = PyStackRef_AsPyObjectBorrow(src[i]);
|
||||
}
|
||||
return PyStackRef_NewRef(obj);
|
||||
}
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue