mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 05:58:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

GH-132508: Use tagged integers on the evaluation stack for the last instruction offset (GH-132545)

Browse source
This commit is contained in:
Mark Shannon 2025-04-29 18:00:35 +01:00 • committed by GitHub
parent caee16f052
commit ccf1b0b1c1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 129 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Include/internal/pycore_interpframe.h
View file

@ -18,6 +18,7 @@ extern "C" {
((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF))))
static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
assert(!PyStackRef_IsNull(f->f_executable));
PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable);
assert(PyCode_Check(executable));
return (PyCodeObject *)executable;

72
Include/internal/pycore_stackref.h
View file

@ -63,11 +63,13 @@ extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _P
static const _PyStackRef PyStackRef_NULL = { .index = 0 };
#define PyStackRef_None ((_PyStackRef){ .index = 1 } )
#define PyStackRef_False ((_PyStackRef){ .index = 2 })
#define PyStackRef_True ((_PyStackRef){ .index = 3 })
// Use the first 3 even numbers for None, True and False.
// Odd numbers are reserved for (tagged) integers
#define PyStackRef_None ((_PyStackRef){ .index = 2 } )
#define PyStackRef_False ((_PyStackRef){ .index = 4 })
#define PyStackRef_True ((_PyStackRef){ .index = 6 })
#define LAST_PREDEFINED_STACKREF_INDEX 3
#define INITIAL_STACKREF_INDEX 8
static inline int
PyStackRef_IsNull(_PyStackRef ref)
@ -96,6 +98,7 @@ PyStackRef_IsNone(_PyStackRef ref)
static inline PyObject *
_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
{
assert((ref.index & 1) == 0);
_Py_stackref_record_borrow(ref, filename, linenumber);
return _Py_stackref_get_object(ref);
}
@ -132,32 +135,46 @@ _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenu
}
#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__)
static inline bool
PyStackRef_IsTaggedInt(_PyStackRef ref)
{
return (ref.index & 1) == 1;
}
static inline void
_PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
if (PyStackRef_IsTaggedInt(ref)) {
return;
}
PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
Py_DECREF(obj);
}
#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
static inline void
_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
if (PyStackRef_IsNull(ref)) {
return;
}
PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
Py_DECREF(obj);
_PyStackRef_CLOSE(ref, filename, linenumber);
}
#define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)
static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
if (PyStackRef_IsTaggedInt(ref)) {
return ref;
}
else {
PyObject *obj = _Py_stackref_get_object(ref);
Py_INCREF(obj);
return _Py_stackref_create(obj, filename, linenumber);
}
}
#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber);
@ -210,8 +227,40 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linen
extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);
extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);
extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);
extern _PyStackRef PyStackRef_TagInt(intptr_t i);
extern bool
PyStackRef_IsNullOrInt(_PyStackRef ref);
#else
#define Py_INT_TAG 3
static inline bool
PyStackRef_IsTaggedInt(_PyStackRef i)
{
return (i.bits & Py_INT_TAG) == Py_INT_TAG;
}
static inline _PyStackRef
PyStackRef_TagInt(intptr_t i)
{
assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i);
return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) };
}
static inline intptr_t
PyStackRef_UntagInt(_PyStackRef i)
{
assert((i.bits & Py_INT_TAG) == Py_INT_TAG);
intptr_t val = (intptr_t)i.bits;
return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2);
}
#ifdef Py_GIL_DISABLED
@ -232,6 +281,8 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
{
@ -451,6 +502,7 @@ PyStackRef_RefcountOnObject(_PyStackRef ref)
static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
{
assert(!PyStackRef_IsTaggedInt(ref));
return BITS_TO_PTR_MASKED(ref);
}
@ -587,6 +639,12 @@ PyStackRef_CLOSE(_PyStackRef ref)
}
#endif
static inline bool
PyStackRef_IsNullOrInt(_PyStackRef ref)
{
return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
}
static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
@ -726,7 +784,7 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
// Like Py_VISIT but for _PyStackRef fields
#define _Py_VISIT_STACKREF(ref) \
do { \
if (!PyStackRef_IsNull(ref)) { \
if (!PyStackRef_IsNullOrInt(ref)) { \
int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \
if (vret) \
return vret; \

3
Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst Normal file
View file

@ -0,0 +1,3 @@
Uses tagged integers on the evaluation stack to represent the instruction
offsets when reraising an exception. This avoids the need to box the integer
which could fail in low memory conditions.

5
Objects/frameobject.c
View file

@ -1813,15 +1813,16 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value)
start_stack = pop_value(start_stack);
}
while (start_stack > best_stack) {
_PyStackRef popped = _PyFrame_StackPop(self->f_frame);
if (top_of_stack(start_stack) == Except) {
/* Pop exception stack as well as the evaluation stack */
PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(self->f_frame));
PyObject *exc = PyStackRef_AsPyObjectBorrow(popped);
assert(PyExceptionInstance_Check(exc) || exc == Py_None);
PyThreadState *tstate = _PyThreadState_GET();
Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc);
}
else {
PyStackRef_XCLOSE(_PyFrame_StackPop(self->f_frame));
PyStackRef_XCLOSE(popped);
}
start_stack = pop_value(start_stack);
}

20
Python/bytecodes.c
View file

@ -1384,16 +1384,7 @@ dummy_func(
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
}
else {
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
Py_DECREF(exc);
ERROR_NO_POP();
}
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]);
}
assert(exc && PyExceptionInstance_Check(exc));
_PyErr_SetRaisedException(tstate, exc);
@ -3472,7 +3463,7 @@ dummy_func(
if (tb == NULL) {
tb = Py_None;
}
assert(PyStackRef_LongCheck(lasti));
assert(PyStackRef_IsTaggedInt(lasti));
(void)lasti; // Shut up compiler warning if asserts are off
PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
int has_self = !PyStackRef_IsNull(exit_self);
@ -5378,11 +5369,8 @@ dummy_func(
}
if (lasti) {
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
PyObject *lasti = PyLong_FromLong(frame_lasti);
if (lasti == NULL) {
goto exception_unwind;
}
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti));
_PyStackRef lasti = PyStackRef_TagInt(frame_lasti);
_PyFrame_StackPush(frame, lasti);
}
/* Make the raw exception data

4
Python/ceval.c
View file

@ -146,6 +146,10 @@ dump_item(_PyStackRef item)
printf("<NULL>");
return;
}
if (PyStackRef_IsTaggedInt(item)) {
printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
return;
}
PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
if (obj == NULL) {
printf("<nil>");

2
Python/executor_cases.c.h generated
View file

@ -4470,7 +4470,7 @@
if (tb == NULL) {
tb = Py_None;
}
assert(PyStackRef_LongCheck(lasti));
assert(PyStackRef_IsTaggedInt(lasti));
(void)lasti;
PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
int has_self = !PyStackRef_IsNull(exit_self);

8
Python/gc.c
View file

@ -547,6 +547,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
// This is a bit tricky! We want to ignore stackrefs with embedded
// refcounts when computing the incoming references, but otherwise treat
// them like normal.
assert(!PyStackRef_IsTaggedInt(*ref));
if (!PyStackRef_RefcountOnObject(*ref) && (visit == visit_decref)) {
return 0;
}
@ -560,8 +561,10 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg)
_PyStackRef *ref = _PyFrame_GetLocalsArray(frame);
/* locals and stack */
for (; ref < frame->stackpointer; ref++) {
if (!PyStackRef_IsTaggedInt(*ref)) {
_Py_VISIT_STACKREF(*ref);
}
}
return 0;
}
@ -1495,8 +1498,11 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b
objects_marked += move_to_reachable(func, &reachable, visited_space);
while (sp > locals) {
sp--;
if (PyStackRef_IsNullOrInt(*sp)) {
continue;
}
PyObject *op = PyStackRef_AsPyObjectBorrow(*sp);
if (op == NULL || _Py_IsImmortal(op)) {
if (_Py_IsImmortal(op)) {
continue;
}
if (_PyObject_IS_GC(op)) {

7
Python/gc_free_threading.c
View file

@ -265,7 +265,7 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame)
frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj);
for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) {
if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) {
if (!PyStackRef_IsNullOrInt(*ref) && PyStackRef_IsDeferred(*ref)) {
*ref = PyStackRef_AsStrongReference(*ref);
}
}
@ -420,7 +420,7 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor,
static inline void
gc_visit_stackref(_PyStackRef stackref)
{
if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) {
if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNullOrInt(stackref)) {
PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
if (_PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) {
gc_add_refs(obj, 1);
@ -817,7 +817,7 @@ gc_abort_mark_alive(PyInterpreterState *interp,
static int
gc_visit_stackref_mark_alive(gc_mark_args_t *args, _PyStackRef stackref)
{
if (!PyStackRef_IsNull(stackref)) {
if (!PyStackRef_IsNullOrInt(stackref)) {
PyObject *op = PyStackRef_AsPyObjectBorrow(stackref);
if (gc_mark_enqueue(op, args) < 0) {
return -1;
@ -1706,6 +1706,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg)
// This is a bit tricky! We want to ignore deferred references when
// computing the incoming references, but otherwise treat them like
// regular references.
assert(!PyStackRef_IsTaggedInt(*ref));
if (!PyStackRef_IsDeferred(*ref) ||
(visit != visit_decref && visit != visit_decref_unreachable))
{

24
Python/generated_cases.c.h generated
View file

@ -10191,20 +10191,7 @@
PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st);
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]);
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
}
else {
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
Py_DECREF(exc);
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_LABEL(error);
}
frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]);
}
assert(exc && PyExceptionInstance_Check(exc));
stack_pointer += -1;
@ -12059,7 +12046,7 @@
if (tb == NULL) {
tb = Py_None;
}
assert(PyStackRef_LongCheck(lasti));
assert(PyStackRef_IsTaggedInt(lasti));
(void)lasti;
PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb};
int has_self = !PyStackRef_IsNull(exit_self);
@ -12231,11 +12218,8 @@ JUMP_TO_LABEL(error);
}
if (lasti) {
int frame_lasti = _PyInterpreterFrame_LASTI(frame);
PyObject *lasti = PyLong_FromLong(frame_lasti);
if (lasti == NULL) {
JUMP_TO_LABEL(exception_unwind);
}
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti));
_PyStackRef lasti = PyStackRef_TagInt(frame_lasti);
_PyFrame_StackPush(frame, lasti);
}
PyObject *exc = _PyErr_GetRaisedException(tstate);
_PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc));

2
Python/pystate.c
View file

@ -684,7 +684,7 @@ init_interpreter(PyInterpreterState *interp,
interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp);
}
#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
interp->next_stackref = 1;
interp->next_stackref = INITIAL_STACKREF_INDEX;
_Py_hashtable_allocator_t alloc = {
.malloc = malloc,
.free = free,

29
Python/stackrefs.c
View file

@ -70,7 +70,7 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber)
}
PyObject *obj;
if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) {
if (ref.index < INITIAL_STACKREF_INDEX) {
if (ref.index == 0) {
_Py_FatalErrorFormat(__func__, "Passing NULL to PyStackRef_CLOSE at %s:%d\n", filename, linenumber);
}
@ -113,7 +113,8 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber)
Py_FatalError("Cannot create a stackref for NULL");
}
PyInterpreterState *interp = PyInterpreterState_Get();
uint64_t new_id = interp->next_stackref++;
uint64_t new_id = interp->next_stackref;
interp->next_stackref = new_id + 2;
TableEntry *entry = make_table_entry(obj, filename, linenumber);
if (entry == NULL) {
Py_FatalError("No memory left for stackref debug table");
@ -127,7 +128,7 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber)
void
_Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber)
{
if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) {
if (ref.index < INITIAL_STACKREF_INDEX) {
return;
}
PyInterpreterState *interp = PyInterpreterState_Get();
@ -151,8 +152,7 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber
void
_Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref)
{
assert(interp->next_stackref >= ref.index);
interp->next_stackref = ref.index+1;
assert(ref.index < INITIAL_STACKREF_INDEX);
TableEntry *entry = make_table_entry(obj, "builtin-object", 0);
if (entry == NULL) {
Py_FatalError("No memory left for stackref debug table");
@ -197,4 +197,23 @@ _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *
_Py_DECREF_SPECIALIZED(obj, destruct);
}
_PyStackRef PyStackRef_TagInt(intptr_t i)
{
return (_PyStackRef){ .index = (i << 1) + 1 };
}
intptr_t
PyStackRef_UntagInt(_PyStackRef i)
{
assert(PyStackRef_IsTaggedInt(i));
intptr_t val = (intptr_t)i.index;
return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 1);
}
bool
PyStackRef_IsNullOrInt(_PyStackRef ref)
{
return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
}
#endif

3
Tools/cases_generator/analyzer.py
View file

@ -678,6 +678,9 @@ NON_ESCAPING_FUNCTIONS = (
"JUMP_TO_LABEL",
"restart_backoff_counter",
"_Py_ReachedRecursionLimit",
"PyStackRef_IsTaggedInt",
"PyStackRef_TagInt",
"PyStackRef_UntagInt",
)
def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None:

1
Tools/jit/_stencils.py
View file

@ -293,6 +293,7 @@ class StencilGroup:
hole.kind
in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"}
and hole.value is HoleValue.ZERO
and hole.symbol not in self.symbols
):
hole.func = "patch_aarch64_trampoline"
hole.need_state = True