mirror of
https://github.com/python/cpython.git
synced 2025-10-15 11:22:18 +00:00
22b0de2755
This PR sets up tagged pointers for CPython. The general idea is to create a separate struct _PyStackRef for everything on the evaluation stack to store the bits. This forces the C compiler to warn us if we try to cast things or pull things out of the struct directly. Only for free threading: We tag the low bit if something is deferred - that means we skip incref and decref operations on it. This behavior may change in the future if Mark's plans to defer all objects in the interpreter loop pans out. This implies a strict stack reference discipline is required. ALL incref and decref operations on stackrefs must use the stackref variants. It is unsafe to untag something then do normal incref/decref ops on it. The new incref and decref variants are called dup and close. They mimic a "handle" API operating on these stackrefs. Please read Include/internal/pycore_stackref.h for more information! --------- Co-authored-by: Mark Shannon <9448417+markshannon@users.noreply.github.com>
136 lines
4 KiB
C
136 lines
4 KiB
C
#include "Python.h"
|
|
|
|
#include "pycore_backoff.h"
|
|
#include "pycore_call.h"
|
|
#include "pycore_ceval.h"
|
|
#include "pycore_cell.h"
|
|
#include "pycore_dict.h"
|
|
#include "pycore_emscripten_signal.h"
|
|
#include "pycore_intrinsics.h"
|
|
#include "pycore_jit.h"
|
|
#include "pycore_long.h"
|
|
#include "pycore_opcode_metadata.h"
|
|
#include "pycore_opcode_utils.h"
|
|
#include "pycore_optimizer.h"
|
|
#include "pycore_pyatomic_ft_wrappers.h"
|
|
#include "pycore_range.h"
|
|
#include "pycore_setobject.h"
|
|
#include "pycore_sliceobject.h"
|
|
#include "pycore_descrobject.h"
|
|
#include "pycore_stackref.h"
|
|
|
|
#include "ceval_macros.h"
|
|
|
|
#undef CURRENT_OPARG
|
|
#define CURRENT_OPARG() (_oparg)
|
|
|
|
#undef CURRENT_OPERAND
|
|
#define CURRENT_OPERAND() (_operand)
|
|
|
|
#undef DEOPT_IF
|
|
#define DEOPT_IF(COND, INSTNAME) \
|
|
do { \
|
|
if ((COND)) { \
|
|
goto deoptimize; \
|
|
} \
|
|
} while (0)
|
|
|
|
#undef ENABLE_SPECIALIZATION
|
|
#define ENABLE_SPECIALIZATION (0)
|
|
|
|
#undef GOTO_ERROR
|
|
#define GOTO_ERROR(LABEL) \
|
|
do { \
|
|
goto LABEL ## _tier_two; \
|
|
} while (0)
|
|
|
|
#undef GOTO_TIER_TWO
|
|
#define GOTO_TIER_TWO(EXECUTOR) \
|
|
do { \
|
|
OPT_STAT_INC(traces_executed); \
|
|
__attribute__((musttail)) \
|
|
return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \
|
|
} while (0)
|
|
|
|
#undef GOTO_TIER_ONE
|
|
#define GOTO_TIER_ONE(TARGET) \
|
|
do { \
|
|
_PyFrame_SetStackPointer(frame, stack_pointer); \
|
|
return TARGET; \
|
|
} while (0)
|
|
|
|
#undef LOAD_IP
|
|
#define LOAD_IP(UNUSED) \
|
|
do { \
|
|
} while (0)
|
|
|
|
#define PATCH_VALUE(TYPE, NAME, ALIAS) \
|
|
PyAPI_DATA(void) ALIAS; \
|
|
TYPE NAME = (TYPE)(uintptr_t)&ALIAS;
|
|
|
|
#define PATCH_JUMP(ALIAS) \
|
|
do { \
|
|
PyAPI_DATA(void) ALIAS; \
|
|
__attribute__((musttail)) \
|
|
return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \
|
|
} while (0)
|
|
|
|
#undef JUMP_TO_JUMP_TARGET
|
|
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
|
|
|
|
#undef JUMP_TO_ERROR
|
|
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
|
|
|
|
#undef WITHIN_STACK_BOUNDS
|
|
#define WITHIN_STACK_BOUNDS() 1
|
|
|
|
_Py_CODEUNIT *
|
|
_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate)
|
|
{
|
|
// Locals that the instruction implementations expect to exist:
|
|
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
|
|
int oparg;
|
|
int uopcode = _JIT_OPCODE;
|
|
_Py_CODEUNIT *next_instr;
|
|
// Other stuff we need handy:
|
|
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
|
|
#if SIZEOF_VOID_P == 8
|
|
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
|
|
#else
|
|
assert(SIZEOF_VOID_P == 4);
|
|
PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI)
|
|
PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO)
|
|
uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo;
|
|
#endif
|
|
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
|
|
PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX)
|
|
|
|
OPT_STAT_INC(uops_executed);
|
|
UOP_STAT_INC(uopcode, execution_count);
|
|
|
|
// The actual instruction definitions (only one will be used):
|
|
switch (uopcode) {
|
|
#include "executor_cases.c.h"
|
|
default:
|
|
Py_UNREACHABLE();
|
|
}
|
|
PATCH_JUMP(_JIT_CONTINUE);
|
|
// Labels that the instruction implementations expect to exist:
|
|
|
|
error_tier_two:
|
|
tstate->previous_executor = (PyObject *)current_executor;
|
|
GOTO_TIER_ONE(NULL);
|
|
exit_to_tier1:
|
|
tstate->previous_executor = (PyObject *)current_executor;
|
|
GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
|
|
exit_to_tier1_dynamic:
|
|
tstate->previous_executor = (PyObject *)current_executor;
|
|
GOTO_TIER_ONE(frame->instr_ptr);
|
|
exit_to_trace:
|
|
{
|
|
_PyExitData *exit = ¤t_executor->exits[_exit_index];
|
|
Py_INCREF(exit->executor);
|
|
tstate->previous_executor = (PyObject *)current_executor;
|
|
GOTO_TIER_TWO(exit->executor);
|
|
}
|
|
}
|