diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fea8665ae39..6265afba38b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -274,7 +274,7 @@ PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame * PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v, int argcnt, int argcntafter, _PyStackRef *sp); +PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 24c698adb31..a59150bb69b 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -2001,21 +2001,21 @@ struct opcode_metadata { extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { - [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2053,7 +2053,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -2075,7 +2075,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, - [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -2132,8 +2132,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -2169,12 +2169,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_ITER] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -2183,7 +2183,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, 0 }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -2206,12 +2206,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -2219,7 +2219,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 86a4843ea05..1a1fd328195 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -53,26 +53,26 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_POP_TOP] = HAS_PURE_FLAG, + [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, - [_END_SEND] = HAS_PURE_FLAG, + [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, - [_TO_BOOL_INT] = HAS_EXIT_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_LIST] = HAS_EXIT_FLAG, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_EXIT_FLAG, - [_REPLACE_WITH_TRUE] = 0, + [_TO_BOOL_STR] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_REPLACE_WITH_TRUE] = HAS_ESCAPES_FLAG, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BOTH_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, @@ -81,7 +81,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -100,7 +100,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = 0, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -112,7 +112,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -175,7 +175,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = 0, @@ -204,8 +204,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, - [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG, - [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, @@ -261,7 +261,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 0e0f28be6b2..f9a54f3b412 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1723,6 +1723,7 @@ class TestGeneratedCases(unittest.TestCase): input = """ inst(BALANCED, ( -- )) { SAVE_STACK(); + code(); RELOAD_STACK(); } """ @@ -1737,12 +1738,36 @@ class TestGeneratedCases(unittest.TestCase): next_instr += 1; INSTRUCTION_STATS(BALANCED); _PyFrame_SetStackPointer(frame, stack_pointer); + code(); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } """ self.run_cases_test(input, output) + def test_stack_save_reload_paired(self): + + input = """ + inst(BALANCED, ( -- )) { + SAVE_STACK(); + RELOAD_STACK(); + } + """ + + output = """ + TARGET(BALANCED) { + #if defined(Py_TAIL_CALL_INTERP) + int opcode = BALANCED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BALANCED); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + def test_stack_reload_only(self): input = """ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-10-11-30-13.gh-issue-129953.wipsl_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-10-11-30-13.gh-issue-129953.wipsl_.rst new file mode 100644 index 00000000000..db9905eeaee --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-10-11-30-13.gh-issue-129953.wipsl_.rst @@ -0,0 +1,4 @@ +The internal (evaluation) stack is now spilled to memory whenever execution +esacpes from the interpreter or JIT compiled code. This should have no +observable effect in either Python or builtin extensions, but will allow +various important optimizations in the future. diff --git a/Objects/object.c b/Objects/object.c index c3f57f47333..2dd50339c58 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2994,6 +2994,11 @@ _Py_Dealloc(PyObject *op) destructor dealloc = type->tp_dealloc; #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); +#ifndef Py_GIL_DISABLED + /* This assertion doesn't hold for the free-threading build, as + * PyStackRef_CLOSE_SPECIALIZED is not implemented */ + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#endif PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior // on (tstate->curexc_type != old_exc_type) below diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 703d7ec61eb..b18b0075315 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -361,7 +361,7 @@ dummy_func( } pure inst(POP_TOP, (value --)) { - DECREF_INPUTS(); + PyStackRef_CLOSE(value); } pure inst(PUSH_NULL, (-- res)) { @@ -388,7 +388,7 @@ dummy_func( ERROR_NO_POP(); } } - DECREF_INPUTS(); + PyStackRef_CLOSE(value); } tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) { @@ -400,7 +400,7 @@ dummy_func( (void)receiver; val = value; DEAD(value); - DECREF_INPUTS(); + PyStackRef_CLOSE(receiver); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- val)) { @@ -418,7 +418,7 @@ dummy_func( inst(UNARY_NEGATIVE, (value -- res)) { PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -453,7 +453,7 @@ dummy_func( op(_TO_BOOL, (value -- res)) { int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(err < 0, error); res = err ? PyStackRef_True : PyStackRef_False; } @@ -475,7 +475,7 @@ dummy_func( res = PyStackRef_False; } else { - DECREF_INPUTS(); + PyStackRef_CLOSE(value); res = PyStackRef_True; } } @@ -507,13 +507,13 @@ dummy_func( } else { assert(Py_SIZE(value_o)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); res = PyStackRef_True; } } op(_REPLACE_WITH_TRUE, (value -- res)) { - DECREF_INPUTS(); + PyStackRef_CLOSE(value); res = PyStackRef_True; } @@ -524,7 +524,7 @@ dummy_func( inst(UNARY_INVERT, (value -- res)) { PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -721,7 +721,7 @@ dummy_func( // At the end we just skip over the STORE_FAST. op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); @@ -752,8 +752,7 @@ dummy_func( PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyUnicode_Append(&temp, right_o); *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - DEAD(right); + Py_DECREF(right_o); ERROR_IF(PyStackRef_IsNull(*target_local), error); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -967,7 +966,7 @@ dummy_func( inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(err, error); } @@ -1048,7 +1047,7 @@ dummy_func( inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - DECREF_INPUTS(); + PyStackRef_CLOSE(value); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1135,12 +1134,12 @@ dummy_func( "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - DECREF_INPUTS(); + PyStackRef_CLOSE(obj); ERROR_IF(true, error); } iter_o = (*getter)(obj_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(obj); ERROR_IF(iter_o == NULL, error); if (Py_TYPE(iter_o)->tp_as_async == NULL || @@ -1166,7 +1165,7 @@ dummy_func( inst(GET_AWAITABLE, (iterable -- iter)) { PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); } @@ -1228,7 +1227,7 @@ dummy_func( JUMPBY(oparg); } else { - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(true, error); } } @@ -1357,7 +1356,7 @@ dummy_func( } } - tier1 inst(CLEANUP_THROW, (sub_iter_st, last_sent_val_st, exc_value_st -- none, value)) { + tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value_st -- none, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #ifndef Py_TAIL_CALL_INTERP assert(throwflag); @@ -1366,9 +1365,9 @@ dummy_func( int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { - none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); DECREF_INPUTS(); + none = PyStackRef_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); @@ -1410,7 +1409,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(true, error); } if (PyDict_CheckExact(ns)) { @@ -1419,7 +1418,7 @@ dummy_func( else { err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); } - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(err, error); } @@ -1462,10 +1461,10 @@ dummy_func( (void)counter; } - op(_UNPACK_SEQUENCE, (seq -- output[oparg])) { - _PyStackRef *top = output + oparg; - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - DECREF_INPUTS(); + op(_UNPACK_SEQUENCE, (seq -- output[oparg], top[0])) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); ERROR_IF(res == 0, error); } @@ -1479,7 +1478,7 @@ dummy_func( STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - DECREF_INPUTS(); + PyStackRef_CLOSE(seq); } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { @@ -1511,10 +1510,10 @@ dummy_func( DECREF_INPUTS(); } - inst(UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8])) { - _PyStackRef *top = right + (oparg >> 8); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); - DECREF_INPUTS(); + inst(UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8], top[0])) { + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); ERROR_IF(res == 0, error); } @@ -1550,14 +1549,14 @@ dummy_func( inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - DECREF_INPUTS(); + PyStackRef_CLOSE(v); ERROR_IF(err, error); } @@ -1589,7 +1588,7 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *v_o; int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(mod_or_class_dict); ERROR_IF(err < 0, error); if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -1885,17 +1884,17 @@ dummy_func( "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable_st); ERROR_IF(true, error); } assert(Py_IsNone(none_val)); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable_st); } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable); ERROR_IF(err < 0, error); } @@ -1970,10 +1969,10 @@ dummy_func( "'%.200s' object is not a mapping", Py_TYPE(update_o)->tp_name); } - DECREF_INPUTS(); + PyStackRef_CLOSE(update); ERROR_IF(true, error); } - DECREF_INPUTS(); + PyStackRef_CLOSE(update); } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { @@ -1984,10 +1983,10 @@ dummy_func( int err = _PyDict_MergeEx(dict_o, update_o, 2); if (err < 0) { _PyEval_FormatKwargsError(tstate, callable_o, update_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(update); ERROR_IF(true, error); } - DECREF_INPUTS(); + PyStackRef_CLOSE(update); } inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) { @@ -2172,7 +2171,7 @@ dummy_func( CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL, error); self_or_null[0] = PyStackRef_NULL; } @@ -2180,13 +2179,12 @@ dummy_func( else { /* Classic, pushes one value. */ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL, error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } - macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + unused/8 + @@ -2736,12 +2734,11 @@ dummy_func( assert(PyExceptionInstance_Check(left_o)); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); if (err < 0) { - DECREF_INPUTS(); - ERROR_IF(true, error); + ERROR_NO_POP(); } int res = PyErr_GivenExceptionMatches(left_o, right_o); - DECREF_INPUTS(); + PyStackRef_CLOSE(right); b = res ? PyStackRef_True : PyStackRef_False; } @@ -2968,7 +2965,7 @@ dummy_func( inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - DECREF_INPUTS(); + PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); } @@ -3410,7 +3407,7 @@ dummy_func( assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); } @@ -3426,7 +3423,7 @@ dummy_func( assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - DECREF_INPUTS(); + PyStackRef_CLOSE(owner); attr = PyStackRef_FromPyObjectNew(descr); } @@ -4797,9 +4794,11 @@ dummy_func( assert(_PyEval_BinaryOps[oparg]); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - DECREF_INPUTS(); - ERROR_IF(res_o == NULL, error); + if (res_o == NULL) { + ERROR_NO_POP(); + } res = PyStackRef_FromPyObjectSteal(res_o); + DECREF_INPUTS(); } macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; @@ -5267,10 +5266,12 @@ dummy_func( goto exit_unwind; } next_instr = frame->instr_ptr; - - LLTRACE_RESUME_FRAME(); - #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; /* _PyEval_EvalFrameDefault() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ diff --git a/Python/ceval.c b/Python/ceval.c index 1684fd7ecbd..931915c8cf7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -135,6 +135,7 @@ static void dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef *stack_base = _PyFrame_Stackbase(frame); PyObject *exc = PyErr_GetRaisedException(); printf(" stack=["); @@ -165,6 +166,7 @@ dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) printf("]\n"); fflush(stdout); PyErr_SetRaisedException(exc); + _PyFrame_GetStackPointer(frame); } static void @@ -799,6 +801,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifndef Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); #endif _PyInterpreterFrame entry_frame; @@ -858,8 +861,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, we need to update instrumentation */ _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); monitor_throw(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef Py_TAIL_CALL_INTERP return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0); #else @@ -2012,7 +2015,7 @@ _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *frame, PyObject* exc_value, */ int -_PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, +_PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp) { int i = 0, j = 0; @@ -2020,8 +2023,6 @@ _PyEval_UnpackIterableStackRef(PyThreadState *tstate, _PyStackRef v_stackref, PyObject *it; /* iter(v) */ PyObject *w; PyObject *l = NULL; /* variable list */ - - PyObject *v = PyStackRef_AsPyObjectBorrow(v_stackref); assert(v != NULL); it = PyObject_GetIter(v); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0a4f65feb3b..50e940c1973 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -118,7 +118,9 @@ #ifdef Py_DEBUG #define LLTRACE_RESUME_FRAME() \ do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ if (lltrace < 0) { \ JUMP_TO_LABEL(exit_unwind); \ } \ @@ -409,17 +411,21 @@ do { \ } while (0) #endif -#define GOTO_TIER_ONE(TARGET) \ -do { \ - next_instr = (TARGET); \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - Py_CLEAR(tstate->previous_executor); \ - if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ - goto error; \ - } \ - DISPATCH(); \ -} while (0) +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + next_instr = (TARGET); \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + Py_CLEAR(tstate->previous_executor); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) \ + { \ + next_instr = frame->instr_ptr; \ + goto error; \ + } \ + DISPATCH(); \ + } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND0() (next_uop[-1].operand0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 96b7386bd24..5962e0ebd4f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -438,9 +438,11 @@ case _POP_TOP: { _PyStackRef value; value = stack_pointer[-1]; - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -477,10 +479,12 @@ receiver = stack_pointer[-2]; (void)receiver; val = value; - PyStackRef_CLOSE(receiver); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -491,14 +495,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -520,14 +528,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = err ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -557,8 +569,14 @@ res = PyStackRef_False; } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; break; @@ -575,7 +593,12 @@ } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = res; break; } @@ -611,8 +634,14 @@ } else { assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; break; @@ -622,9 +651,15 @@ _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -635,14 +670,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -697,7 +736,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -723,7 +764,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -749,7 +792,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -935,7 +980,7 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); int next_oparg; @@ -965,12 +1010,16 @@ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*target_local)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } #if TIER_ONE @@ -979,8 +1028,6 @@ assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); #endif - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1019,12 +1066,20 @@ STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1099,15 +1154,23 @@ stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-1] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -1276,17 +1339,25 @@ _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (rc <= 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1372,14 +1443,14 @@ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -1393,17 +1464,24 @@ /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -1495,16 +1573,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -1517,14 +1599,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1540,17 +1626,23 @@ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1598,24 +1690,26 @@ "__aiter__ method, got %.100s", type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(obj); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " @@ -1626,7 +1720,9 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1655,14 +1751,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -1810,9 +1910,11 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) { @@ -1825,14 +1927,14 @@ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(v); - if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -1865,21 +1967,21 @@ case _UNPACK_SEQUENCE: { _PyStackRef seq; - _PyStackRef *output; + _PyStackRef *top; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += -1 + oparg; + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1903,11 +2005,13 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -1931,8 +2035,12 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); break; } @@ -1965,29 +2073,33 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _UNPACK_EX: { _PyStackRef seq; - _PyStackRef *right; + _PyStackRef *top; oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } - stack_pointer += (oparg & 0xFF) + (oparg >> 8); + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2002,16 +2114,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -2023,14 +2139,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -2042,14 +2158,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_ERROR(); + } break; } @@ -2401,26 +2517,36 @@ pieces = &stack_pointer[-oparg]; STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + stack_pointer[0] = str; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2482,15 +2608,19 @@ Py_TYPE(iterable)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2504,14 +2634,14 @@ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_ERROR(); + } break; } @@ -2524,9 +2654,14 @@ PyObject *set_o = PySet_New(NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -2539,20 +2674,25 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (err != 0) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + stack_pointer[0] = set; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2564,9 +2704,14 @@ values = &stack_pointer[-oparg*2]; STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -2578,17 +2723,22 @@ oparg); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + stack_pointer[0] = map; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2655,14 +2805,18 @@ Py_TYPE(update_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2684,14 +2838,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -2749,18 +2907,27 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attr == NULL) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; + stack_pointer[0] = attr_st; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -2810,13 +2977,26 @@ stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2850,13 +3030,17 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } } else { @@ -2864,12 +3048,16 @@ _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; @@ -3095,10 +3283,15 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - PyStackRef_CLOSE(owner); - stack_pointer[-2] = attr; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; break; } @@ -3124,7 +3317,12 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = attr; break; } @@ -3154,7 +3352,12 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = attr; break; } @@ -3345,17 +3548,21 @@ assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); @@ -3367,8 +3574,6 @@ } else { res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } stack_pointer[0] = res; stack_pointer += 1; @@ -3467,11 +3672,21 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3487,17 +3702,23 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3519,17 +3740,23 @@ // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3550,17 +3777,23 @@ STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3578,8 +3811,16 @@ int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -3589,33 +3830,35 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -3632,17 +3875,20 @@ int err = _PyEval_CheckExceptTypeValid(tstate, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(right); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -3658,17 +3904,23 @@ PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3705,7 +3957,13 @@ } else { b = PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = b; } stack_pointer[-1] = b; break; @@ -3750,25 +4008,34 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); // Success! attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) { - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } // Error! attrs = PyStackRef_None; // Failure! } - stack_pointer[-3] = attrs; - stack_pointer += -2; + stack_pointer[0] = attrs; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -3826,14 +4093,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -3870,7 +4141,13 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = iter; } } stack_pointer[-1] = iter; @@ -4276,9 +4553,15 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -4292,9 +4575,15 @@ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -4517,11 +4806,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -4534,19 +4832,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -4980,7 +5287,9 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } init_frame = temp; @@ -5040,11 +5349,20 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5053,19 +5371,28 @@ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5160,11 +5487,20 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5177,19 +5513,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5228,11 +5573,20 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5242,19 +5596,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5350,11 +5713,19 @@ } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = callable[0]; + callable[0] = res; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -5463,19 +5834,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5516,11 +5896,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5532,19 +5921,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5652,11 +6050,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5668,19 +6075,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -5876,12 +6292,23 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); @@ -5901,19 +6328,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6051,17 +6487,22 @@ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[0] = slice; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6132,17 +6573,23 @@ value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); break; } @@ -6173,17 +6620,22 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(rhs); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = res; break; } @@ -6480,7 +6932,12 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr_o); attr = PyStackRef_FromPyObjectSteal(attr_o); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = attr; break; } diff --git a/Python/gc.c b/Python/gc.c index 0fb2f03b040..68879b9efa1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1996,6 +1996,7 @@ Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { GCState *gcstate = &tstate->interp->gc; + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); int expected = 0; if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f02e13f5e3f..094ca1f3c52 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -61,16 +61,23 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(rhs); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = res; } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -167,7 +174,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -279,13 +288,21 @@ STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -324,7 +341,7 @@ // _BINARY_OP_INPLACE_ADD_UNICODE { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); int next_oparg; @@ -355,11 +372,17 @@ assert(Py_REFCNT(left_o) >= 2); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); *target_local = PyStackRef_FromPyObjectSteal(temp); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*target_local)) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, @@ -368,8 +391,6 @@ SKIP_OVER(1); #endif } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -466,7 +487,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -514,15 +537,25 @@ _PyErr_SetKeyError(sub); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(dict_st); - PyStackRef_CLOSE(sub_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (rc <= 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } // not found or error res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -895,7 +928,9 @@ assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) { @@ -1004,9 +1039,14 @@ values = &stack_pointer[-oparg*2]; STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1018,17 +1058,22 @@ oparg); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg*2; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectSteal(map_o); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; + stack_pointer[0] = map; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1048,9 +1093,14 @@ PyObject *set_o = PySet_New(NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1063,20 +1113,25 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(values[_i]); + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (err != 0) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectSteal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + stack_pointer[0] = set; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1096,17 +1151,22 @@ PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectSteal(slice_o); - stack_pointer[-oparg] = slice; - stack_pointer += 1 - oparg; + stack_pointer[0] = slice; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1124,26 +1184,36 @@ pieces = &stack_pointer[-oparg]; STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(pieces[_i]); + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; + stack_pointer[0] = str; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1279,11 +1349,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1319,14 +1398,23 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1336,8 +1424,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -1345,12 +1433,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1454,7 +1542,9 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } init_frame = temp; @@ -1787,11 +1877,20 @@ STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1800,14 +1899,23 @@ PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1817,8 +1925,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -1826,12 +1934,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1881,11 +1989,20 @@ /* res = func(self, args, nargs) */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1898,14 +2015,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -1915,8 +2041,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -1924,12 +2050,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1983,11 +2109,20 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -1997,14 +2132,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -2014,8 +2158,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -2023,12 +2167,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2336,12 +2480,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2363,15 +2513,23 @@ PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value2_st); - PyStackRef_CLOSE(value1_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2426,11 +2584,19 @@ } res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = callable[0]; + callable[0] = res; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2548,12 +2714,24 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -2589,21 +2767,32 @@ } } } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2793,12 +2982,23 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -2818,14 +3018,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -2835,8 +3044,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -2844,12 +3053,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3144,11 +3353,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -3160,14 +3378,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3177,8 +3404,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3186,12 +3413,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3248,11 +3475,20 @@ int nargs = total_args - 1; STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -3264,14 +3500,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3281,8 +3526,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3290,12 +3535,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3474,14 +3719,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3491,8 +3745,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3500,12 +3754,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3560,11 +3814,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -3577,14 +3840,23 @@ stack_pointer = _PyFrame_GetStackPointer(frame); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -3594,8 +3866,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -3603,12 +3875,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4030,38 +4302,54 @@ int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); - JUMP_TO_LABEL(pop_2_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(exc_value_st); - PyStackRef_CLOSE(match_type_st); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } if (!Py_IsNone(match_o)) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_SetHandledException(match_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[-2] = rest; - stack_pointer[-1] = match; + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4085,15 +4373,20 @@ int err = _PyEval_CheckExceptTypeValid(tstate, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - PyStackRef_CLOSE(right); - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = b; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4107,14 +4400,14 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - _PyStackRef sub_iter_st; - _PyStackRef last_sent_val_st; + _PyStackRef sub_iter; + _PyStackRef last_sent_val; _PyStackRef exc_value_st; _PyStackRef none; _PyStackRef value; exc_value_st = stack_pointer[-1]; - last_sent_val_st = stack_pointer[-2]; - sub_iter_st = stack_pointer[-3]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #ifndef Py_TAIL_CALL_INTERP assert(throwflag); @@ -4124,23 +4417,34 @@ int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - none = PyStackRef_None; value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - PyStackRef_CLOSE(sub_iter_st); - PyStackRef_CLOSE(last_sent_val_st); - PyStackRef_CLOSE(exc_value_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + none = PyStackRef_None; } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } - stack_pointer[-3] = none; - stack_pointer[-2] = value; - stack_pointer += -1; + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4184,15 +4488,21 @@ assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int res_bool = PyObject_IsTrue(res_o); Py_DECREF(res_o); @@ -4204,8 +4514,6 @@ } else { res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); } } stack_pointer[0] = res; @@ -4421,16 +4729,24 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4462,15 +4778,23 @@ STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4503,15 +4827,23 @@ // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4603,12 +4935,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (err) { - JUMP_TO_LABEL(pop_1_error); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -4737,14 +5071,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) { - JUMP_TO_LABEL(pop_2_error); - } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -4772,12 +5112,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4809,12 +5155,18 @@ Py_TYPE(update_o)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } - PyStackRef_CLOSE(update); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -4838,20 +5190,26 @@ int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - PyStackRef_CLOSE(awaitable_st); - PyStackRef_CLOSE(exc_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } else { Py_INCREF(exc); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4892,10 +5250,12 @@ receiver = stack_pointer[-2]; (void)receiver; val = value; - PyStackRef_CLOSE(receiver); stack_pointer[-2] = val; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5032,15 +5392,23 @@ value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - PyStackRef_CLOSE(fmt_spec); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5376,20 +5744,26 @@ "__aiter__ method, got %.100s", type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " @@ -5400,7 +5774,9 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5442,12 +5818,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5466,12 +5848,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[-1] = iter; + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5544,7 +5932,13 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - PyStackRef_CLOSE(iterable); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = iter; } } stack_pointer[-1] = iter; @@ -5594,15 +5988,23 @@ PyObject *res_o = _PyEval_ImportName(tstate, frame, name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(level); - PyStackRef_CLOSE(fromlist); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5714,11 +6116,20 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -5754,14 +6165,23 @@ } } assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -5771,8 +6191,8 @@ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); @@ -5780,12 +6200,12 @@ if (err != 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1 + oparg; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6102,12 +6522,24 @@ /* Callable is not a normal Python function */ STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); @@ -6143,21 +6575,32 @@ } } } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); } - PyStackRef_CLOSE(kwnames); + tmp = self_or_null[0]; + self_or_null[0] = PyStackRef_NULL; + PyStackRef_XCLOSE(tmp); + tmp = callable[0]; + callable[0] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6185,9 +6628,11 @@ JUMP_TO_LABEL(error); } } - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -6414,10 +6859,23 @@ frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - JUMP_TO_LABEL(pop_3_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } } // we make no attempt to optimize here; specializations should @@ -6448,15 +6906,26 @@ } } } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) { - JUMP_TO_LABEL(pop_3_error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); @@ -6845,11 +7314,21 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[0] = b; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7095,13 +7574,19 @@ Py_TYPE(iterable)->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } assert(Py_IsNone(none_val)); - PyStackRef_CLOSE(iterable_st); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -7164,11 +7649,17 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } } else { @@ -7176,10 +7667,16 @@ _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -7228,13 +7725,18 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -7290,13 +7792,18 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -7722,10 +8229,16 @@ assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); } - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7784,10 +8297,16 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); } - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7935,14 +8454,19 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -8039,14 +8563,19 @@ STAT_INC(LOAD_ATTR, hit); attr = PyStackRef_FromPyObjectNew(attr_o); UNLOCK_OBJECT(dict); - PyStackRef_CLOSE(owner); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = attr; } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = PyStackRef_NULL; } - stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); @@ -8349,16 +8878,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), @@ -8379,8 +8910,6 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8405,11 +8934,11 @@ } } } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -8775,10 +9304,23 @@ frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - JUMP_TO_LABEL(pop_3_error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); } } // we make no attempt to optimize here; specializations should @@ -8809,15 +9351,26 @@ } } } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) { - JUMP_TO_LABEL(pop_3_error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr_o = PyObject_GetAttr(super, name); Py_DECREF(super); @@ -8875,16 +9428,27 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attr == NULL) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[-3] = attr_st; - stack_pointer += -2; + stack_pointer[0] = attr_st; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -8946,13 +9510,26 @@ stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -9068,23 +9645,34 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(subject); - PyStackRef_CLOSE(type); - PyStackRef_CLOSE(names); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); // Success! attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } // Error! attrs = PyStackRef_None; // Failure! } - stack_pointer[-3] = attrs; - stack_pointer += -2; + stack_pointer[0] = attrs; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -9208,9 +9796,11 @@ INSTRUCTION_STATS(POP_ITER); _PyStackRef value; value = stack_pointer[-1]; - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -9258,7 +9848,13 @@ } else { b = PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = b; } } // _POP_JUMP_IF_TRUE @@ -9296,7 +9892,13 @@ } else { b = PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer[-1] = b; } } // _POP_JUMP_IF_FALSE @@ -9344,9 +9946,11 @@ INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; - PyStackRef_CLOSE(value); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -9419,8 +10023,6 @@ assert(oparg == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -9464,8 +10066,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } @@ -9750,8 +10350,12 @@ JUMPBY(oparg); } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -9900,12 +10504,14 @@ int err = PySet_Add(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - JUMP_TO_LABEL(pop_1_error); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -9953,12 +10559,14 @@ int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(iterable); - if (err < 0) { - JUMP_TO_LABEL(pop_1_error); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -10001,15 +10609,21 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(pop_2_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10347,12 +10961,14 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - if (err) { - JUMP_TO_LABEL(pop_1_error); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -10374,8 +10990,12 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); - JUMP_TO_LABEL(pop_1_error); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10387,12 +11007,14 @@ err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); } - PyStackRef_CLOSE(v); - if (err) { - JUMP_TO_LABEL(pop_1_error); - } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } DISPATCH(); } @@ -10439,14 +11061,24 @@ stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); } - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-1] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(pop_4_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10488,16 +11120,25 @@ /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(pop_3_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10663,13 +11304,19 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = err ? PyStackRef_True : PyStackRef_False; } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10703,10 +11350,16 @@ // _REPLACE_WITH_TRUE { value = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; } - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10762,8 +11415,14 @@ res = PyStackRef_False; } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; DISPATCH(); @@ -10793,7 +11452,12 @@ } STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - PyStackRef_CLOSE(value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-1] = res; DISPATCH(); } @@ -10855,8 +11519,14 @@ } else { assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } stack_pointer[-1] = res; DISPATCH(); @@ -10876,12 +11546,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10899,12 +11575,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10935,18 +11617,20 @@ next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; - _PyStackRef *right; + _PyStackRef *top; seq = stack_pointer[-1]; - right = &stack_pointer[(oparg & 0xFF)]; - _PyStackRef *top = right + (oparg >> 8); + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg & 0xFF, oparg >> 8, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } - stack_pointer += (oparg & 0xFF) + (oparg >> 8); + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -10963,7 +11647,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; - _PyStackRef *output; + _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE { seq = stack_pointer[-1]; @@ -10985,17 +11669,19 @@ } // _UNPACK_SEQUENCE { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); if (res == 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } } - stack_pointer += -1 + oparg; + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11041,8 +11727,12 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11079,8 +11769,12 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); - stack_pointer += -1 + oparg; + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -11116,11 +11810,13 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - PyStackRef_CLOSE(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -11310,8 +12006,6 @@ JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_MonitorRaise(tstate, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); JUMP_TO_LABEL(exception_unwind); } @@ -11403,8 +12097,12 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(exit_unwind); } next_instr = frame->instr_ptr; - LLTRACE_RESUME_FRAME(); #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; /* _PyEval_EvalFrameDefault() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index fd8486785ed..51d0fa63e64 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -273,14 +273,16 @@ { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -306,14 +308,16 @@ { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: @@ -339,14 +343,16 @@ { assert(PyLong_CheckExact(sym_get_const(left))); assert(PyLong_CheckExact(sym_get_const(right))); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), (PyLongObject *)sym_get_const(right)); if (temp == NULL) { goto error; } res = sym_new_const(ctx, temp); - stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); Py_DECREF(temp); // TODO gh-115506: diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c127d0334d9..295bcc87b2c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -602,7 +602,6 @@ NON_ESCAPING_FUNCTIONS = ( "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", "PyType_HasFeature", - "PyUnicode_Append", "PyUnicode_Concat", "PyUnicode_GET_LENGTH", "PyUnicode_READ_CHAR", @@ -619,7 +618,6 @@ NON_ESCAPING_FUNCTIONS = ( "_PyCode_CODE", "_PyDictValues_AddToInsertionOrder", "_PyErr_Occurred", - "_PyEval_FrameClearAndPop", "_PyFloat_FromDouble_ConsumeInputs", "_PyFrame_GetBytecode", "_PyFrame_GetCode", @@ -633,15 +631,12 @@ NON_ESCAPING_FUNCTIONS = ( "_PyList_AppendTakeRef", "_PyList_FromStackRefStealOnSuccess", "_PyList_ITEMS", - "_PyLong_Add", "_PyLong_CompactValue", "_PyLong_DigitCount", "_PyLong_IsCompact", "_PyLong_IsNegative", "_PyLong_IsNonNegativeCompact", "_PyLong_IsZero", - "_PyLong_Multiply", - "_PyLong_Subtract", "_PyManagedDictPointer_IsValues", "_PyObject_GC_IS_TRACKED", "_PyObject_GC_MAY_BE_TRACKED", diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 8cba9127fa3..07a785e4312 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -15,6 +15,8 @@ class CWriter: self.line_directives = line_directives self.last_token = None self.newline = True + self.pending_spill = False + self.pending_reload = False def set_position(self, tkn: Token) -> None: if self.last_token is not None: @@ -33,6 +35,7 @@ class CWriter: self.newline = False def emit_at(self, txt: str, where: Token) -> None: + self.maybe_write_spill() self.set_position(where) self.out.write(txt) @@ -109,6 +112,7 @@ class CWriter: self.last_token = None def emit(self, txt: str | Token) -> None: + self.maybe_write_spill() if isinstance(txt, Token): self.emit_token(txt) elif isinstance(txt, str): @@ -122,6 +126,28 @@ class CWriter: self.newline = True self.last_token = None + def emit_spill(self) -> None: + if self.pending_reload: + self.pending_reload = False + return + assert not self.pending_spill + self.pending_spill = True + + def maybe_write_spill(self) -> None: + if self.pending_spill: + self.pending_spill = False + self.emit_str("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + elif self.pending_reload: + self.pending_reload = False + self.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + + def emit_reload(self) -> None: + if self.pending_spill: + self.pending_spill = False + return + assert not self.pending_reload + self.pending_reload = True + @contextlib.contextmanager def header_guard(self, name: str) -> Iterator[None]: self.out.write( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index cd52f181b60..6b2ef51b29f 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -243,29 +243,13 @@ class Emitter: next(tkn_iter) next(tkn_iter) next(tkn_iter) - self.out.emit_at("", tkn) - for var in storage.inputs: - if not var.defined: - continue - if var.name == "null": - continue - close = "PyStackRef_CLOSE" - if "null" in var.name or var.condition and var.condition != "1": - close = "PyStackRef_XCLOSE" - if var.size: - if var.size == "1": - self.out.emit(f"{close}({var.name}[0]);\n") - else: - self.out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - self.out.emit(f"{close}({var.name}[_i]);\n") - self.out.emit("}\n") - elif var.condition: - if var.condition != "0": - self.out.emit(f"{close}({var.name});\n") - else: - self.out.emit(f"{close}({var.name});\n") - for input in storage.inputs: - input.defined = False + try: + storage.close_inputs(self.out) + except StackError as ex: + raise analysis_error(ex.args[0], tkn) + except Exception as ex: + ex.args = (ex.args[0] + str(tkn),) + raise return True def kill_inputs( @@ -301,7 +285,9 @@ class Emitter: var.defined = False break else: - raise analysis_error(f"'{name}' is not a live input-only variable", name_tkn) + raise analysis_error( + f"'{name}' is not a live input-only variable", name_tkn + ) return True def stackref_kill( @@ -344,7 +330,7 @@ class Emitter: self.out.emit(comma) dealloc = next(tkn_iter) if dealloc.kind != "IDENTIFIER": - raise analysis_error("Expected identifier", dealloc) + raise analysis_error("Expected identifier", dealloc) self.out.emit(dealloc) if name.kind == "IDENTIFIER": escapes = dealloc.text not in NON_ESCAPING_DEALLOCS @@ -518,7 +504,7 @@ class Emitter: self.emit(next(tkn_iter)) maybe_if = tkn_iter.peek() if maybe_if and maybe_if.kind == "IF": - #Emit extra braces around the if to get scoping right + # Emit extra braces around the if to get scoping right self.emit(" {\n") self.emit(next(tkn_iter)) else_reachable, rbrace, else_storage = self._emit_if(tkn_iter, uop, storage, inst) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 6c33debd58e..b679ec84707 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -154,11 +154,11 @@ def write_uop( var.defined = False storage = emitter.emit_tokens(override, storage, None) out.start_line() - storage.flush(out, cast_type="JitOptSymbol *") + storage.flush(out) else: emit_default(out, uop, stack) out.start_line() - stack.flush(out, cast_type="JitOptSymbol *") + stack.flush(out) except StackError as ex: raise analysis_error(ex.args[0], prototype.body[0]) # from None @@ -201,7 +201,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack(False) + stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 729973f1e32..7866857023f 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -224,12 +224,13 @@ def array_or_scalar(var: StackItem | Local) -> str: return "array" if var.is_array() else "scalar" class Stack: - def __init__(self, extract_bits: bool=True) -> None: + def __init__(self, extract_bits: bool=True, cast_type: str = "uintptr_t") -> None: self.top_offset = StackOffset.empty() self.base_offset = StackOffset.empty() self.variables: list[Local] = [] self.defined: set[str] = set() self.extract_bits = extract_bits + self.cast_type = cast_type def pop(self, var: StackItem) -> tuple[str, Local]: self.top_offset.pop(var) @@ -298,8 +299,8 @@ class Stack: out: CWriter, var: StackItem, base_offset: StackOffset, - cast_type: str = "uintptr_t", - extract_bits: bool = True, + cast_type: str, + extract_bits: bool, ) -> None: cast = f"({cast_type})" if var.type else "" bits = ".bits" if cast and extract_bits else "" @@ -315,9 +316,7 @@ class Stack: out.emit(f"stack_pointer += {number};\n") out.emit("assert(WITHIN_STACK_BOUNDS());\n") - def flush( - self, out: CWriter, cast_type: str = "uintptr_t" - ) -> None: + def flush(self, out: CWriter) -> None: out.start_line() var_offset = self.base_offset.copy() for var in self.variables: @@ -325,7 +324,7 @@ class Stack: var.defined and not var.in_memory ): - Stack._do_emit(out, var.item, var_offset, cast_type, self.extract_bits) + Stack._do_emit(out, var.item, var_offset, self.cast_type, self.extract_bits) var.in_memory = True var_offset.push(var.item) number = self.top_offset.to_c() @@ -347,7 +346,7 @@ class Stack: ) def copy(self) -> "Stack": - other = Stack(self.extract_bits) + other = Stack(self.extract_bits, self.cast_type) other.top_offset = self.top_offset.copy() other.base_offset = self.base_offset.copy() other.variables = [var.copy() for var in self.variables] @@ -508,17 +507,26 @@ class Storage: return True return False - def flush(self, out: CWriter, cast_type: str = "uintptr_t") -> None: + def flush(self, out: CWriter) -> None: self.clear_dead_inputs() self._push_defined_outputs() - self.stack.flush(out, cast_type) + self.stack.flush(out) def save(self, out: CWriter) -> None: assert self.spilled >= 0 if self.spilled == 0: self.flush(out) out.start_line() - out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + out.emit_spill() + self.spilled += 1 + + def save_inputs(self, out: CWriter) -> None: + assert self.spilled >= 0 + if self.spilled == 0: + self.clear_dead_inputs() + self.stack.flush(out) + out.start_line() + out.emit_spill() self.spilled += 1 def reload(self, out: CWriter) -> None: @@ -528,7 +536,7 @@ class Storage: self.spilled -= 1 if self.spilled == 0: out.start_line() - out.emit("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + out.emit_reload() @staticmethod def for_uop(stack: Stack, uop: Uop) -> tuple[list[str], "Storage"]: @@ -637,3 +645,91 @@ class Storage: outputs = ", ".join([var.compact_str() for var in self.outputs]) peeks = ", ".join([var.name for var in self.peeks]) return f"{stack_comment[:-2]}{next_line}inputs: {inputs}{next_line}outputs: {outputs}{next_line}peeks: {peeks} */" + + def close_inputs(self, out: CWriter) -> None: + tmp_defined = False + def close_named(close: str, name: str, overwrite: str) -> None: + nonlocal tmp_defined + if overwrite: + if not tmp_defined: + out.emit("_PyStackRef ") + tmp_defined = True + out.emit(f"tmp = {name};\n") + out.emit(f"{name} = {overwrite};\n") + if not var.is_array(): + var.in_memory = False + self.flush(out) + out.emit(f"{close}(tmp);\n") + else: + out.emit(f"{close}({name});\n") + + def close_variable(var: Local, overwrite: str) -> None: + nonlocal tmp_defined + close = "PyStackRef_CLOSE" + if "null" in var.name or var.condition and var.condition != "1": + close = "PyStackRef_XCLOSE" + if var.size: + if var.size == "1": + close_named(close, f"{var.name}[0]", overwrite) + else: + if overwrite and not tmp_defined: + out.emit("_PyStackRef tmp;\n") + tmp_defined = True + out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") + close_named(close, f"{var.name}[_i]", overwrite) + out.emit("}\n") + else: + if var.condition and var.condition == "0": + return + close_named(close, var.name, overwrite) + + self.clear_dead_inputs() + if not self.inputs: + return + output: Local | None = None + for var in self.outputs: + if var.is_array(): + if len(self.inputs) > 1: + raise StackError("Cannot call DECREF_INPUTS with multiple live input(s) and array output") + elif var.defined: + if output is not None: + raise StackError("Cannot call DECREF_INPUTS with more than one live output") + output = var + self.save_inputs(out) + if output is not None: + lowest = self.inputs[0] + if lowest.is_array(): + try: + size = int(lowest.size) + except: + size = -1 + if size <= 0: + raise StackError("Cannot call DECREF_INPUTS with non fixed size array as lowest input on stack") + if size > 1: + raise StackError("Cannot call DECREF_INPUTS with array size > 1 as lowest input on stack") + output.defined = False + close_variable(lowest, output.name) + else: + lowest.in_memory = False + output.defined = False + close_variable(lowest, output.name) + to_close = self.inputs[: 0 if output is not None else None: -1] + if len(to_close) == 1 and not to_close[0].is_array(): + self.reload(out) + to_close[0].defined = False + self.flush(out) + self.save_inputs(out) + close_variable(to_close[0], "") + self.reload(out) + else: + for var in to_close: + assert var.defined or var.is_array() + close_variable(var, "PyStackRef_NULL") + self.reload(out) + for var in self.inputs: + var.defined = False + if output is not None: + output.defined = True + # MyPy false positive + lowest.defined = False # type: ignore[possibly-undefined] + self.flush(out)