mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-12 13:55:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 14
cpython/Python/executor_cases.c.h
Jelle Zijlstra a0559849ac
[3.13] gh-119821: Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS (#119822) (#119889) 
dSupport non-dict globals in LOAD_FROM_DICT_OR_GLOBALS

The implementation basically copies LOAD_GLOBAL. Possibly it could be deduplicated,
but that seems like it may get hairy since the two operations have different operands.

This is important to fix in 3.14 for PEP 649, but it's a bug in earlier versions too,
and we should backport to 3.13 and 3.12 if possible.

(cherry picked from commit 80a4e38994)
2024-05-31 21:56:26 -07:00

4464 lines
154 KiB
C
Generated
Raw Blame History

// This file is generated by Tools/cases_generator/tier2_generator.py
// from:
// Python/bytecodes.c
// Do not edit!
#ifdef TIER_ONE
#error "This file is for Tier 2 only"
#endif
#define TIER_TWO 2
case _NOP: {
break;
}
case _RESUME_CHECK: {
#if defined(__EMSCRIPTEN__)
if (_Py_emscripten_signal_clock == 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
if (eval_breaker != version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
/* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_FAST_CHECK: {
PyObject *value;
oparg = CURRENT_OPARG();
value = GETLOCAL(oparg);
if (value == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) JUMP_TO_ERROR();
}
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_0: {
PyObject *value;
oparg = 0;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_1: {
PyObject *value;
oparg = 1;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_2: {
PyObject *value;
oparg = 2;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_3: {
PyObject *value;
oparg = 3;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_4: {
PyObject *value;
oparg = 4;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_5: {
PyObject *value;
oparg = 5;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_6: {
PyObject *value;
oparg = 6;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_7: {
PyObject *value;
oparg = 7;
assert(oparg == CURRENT_OPARG());
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST: {
PyObject *value;
oparg = CURRENT_OPARG();
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_FAST_AND_CLEAR: {
PyObject *value;
oparg = CURRENT_OPARG();
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
GETLOCAL(oparg) = NULL;
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_CONST: {
PyObject *value;
oparg = CURRENT_OPARG();
value = GETITEM(FRAME_CO_CONSTS, oparg);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _STORE_FAST_0: {
PyObject *value;
oparg = 0;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_1: {
PyObject *value;
oparg = 1;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_2: {
PyObject *value;
oparg = 2;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_3: {
PyObject *value;
oparg = 3;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_4: {
PyObject *value;
oparg = 4;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_5: {
PyObject *value;
oparg = 5;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_6: {
PyObject *value;
oparg = 6;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST_7: {
PyObject *value;
oparg = 7;
assert(oparg == CURRENT_OPARG());
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _STORE_FAST: {
PyObject *value;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
break;
}
case _POP_TOP: {
PyObject *value;
value = stack_pointer[-1];
Py_DECREF(value);
stack_pointer += -1;
break;
}
case _PUSH_NULL: {
PyObject *res;
res = NULL;
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
case _END_SEND: {
PyObject *value;
PyObject *receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
Py_DECREF(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
break;
}
case _UNARY_NEGATIVE: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
res = PyNumber_Negative(value);
Py_DECREF(value);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = res;
break;
}
case _UNARY_NOT: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
assert(PyBool_Check(value));
res = Py_IsFalse(value) ? Py_True : Py_False;
stack_pointer[-1] = res;
break;
}
case _TO_BOOL: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
int err = PyObject_IsTrue(value);
Py_DECREF(value);
if (err < 0) JUMP_TO_ERROR();
res = err ? Py_True : Py_False;
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_BOOL: {
PyObject *value;
value = stack_pointer[-1];
if (!PyBool_Check(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
break;
}
case _TO_BOOL_INT: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
if (!PyLong_CheckExact(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value)) {
assert(_Py_IsImmortal(value));
res = Py_False;
}
else {
Py_DECREF(value);
res = Py_True;
}
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_LIST: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
if (!PyList_CheckExact(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
Py_DECREF(value);
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_NONE: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
if (!Py_IsNone(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
res = Py_False;
stack_pointer[-1] = res;
break;
}
case _TO_BOOL_STR: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
if (!PyUnicode_CheckExact(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(TO_BOOL, hit);
if (value == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value));
res = Py_False;
}
else {
assert(Py_SIZE(value));
Py_DECREF(value);
res = Py_True;
}
stack_pointer[-1] = res;
break;
}
case _REPLACE_WITH_TRUE: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
Py_DECREF(value);
res = Py_True;
stack_pointer[-1] = res;
break;
}
case _UNARY_INVERT: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
res = PyNumber_Invert(value);
Py_DECREF(value);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = res;
break;
}
case _GUARD_BOTH_INT: {
PyObject *right;
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (!PyLong_CheckExact(left)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyLong_CheckExact(right)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_NOS_INT: {
PyObject *left;
left = stack_pointer[-2];
if (!PyLong_CheckExact(left)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TOS_INT: {
PyObject *value;
value = stack_pointer[-1];
if (!PyLong_CheckExact(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _BINARY_OP_MULTIPLY_INT: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_OP_ADD_INT: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_OP_SUBTRACT_INT: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _GUARD_BOTH_FLOAT: {
PyObject *right;
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (!PyFloat_CheckExact(left)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyFloat_CheckExact(right)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_NOS_FLOAT: {
PyObject *left;
left = stack_pointer[-2];
if (!PyFloat_CheckExact(left)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TOS_FLOAT: {
PyObject *value;
value = stack_pointer[-1];
if (!PyFloat_CheckExact(value)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _BINARY_OP_MULTIPLY_FLOAT: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left)->ob_fval *
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_OP_ADD_FLOAT: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left)->ob_fval +
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_OP_SUBTRACT_FLOAT: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left)->ob_fval -
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _GUARD_BOTH_UNICODE: {
PyObject *right;
PyObject *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (!PyUnicode_CheckExact(left)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyUnicode_CheckExact(right)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _BINARY_OP_ADD_UNICODE: {
PyObject *right;
PyObject *left;
PyObject *res;
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(BINARY_OP, hit);
res = PyUnicode_Concat(left, right);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_SUBSCR: {
PyObject *sub;
PyObject *container;
PyObject *res;
sub = stack_pointer[-1];
container = stack_pointer[-2];
res = PyObject_GetItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_SLICE: {
PyObject *stop;
PyObject *start;
PyObject *container;
PyObject *res;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
// Can't use ERROR_IF() here, because we haven't
// DECREF'ed container yet, and we still own slice.
if (slice == NULL) {
res = NULL;
}
else {
res = PyObject_GetItem(container, slice);
Py_DECREF(slice);
}
Py_DECREF(container);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
break;
}
case _STORE_SLICE: {
PyObject *stop;
PyObject *start;
PyObject *container;
PyObject *v;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
v = stack_pointer[-4];
PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
int err;
if (slice == NULL) {
err = 1;
}
else {
err = PyObject_SetItem(container, slice, v);
Py_DECREF(slice);
}
Py_DECREF(v);
Py_DECREF(container);
if (err) JUMP_TO_ERROR();
stack_pointer += -4;
break;
}
case _BINARY_SUBSCR_LIST_INT: {
PyObject *sub;
PyObject *list;
PyObject *res;
sub = stack_pointer[-1];
list = stack_pointer[-2];
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyList_CheckExact(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// Deopt unless 0 <= sub < PyList_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (index >= PyList_GET_SIZE(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
res = PyList_GET_ITEM(list, index);
assert(res != NULL);
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_SUBSCR_STR_INT: {
PyObject *sub;
PyObject *str;
PyObject *res;
sub = stack_pointer[-1];
str = stack_pointer[-2];
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyUnicode_CheckExact(str)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (PyUnicode_GET_LENGTH(str) <= index) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(str);
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_SUBSCR_TUPLE_INT: {
PyObject *sub;
PyObject *tuple;
PyObject *res;
sub = stack_pointer[-1];
tuple = stack_pointer[-2];
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyTuple_CheckExact(tuple)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// Deopt unless 0 <= sub < PyTuple_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (index >= PyTuple_GET_SIZE(tuple)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
res = PyTuple_GET_ITEM(tuple, index);
assert(res != NULL);
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(tuple);
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _BINARY_SUBSCR_DICT: {
PyObject *sub;
PyObject *dict;
PyObject *res;
sub = stack_pointer[-1];
dict = stack_pointer[-2];
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(BINARY_SUBSCR, hit);
int rc = PyDict_GetItemRef(dict, sub, &res);
if (rc == 0) {
_PyErr_SetKeyError(sub);
}
Py_DECREF(dict);
Py_DECREF(sub);
if (rc <= 0) JUMP_TO_ERROR();
// not found or error
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
/* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _LIST_APPEND: {
PyObject *v;
PyObject *list;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
case _SET_ADD: {
PyObject *v;
PyObject *set;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
int err = PySet_Add(set, v);
Py_DECREF(v);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
case _STORE_SUBSCR: {
PyObject *sub;
PyObject *container;
PyObject *v;
sub = stack_pointer[-1];
container = stack_pointer[-2];
v = stack_pointer[-3];
/* container[sub] = v */
int err = PyObject_SetItem(container, sub, v);
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
if (err) JUMP_TO_ERROR();
stack_pointer += -3;
break;
}
case _STORE_SUBSCR_LIST_INT: {
PyObject *sub;
PyObject *list;
PyObject *value;
sub = stack_pointer[-1];
list = stack_pointer[-2];
value = stack_pointer[-3];
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyList_CheckExact(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// Ensure nonnegative, zero-or-one-digit ints.
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
// Ensure index < len(list)
if (index >= PyList_GET_SIZE(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, value);
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
stack_pointer += -3;
break;
}
case _STORE_SUBSCR_DICT: {
PyObject *sub;
PyObject *dict;
PyObject *value;
sub = stack_pointer[-1];
dict = stack_pointer[-2];
value = stack_pointer[-3];
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
if (err) JUMP_TO_ERROR();
stack_pointer += -3;
break;
}
case _DELETE_SUBSCR: {
PyObject *sub;
PyObject *container;
sub = stack_pointer[-1];
container = stack_pointer[-2];
/* del container[sub] */
int err = PyObject_DelItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
if (err) JUMP_TO_ERROR();
stack_pointer += -2;
break;
}
case _CALL_INTRINSIC_1: {
PyObject *value;
PyObject *res;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
assert(oparg <= MAX_INTRINSIC_1);
res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
Py_DECREF(value);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = res;
break;
}
case _CALL_INTRINSIC_2: {
PyObject *value1;
PyObject *value2;
PyObject *res;
oparg = CURRENT_OPARG();
value1 = stack_pointer[-1];
value2 = stack_pointer[-2];
assert(oparg <= MAX_INTRINSIC_2);
res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
Py_DECREF(value2);
Py_DECREF(value1);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _POP_FRAME: {
PyObject *retval;
retval = stack_pointer[-1];
#if TIER_ONE
assert(frame != &entry_frame);
#endif
stack_pointer += -1;
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, retval);
LOAD_SP();
LOAD_IP(frame->return_offset);
LLTRACE_RESUME_FRAME();
break;
}
/* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */
case _GET_AITER: {
PyObject *obj;
PyObject *iter;
obj = stack_pointer[-1];
unaryfunc getter = NULL;
PyTypeObject *type = Py_TYPE(obj);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
if (getter == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
if (true) JUMP_TO_ERROR();
}
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) JUMP_TO_ERROR();
if (Py_TYPE(iter)->tp_as_async == NULL ||
Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
if (true) JUMP_TO_ERROR();
}
stack_pointer[-1] = iter;
break;
}
case _GET_ANEXT: {
PyObject *aiter;
PyObject *awaitable;
aiter = stack_pointer[-1];
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
PyTypeObject *type = Py_TYPE(aiter);
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
JUMP_TO_ERROR();
}
} else {
if (type->tp_as_async != NULL){
getter = type->tp_as_async->am_anext;
}
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
JUMP_TO_ERROR();
}
}
else {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
JUMP_TO_ERROR();
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
_PyErr_FormatFromCause(
PyExc_TypeError,
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
JUMP_TO_ERROR();
} else {
Py_DECREF(next_iter);
}
}
stack_pointer[0] = awaitable;
stack_pointer += 1;
break;
}
case _GET_AWAITABLE: {
PyObject *iterable;
PyObject *iter;
oparg = CURRENT_OPARG();
iterable = stack_pointer[-1];
iter = _PyCoro_GetAwaitableIter(iterable);
if (iter == NULL) {
_PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
}
Py_DECREF(iterable);
if (iter != NULL && PyCoro_CheckExact(iter)) {
PyObject *yf = _PyGen_yf((PyGenObject*)iter);
if (yf != NULL) {
/* `iter` is a coroutine object that is being
awaited, `yf` is a pointer to the current awaitable
being awaited on. */
Py_DECREF(yf);
Py_CLEAR(iter);
_PyErr_SetString(tstate, PyExc_RuntimeError,
"coroutine is being awaited already");
/* The code below jumps to `error` if `iter` is NULL. */
}
}
if (iter == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = iter;
break;
}
/* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */
case _YIELD_VALUE: {
PyObject *retval;
PyObject *value;
oparg = CURRENT_OPARG();
retval = stack_pointer[-1];
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
// or throw() call.
#if TIER_ONE
assert(frame != &entry_frame);
#endif
frame->instr_ptr++;
PyGenObject *gen = _PyFrame_GetGenerator(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
gen->gi_frame_state = FRAME_SUSPENDED + oparg;
stack_pointer += -1;
_PyFrame_SetStackPointer(frame, stack_pointer);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
/* We don't know which of these is relevant here, so keep them equal */
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
#if TIER_ONE
assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE ||
frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR);
#endif
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
LOAD_SP();
value = retval;
LLTRACE_RESUME_FRAME();
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _POP_EXCEPT: {
PyObject *exc_value;
exc_value = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value);
stack_pointer += -1;
break;
}
case _LOAD_ASSERTION_ERROR: {
PyObject *value;
value = Py_NewRef(PyExc_AssertionError);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_BUILD_CLASS: {
PyObject *bc;
if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR();
if (bc == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
if (true) JUMP_TO_ERROR();
}
stack_pointer[0] = bc;
stack_pointer += 1;
break;
}
case _STORE_NAME: {
PyObject *v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
int err;
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
Py_DECREF(v);
if (true) JUMP_TO_ERROR();
}
if (PyDict_CheckExact(ns))
err = PyDict_SetItem(ns, name, v);
else
err = PyObject_SetItem(ns, name, v);
Py_DECREF(v);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
case _DELETE_NAME: {
oparg = CURRENT_OPARG();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
int err;
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
JUMP_TO_ERROR();
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
if (err != 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
JUMP_TO_ERROR();
}
break;
}
case _UNPACK_SEQUENCE: {
PyObject *seq;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
PyObject **top = stack_pointer + oparg - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
Py_DECREF(seq);
if (res == 0) JUMP_TO_ERROR();
stack_pointer += -1 + oparg;
break;
}
case _UNPACK_SEQUENCE_TWO_TUPLE: {
PyObject *seq;
PyObject *val1;
PyObject *val0;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
assert(oparg == 2);
if (!PyTuple_CheckExact(seq)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (PyTuple_GET_SIZE(seq) != 2) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
Py_DECREF(seq);
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
break;
}
case _UNPACK_SEQUENCE_TUPLE: {
PyObject *seq;
PyObject **values;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
values = &stack_pointer[-1];
if (!PyTuple_CheckExact(seq)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (PyTuple_GET_SIZE(seq) != oparg) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
*values++ = Py_NewRef(items[i]);
}
Py_DECREF(seq);
stack_pointer += -1 + oparg;
break;
}
case _UNPACK_SEQUENCE_LIST: {
PyObject *seq;
PyObject **values;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
values = &stack_pointer[-1];
if (!PyList_CheckExact(seq)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (PyList_GET_SIZE(seq) != oparg) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
*values++ = Py_NewRef(items[i]);
}
Py_DECREF(seq);
stack_pointer += -1 + oparg;
break;
}
case _UNPACK_EX: {
PyObject *seq;
oparg = CURRENT_OPARG();
seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
PyObject **top = stack_pointer + totalargs - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
if (res == 0) JUMP_TO_ERROR();
stack_pointer += (oparg >> 8) + (oparg & 0xFF);
break;
}
case _STORE_ATTR: {
PyObject *owner;
PyObject *v;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
v = stack_pointer[-2];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_SetAttr(owner, name, v);
Py_DECREF(v);
Py_DECREF(owner);
if (err) JUMP_TO_ERROR();
stack_pointer += -2;
break;
}
case _DELETE_ATTR: {
PyObject *owner;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_DelAttr(owner, name);
Py_DECREF(owner);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
case _STORE_GLOBAL: {
PyObject *v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
if (err) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
case _DELETE_GLOBAL: {
oparg = CURRENT_OPARG();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
JUMP_TO_ERROR();
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
JUMP_TO_ERROR();
}
break;
}
case _LOAD_LOCALS: {
PyObject *locals;
locals = LOCALS();
if (locals == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) JUMP_TO_ERROR();
}
Py_INCREF(locals);
stack_pointer[0] = locals;
stack_pointer += 1;
break;
}
/* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
/* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _LOAD_GLOBAL: {
PyObject *res;
PyObject *null = NULL;
oparg = CURRENT_OPARG();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
if (res == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
if (true) JUMP_TO_ERROR();
}
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR();
if (res == NULL) {
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR();
if (res == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) JUMP_TO_ERROR();
}
}
}
null = NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
break;
}
case _GUARD_GLOBALS_VERSION: {
uint16_t version = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)GLOBALS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (dict->ma_keys->dk_version != version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(DK_IS_UNICODE(dict->ma_keys));
break;
}
case _GUARD_BUILTINS_VERSION: {
uint16_t version = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)BUILTINS();
if (!PyDict_CheckExact(dict)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (dict->ma_keys->dk_version != version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(DK_IS_UNICODE(dict->ma_keys));
break;
}
case _LOAD_GLOBAL_MODULE: {
PyObject *res;
PyObject *null = NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
if (res == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
break;
}
case _LOAD_GLOBAL_BUILTINS: {
PyObject *res;
PyObject *null = NULL;
oparg = CURRENT_OPARG();
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
if (res == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
break;
}
case _DELETE_FAST: {
oparg = CURRENT_OPARG();
PyObject *v = GETLOCAL(oparg);
if (v == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) JUMP_TO_ERROR();
}
SETLOCAL(oparg, NULL);
break;
}
case _MAKE_CELL: {
oparg = CURRENT_OPARG();
// "initial" is probably NULL but not if it's an arg (or set
// via the f_locals proxy before MAKE_CELL has run).
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
JUMP_TO_ERROR();
}
SETLOCAL(oparg, cell);
break;
}
case _DELETE_DEREF: {
oparg = CURRENT_OPARG();
PyObject *cell = GETLOCAL(oparg);
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL);
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
JUMP_TO_ERROR();
}
Py_DECREF(oldobj);
break;
}
case _LOAD_FROM_DICT_OR_DEREF: {
PyObject *class_dict;
PyObject *value;
oparg = CURRENT_OPARG();
class_dict = stack_pointer[-1];
PyObject *name;
assert(class_dict);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
JUMP_TO_ERROR();
}
if (!value) {
PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
value = PyCell_GetRef(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
JUMP_TO_ERROR();
}
}
Py_DECREF(class_dict);
stack_pointer[-1] = value;
break;
}
case _LOAD_DEREF: {
PyObject *value;
oparg = CURRENT_OPARG();
PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
value = PyCell_GetRef(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) JUMP_TO_ERROR();
}
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _STORE_DEREF: {
PyObject *v;
oparg = CURRENT_OPARG();
v = stack_pointer[-1];
PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
PyCell_SetTakeRef(cell, v);
stack_pointer += -1;
break;
}
case _COPY_FREE_VARS: {
oparg = CURRENT_OPARG();
/* Copy closure variables to free variables */
PyCodeObject *co = _PyFrame_GetCode(frame);
assert(PyFunction_Check(frame->f_funcobj));
PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure;
assert(oparg == co->co_nfreevars);
int offset = co->co_nlocalsplus - oparg;
for (int i = 0; i < oparg; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
frame->localsplus[offset + i] = Py_NewRef(o);
}
break;
}
case _BUILD_STRING: {
PyObject **pieces;
PyObject *str;
oparg = CURRENT_OPARG();
pieces = &stack_pointer[-oparg];
str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(pieces[_i]);
}
if (str == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg] = str;
stack_pointer += 1 - oparg;
break;
}
case _BUILD_TUPLE: {
PyObject **values;
PyObject *tup;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg];
tup = _PyTuple_FromArraySteal(values, oparg);
if (tup == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg] = tup;
stack_pointer += 1 - oparg;
break;
}
case _BUILD_LIST: {
PyObject **values;
PyObject *list;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg];
list = _PyList_FromArraySteal(values, oparg);
if (list == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg] = list;
stack_pointer += 1 - oparg;
break;
}
case _LIST_EXTEND: {
PyObject *iterable;
PyObject *list;
oparg = CURRENT_OPARG();
iterable = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
(Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
{
_PyErr_Clear(tstate);
_PyErr_Format(tstate, PyExc_TypeError,
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
Py_DECREF(iterable);
if (true) JUMP_TO_ERROR();
}
assert(Py_IsNone(none_val));
Py_DECREF(iterable);
stack_pointer += -1;
break;
}
case _SET_UPDATE: {
PyObject *iterable;
PyObject *set;
oparg = CURRENT_OPARG();
iterable = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
int err = _PySet_Update(set, iterable);
Py_DECREF(iterable);
if (err < 0) JUMP_TO_ERROR();
stack_pointer += -1;
break;
}
/* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _BUILD_MAP: {
PyObject **values;
PyObject *map;
oparg = CURRENT_OPARG();
values = &stack_pointer[-oparg*2];
map = _PyDict_FromItems(
values, 2,
values+1, 2,
oparg);
for (int _i = oparg*2; --_i >= 0;) {
Py_DECREF(values[_i]);
}
if (map == NULL) JUMP_TO_ERROR();
stack_pointer[-oparg*2] = map;
stack_pointer += 1 - oparg*2;
break;
}
case _SETUP_ANNOTATIONS: {
int err;
PyObject *ann_dict;
if (LOCALS() == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
if (true) JUMP_TO_ERROR();
}
/* check if __annotations__ in locals()... */
if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR();
if (ann_dict == NULL) {
ann_dict = PyDict_New();
if (ann_dict == NULL) JUMP_TO_ERROR();
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
ann_dict);
Py_DECREF(ann_dict);
if (err) JUMP_TO_ERROR();
}
else {
Py_DECREF(ann_dict);
}
break;
}
case _BUILD_CONST_KEY_MAP: {
PyObject *keys;
PyObject **values;
PyObject *map;
oparg = CURRENT_OPARG();
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
assert(PyTuple_CheckExact(keys));
assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(values[_i]);
}
Py_DECREF(keys);
if (map == NULL) JUMP_TO_ERROR();
stack_pointer[-1 - oparg] = map;
stack_pointer += -oparg;
break;
}
case _DICT_UPDATE: {
PyObject *update;
PyObject *dict;
oparg = CURRENT_OPARG();
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
if (PyDict_Update(dict, update) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object is not a mapping",
Py_TYPE(update)->tp_name);
}
Py_DECREF(update);
if (true) JUMP_TO_ERROR();
}
Py_DECREF(update);
stack_pointer += -1;
break;
}
case _DICT_MERGE: {
PyObject *update;
PyObject *dict;
PyObject *callable;
oparg = CURRENT_OPARG();
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
callable = stack_pointer[-5 - (oparg - 1)];
if (_PyDict_MergeEx(dict, update, 2) < 0) {
_PyEval_FormatKwargsError(tstate, callable, update);
Py_DECREF(update);
if (true) JUMP_TO_ERROR();
}
Py_DECREF(update);
stack_pointer += -1;
break;
}
case _MAP_ADD: {
PyObject *value;
PyObject *key;
PyObject *dict;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
key = stack_pointer[-2];
dict = stack_pointer[-3 - (oparg - 1)];
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR();
stack_pointer += -2;
break;
}
/* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */
case _LOAD_SUPER_ATTR_ATTR: {
PyObject *self;
PyObject *class;
PyObject *global_super;
PyObject *attr;
oparg = CURRENT_OPARG();
self = stack_pointer[-1];
class = stack_pointer[-2];
global_super = stack_pointer[-3];
assert(!(oparg & 1));
if (global_super != (PyObject *)&PySuper_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyType_Check(class)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
Py_DECREF(global_super);
Py_DECREF(class);
Py_DECREF(self);
if (attr == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = attr;
stack_pointer += -2;
break;
}
case _LOAD_SUPER_ATTR_METHOD: {
PyObject *self;
PyObject *class;
PyObject *global_super;
PyObject *attr;
PyObject *self_or_null;
oparg = CURRENT_OPARG();
self = stack_pointer[-1];
class = stack_pointer[-2];
global_super = stack_pointer[-3];
assert(oparg & 1);
if (global_super != (PyObject *)&PySuper_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyType_Check(class)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
int method_found = 0;
attr = _PySuper_Lookup(cls, self, name,
Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
Py_DECREF(global_super);
Py_DECREF(class);
if (attr == NULL) {
Py_DECREF(self);
if (true) JUMP_TO_ERROR();
}
if (method_found) {
self_or_null = self; // transfer ownership
} else {
Py_DECREF(self);
self_or_null = NULL;
}
stack_pointer[-3] = attr;
stack_pointer[-2] = self_or_null;
stack_pointer += -1;
break;
}
case _LOAD_ATTR: {
PyObject *owner;
PyObject *attr;
PyObject *self_or_null = NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
attr = NULL;
if (_PyObject_GetMethod(owner, name, &attr)) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
Py_DECREF(owner);
if (attr == NULL) JUMP_TO_ERROR();
self_or_null = NULL;
}
}
else {
/* Classic, pushes one value. */
attr = PyObject_GetAttr(owner, name);
Py_DECREF(owner);
if (attr == NULL) JUMP_TO_ERROR();
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
break;
}
case _GUARD_TYPE_VERSION: {
PyObject *owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
if (tp->tp_version_tag != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _CHECK_MANAGED_OBJECT_HAS_VALUES: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (!_PyObject_InlineValues(owner)->valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _LOAD_ATTR_INSTANCE_VALUE_0: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
attr = _PyObject_InlineValues(owner)->values[index];
if (attr == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_INSTANCE_VALUE_1: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
attr = _PyObject_InlineValues(owner)->values[index];
if (attr == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
break;
}
/* _LOAD_ATTR_INSTANCE_VALUE is split on (oparg & 1) */
case _CHECK_ATTR_MODULE: {
PyObject *owner;
owner = stack_pointer[-1];
uint32_t dict_version = (uint32_t)CURRENT_OPERAND();
if (!PyModule_CheckExact(owner)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
if (dict->ma_keys->dk_version != dict_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _LOAD_ATTR_MODULE: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
attr = ep->me_value;
if (attr == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
break;
}
case _CHECK_ATTR_WITH_HINT: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (dict == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyDict_CheckExact((PyObject *)dict));
break;
}
case _LOAD_ATTR_WITH_HINT: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t hint = (uint16_t)CURRENT_OPERAND();
PyDictObject *dict = _PyObject_GetManagedDict(owner);
if (hint >= (size_t)dict->ma_keys->dk_nentries) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
attr = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
if (ep->me_key != name) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
attr = ep->me_value;
}
if (attr == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
break;
}
case _LOAD_ATTR_SLOT_0: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
if (attr == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_SLOT_1: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
if (attr == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
break;
}
/* _LOAD_ATTR_SLOT is split on (oparg & 1) */
case _CHECK_ATTR_CLASS: {
PyObject *owner;
owner = stack_pointer[-1];
uint32_t type_version = (uint32_t)CURRENT_OPERAND();
if (!PyType_Check(owner)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(type_version != 0);
if (((PyTypeObject *)owner)->tp_version_tag != type_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _LOAD_ATTR_CLASS_0: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = Py_NewRef(descr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_CLASS_1: {
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
(void)null;
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = Py_NewRef(descr);
null = NULL;
Py_DECREF(owner);
stack_pointer[-1] = attr;
stack_pointer[0] = null;
stack_pointer += 1;
break;
}
/* _LOAD_ATTR_CLASS is split on (oparg & 1) */
/* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _GUARD_DORV_NO_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (_PyObject_GetManagedDict(owner)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (_PyObject_InlineValues(owner)->valid == 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _STORE_ATTR_INSTANCE_VALUE: {
PyObject *owner;
PyObject *value;
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
STAT_INC(STORE_ATTR, hit);
assert(_PyObject_GetManagedDict(owner) == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
Py_DECREF(owner);
stack_pointer += -2;
break;
}
/* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 because it has unused cache entries */
case _STORE_ATTR_SLOT: {
PyObject *owner;
PyObject *value;
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
char *addr = (char *)owner + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = value;
Py_XDECREF(old_value);
Py_DECREF(owner);
stack_pointer += -2;
break;
}
case _COMPARE_OP: {
PyObject *right;
PyObject *left;
PyObject *res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
assert((oparg >> 5) <= Py_GE);
res = PyObject_RichCompare(left, right, oparg >> 5);
Py_DECREF(left);
Py_DECREF(right);
if (res == NULL) JUMP_TO_ERROR();
if (oparg & 16) {
int res_bool = PyObject_IsTrue(res);
Py_DECREF(res);
if (res_bool < 0) JUMP_TO_ERROR();
res = res_bool ? Py_True : Py_False;
}
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _COMPARE_OP_FLOAT: {
PyObject *right;
PyObject *left;
PyObject *res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(COMPARE_OP, hit);
double dleft = PyFloat_AS_DOUBLE(left);
double dright = PyFloat_AS_DOUBLE(right);
// 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
int sign_ish = COMPARISON_BIT(dleft, dright);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _COMPARE_OP_INT: {
PyObject *right;
PyObject *left;
PyObject *res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
if (!_PyLong_IsCompact((PyLongObject *)left)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!_PyLong_IsCompact((PyLongObject *)right)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right) <= 1);
Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left);
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _COMPARE_OP_STR: {
PyObject *right;
PyObject *left;
PyObject *res;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
STAT_INC(COMPARE_OP, hit);
int eq = _PyUnicode_Equal(left, right);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(eq == 0 || eq == 1);
assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _IS_OP: {
PyObject *right;
PyObject *left;
PyObject *b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
int res = Py_Is(left, right) ^ oparg;
Py_DECREF(left);
Py_DECREF(right);
b = res ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
break;
}
case _CONTAINS_OP: {
PyObject *right;
PyObject *left;
PyObject *b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
int res = PySequence_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
if (res < 0) JUMP_TO_ERROR();
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
break;
}
case _CONTAINS_OP_SET: {
PyObject *right;
PyObject *left;
PyObject *b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
int res = _PySet_Contains((PySetObject *)right, left);
Py_DECREF(left);
Py_DECREF(right);
if (res < 0) JUMP_TO_ERROR();
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
break;
}
case _CONTAINS_OP_DICT: {
PyObject *right;
PyObject *left;
PyObject *b;
oparg = CURRENT_OPARG();
right = stack_pointer[-1];
left = stack_pointer[-2];
if (!PyDict_CheckExact(right)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CONTAINS_OP, hit);
int res = PyDict_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
if (res < 0) JUMP_TO_ERROR();
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
break;
}
case _CHECK_EG_MATCH: {
PyObject *match_type;
PyObject *exc_value;
PyObject *rest;
PyObject *match;
match_type = stack_pointer[-1];
exc_value = stack_pointer[-2];
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (true) JUMP_TO_ERROR();
}
match = NULL;
rest = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
&match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (res < 0) JUMP_TO_ERROR();
assert((match == NULL) == (rest == NULL));
if (match == NULL) JUMP_TO_ERROR();
if (!Py_IsNone(match)) {
PyErr_SetHandledException(match);
}
stack_pointer[-2] = rest;
stack_pointer[-1] = match;
break;
}
case _CHECK_EXC_MATCH: {
PyObject *right;
PyObject *left;
PyObject *b;
right = stack_pointer[-1];
left = stack_pointer[-2];
assert(PyExceptionInstance_Check(left));
if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
Py_DECREF(right);
if (true) JUMP_TO_ERROR();
}
int res = PyErr_GivenExceptionMatches(left, right);
Py_DECREF(right);
b = res ? Py_True : Py_False;
stack_pointer[-1] = b;
break;
}
/* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */
/* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */
case _IS_NONE: {
PyObject *value;
PyObject *b;
value = stack_pointer[-1];
if (Py_IsNone(value)) {
b = Py_True;
}
else {
b = Py_False;
Py_DECREF(value);
}
stack_pointer[-1] = b;
break;
}
case _GET_LEN: {
PyObject *obj;
PyObject *len_o;
obj = stack_pointer[-1];
// PUSH(len(TOS))
Py_ssize_t len_i = PyObject_Length(obj);
if (len_i < 0) JUMP_TO_ERROR();
len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) JUMP_TO_ERROR();
stack_pointer[0] = len_o;
stack_pointer += 1;
break;
}
case _MATCH_CLASS: {
PyObject *names;
PyObject *type;
PyObject *subject;
PyObject *attrs;
oparg = CURRENT_OPARG();
names = stack_pointer[-1];
type = stack_pointer[-2];
subject = stack_pointer[-3];
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
assert(PyTuple_CheckExact(names));
attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
Py_DECREF(subject);
Py_DECREF(type);
Py_DECREF(names);
if (attrs) {
assert(PyTuple_CheckExact(attrs)); // Success!
}
else {
if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR();
// Error!
attrs = Py_None; // Failure!
}
stack_pointer[-3] = attrs;
stack_pointer += -2;
break;
}
case _MATCH_MAPPING: {
PyObject *subject;
PyObject *res;
subject = stack_pointer[-1];
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
res = match ? Py_True : Py_False;
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
case _MATCH_SEQUENCE: {
PyObject *subject;
PyObject *res;
subject = stack_pointer[-1];
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
res = match ? Py_True : Py_False;
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
case _MATCH_KEYS: {
PyObject *keys;
PyObject *subject;
PyObject *values_or_none;
keys = stack_pointer[-1];
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
if (values_or_none == NULL) JUMP_TO_ERROR();
stack_pointer[0] = values_or_none;
stack_pointer += 1;
break;
}
case _GET_ITER: {
PyObject *iterable;
PyObject *iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
iter = PyObject_GetIter(iterable);
Py_DECREF(iterable);
if (iter == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = iter;
break;
}
case _GET_YIELD_FROM_ITER: {
PyObject *iterable;
PyObject *iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
if (PyCoro_CheckExact(iterable)) {
/* `iterable` is a coroutine */
if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
regular generator. */
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
JUMP_TO_ERROR();
}
iter = iterable;
}
else if (PyGen_CheckExact(iterable)) {
iter = iterable;
}
else {
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
JUMP_TO_ERROR();
}
Py_DECREF(iterable);
}
stack_pointer[-1] = iter;
break;
}
/* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */
case _FOR_ITER_TIER_TWO: {
PyObject *iter;
PyObject *next;
iter = stack_pointer[-1];
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
next = (*Py_TYPE(iter)->tp_iternext)(iter);
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
JUMP_TO_ERROR();
}
_PyErr_Clear(tstate);
}
/* iterator ended normally */
/* The translator sets the deopt target just past the matching END_FOR */
if (true) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
// Common case: no jump, leave it to the code generator
stack_pointer[0] = next;
stack_pointer += 1;
break;
}
/* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */
case _ITER_CHECK_LIST: {
PyObject *iter;
iter = stack_pointer[-1];
if (Py_TYPE(iter) != &PyListIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
/* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_LIST: {
PyObject *iter;
iter = stack_pointer[-1];
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
if (seq == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _ITER_NEXT_LIST: {
PyObject *iter;
PyObject *next;
iter = stack_pointer[-1];
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyList_GET_SIZE(seq));
next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++));
stack_pointer[0] = next;
stack_pointer += 1;
break;
}
case _ITER_CHECK_TUPLE: {
PyObject *iter;
iter = stack_pointer[-1];
if (Py_TYPE(iter) != &PyTupleIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
/* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_TUPLE: {
PyObject *iter;
iter = stack_pointer[-1];
_PyTupleIterObject *it = (_PyTupleIterObject *)iter;
assert(Py_TYPE(iter) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
if (seq == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (it->it_index >= PyTuple_GET_SIZE(seq)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _ITER_NEXT_TUPLE: {
PyObject *iter;
PyObject *next;
iter = stack_pointer[-1];
_PyTupleIterObject *it = (_PyTupleIterObject *)iter;
assert(Py_TYPE(iter) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++));
stack_pointer[0] = next;
stack_pointer += 1;
break;
}
case _ITER_CHECK_RANGE: {
PyObject *iter;
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
if (Py_TYPE(r) != &PyRangeIter_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
/* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */
case _GUARD_NOT_EXHAUSTED_RANGE: {
PyObject *iter;
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
assert(Py_TYPE(r) == &PyRangeIter_Type);
if (r->len <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _ITER_NEXT_RANGE: {
PyObject *iter;
PyObject *next;
iter = stack_pointer[-1];
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
assert(Py_TYPE(r) == &PyRangeIter_Type);
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
r->len--;
next = PyLong_FromLong(value);
if (next == NULL) JUMP_TO_ERROR();
stack_pointer[0] = next;
stack_pointer += 1;
break;
}
case _FOR_ITER_GEN_FRAME: {
PyObject *iter;
_PyInterpreterFrame *gen_frame;
oparg = CURRENT_OPARG();
iter = stack_pointer[-1];
PyGenObject *gen = (PyGenObject *)iter;
if (Py_TYPE(gen) != &PyGen_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (gen->gi_frame_state >= FRAME_EXECUTING) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(FOR_ITER, hit);
gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
_PyFrame_StackPush(gen_frame, Py_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
stack_pointer[0] = (PyObject *)gen_frame;
stack_pointer += 1;
break;
}
/* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
/* _BEFORE_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */
case _WITH_EXCEPT_START: {
PyObject *val;
PyObject *lasti;
PyObject *exit_func;
PyObject *res;
val = stack_pointer[-1];
lasti = stack_pointer[-3];
exit_func = stack_pointer[-4];
/* At the top of the stack are 4 values:
- val: TOP = exc_info()
- unused: SECOND = previous exception
- lasti: THIRD = lasti of exception in exc_info()
- exit_func: FOURTH = the context.__exit__ bound method
We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
Then we push the __exit__ return value.
*/
PyObject *exc, *tb;
assert(val && PyExceptionInstance_Check(val));
exc = PyExceptionInstance_Class(val);
tb = PyException_GetTraceback(val);
if (tb == NULL) {
tb = Py_None;
}
else {
Py_DECREF(tb);
}
assert(PyLong_Check(lasti));
(void)lasti; // Shut up compiler warning if asserts are off
PyObject *stack[4] = {NULL, exc, val, tb};
res = PyObject_Vectorcall(exit_func, stack + 1,
3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
case _PUSH_EXC_INFO: {
PyObject *new_exc;
PyObject *prev_exc;
new_exc = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
if (exc_info->exc_value != NULL) {
prev_exc = exc_info->exc_value;
}
else {
prev_exc = Py_None;
}
assert(PyExceptionInstance_Check(new_exc));
exc_info->exc_value = Py_NewRef(new_exc);
stack_pointer[-1] = prev_exc;
stack_pointer[0] = new_exc;
stack_pointer += 1;
break;
}
case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (!_PyObject_InlineValues(owner)->valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_KEYS_VERSION: {
PyObject *owner;
owner = stack_pointer[-1];
uint32_t keys_version = (uint32_t)CURRENT_OPERAND();
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
if (owner_heap_type->ht_cached_keys->dk_version != keys_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _LOAD_ATTR_METHOD_WITH_VALUES: {
PyObject *owner;
PyObject *attr;
PyObject *self = NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = Py_NewRef(descr);
assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
self = owner;
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
break;
}
case _LOAD_ATTR_METHOD_NO_DICT: {
PyObject *owner;
PyObject *attr;
PyObject *self = NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert(oparg & 1);
assert(Py_TYPE(owner)->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = Py_NewRef(descr);
self = owner;
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
break;
}
case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: {
PyObject *owner;
PyObject *attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
Py_DECREF(owner);
attr = Py_NewRef(descr);
stack_pointer[-1] = attr;
break;
}
case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: {
PyObject *owner;
PyObject *attr;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert((oparg & 1) == 0);
assert(Py_TYPE(owner)->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
Py_DECREF(owner);
attr = Py_NewRef(descr);
stack_pointer[-1] = attr;
break;
}
case _CHECK_ATTR_METHOD_LAZY_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND();
char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
if (dict != NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _LOAD_ATTR_METHOD_LAZY_DICT: {
PyObject *owner;
PyObject *attr;
PyObject *self = NULL;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *descr = (PyObject *)CURRENT_OPERAND();
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = Py_NewRef(descr);
self = owner;
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
break;
}
/* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */
/* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _CHECK_PERIODIC: {
CHECK_EVAL_BREAKER();
break;
}
case _PY_FRAME_GENERAL: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
assert(Py_TYPE(callable) == &PyFunction_Type);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)callable, locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
stack_pointer += -2 - oparg;
if (new_frame == NULL) {
JUMP_TO_ERROR();
}
stack_pointer[0] = (PyObject *)new_frame;
stack_pointer += 1;
break;
}
case _CHECK_FUNCTION_VERSION: {
PyObject *callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
if (!PyFunction_Check(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyFunctionObject *func = (PyFunctionObject *)callable;
if (func->func_version != func_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _CHECK_METHOD_VERSION: {
PyObject *null;
PyObject *callable;
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
if (Py_TYPE(callable) != &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyObject *func = ((PyMethodObject *)callable)->im_func;
if (!PyFunction_Check(func)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->func_version != func_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (null != NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _EXPAND_METHOD: {
PyObject *null;
PyObject *callable;
PyObject *method;
PyObject *self;
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(null == NULL);
assert(Py_TYPE(callable) == &PyMethod_Type);
self = ((PyMethodObject *)callable)->im_self;
Py_INCREF(self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL
method = ((PyMethodObject *)callable)->im_func;
assert(PyFunction_Check(method));
Py_INCREF(method);
Py_DECREF(callable);
stack_pointer[-2 - oparg] = method;
stack_pointer[-1 - oparg] = self;
break;
}
case _CHECK_IS_NOT_PY_CALLABLE: {
PyObject *callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
if (PyFunction_Check(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (Py_TYPE(callable) == &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _CALL_NON_PY_GENERAL: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
/* Callable is not a normal Python function */
res = PyObject_Vectorcall(
callable, args,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: {
PyObject *null;
PyObject *callable;
oparg = CURRENT_OPARG();
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
if (null != NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (Py_TYPE(callable) != &PyMethod_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: {
PyObject *callable;
PyObject *func;
PyObject *self;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
STAT_INC(CALL, hit);
self = Py_NewRef(((PyMethodObject *)callable)->im_self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
func = Py_NewRef(((PyMethodObject *)callable)->im_func);
stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization
Py_DECREF(callable);
stack_pointer[-2 - oparg] = func;
stack_pointer[-1 - oparg] = self;
break;
}
case _CHECK_PEP_523: {
if (tstate->interp->eval_frame) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _CHECK_FUNCTION_EXACT_ARGS: {
PyObject *self_or_null;
PyObject *callable;
oparg = CURRENT_OPARG();
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
if (!PyFunction_Check(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyFunctionObject *func = (PyFunctionObject *)callable;
if (func->func_version != func_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyCodeObject *code = (PyCodeObject *)func->func_code;
if (code->co_argcount != oparg + (self_or_null != NULL)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _CHECK_STACK_SPACE: {
PyObject *callable;
oparg = CURRENT_OPARG();
callable = stack_pointer[-2 - oparg];
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (tstate->py_recursion_remaining <= 1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _INIT_CALL_PY_EXACT_ARGS_0: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = 0;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
stack_pointer[-2 - oparg] = (PyObject *)new_frame;
stack_pointer += -1 - oparg;
break;
}
case _INIT_CALL_PY_EXACT_ARGS_1: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = 1;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
stack_pointer[-2 - oparg] = (PyObject *)new_frame;
stack_pointer += -1 - oparg;
break;
}
case _INIT_CALL_PY_EXACT_ARGS_2: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = 2;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
stack_pointer[-2 - oparg] = (PyObject *)new_frame;
stack_pointer += -1 - oparg;
break;
}
case _INIT_CALL_PY_EXACT_ARGS_3: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = 3;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
stack_pointer[-2 - oparg] = (PyObject *)new_frame;
stack_pointer += -1 - oparg;
break;
}
case _INIT_CALL_PY_EXACT_ARGS_4: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = 4;
assert(oparg == CURRENT_OPARG());
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
stack_pointer[-2 - oparg] = (PyObject *)new_frame;
stack_pointer += -1 - oparg;
break;
}
case _INIT_CALL_PY_EXACT_ARGS: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
_PyInterpreterFrame *new_frame;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
stack_pointer[-2 - oparg] = (PyObject *)new_frame;
stack_pointer += -1 - oparg;
break;
}
case _PUSH_FRAME: {
_PyInterpreterFrame *new_frame;
new_frame = (_PyInterpreterFrame *)stack_pointer[-1];
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
stack_pointer += -1;
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
break;
}
case _CALL_TYPE_1: {
PyObject *arg;
PyObject *null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
if (null != NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (callable != (PyObject *)&PyType_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
res = Py_NewRef(Py_TYPE(arg));
Py_DECREF(arg);
stack_pointer[-3] = res;
stack_pointer += -2;
break;
}
case _CALL_STR_1: {
PyObject *arg;
PyObject *null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
if (null != NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (callable != (PyObject *)&PyUnicode_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
res = PyObject_Str(arg);
Py_DECREF(arg);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
break;
}
case _CALL_TUPLE_1: {
PyObject *arg;
PyObject *null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
if (null != NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (callable != (PyObject *)&PyTuple_Type) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
res = PySequence_Tuple(arg);
Py_DECREF(arg);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-3] = res;
stack_pointer += -2;
break;
}
/* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _EXIT_INIT_CHECK: {
PyObject *should_be_none;
should_be_none = stack_pointer[-1];
assert(STACK_LEVEL() == 2);
if (should_be_none != Py_None) {
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
JUMP_TO_ERROR();
}
stack_pointer += -1;
break;
}
case _CALL_BUILTIN_CLASS: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (!PyType_Check(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyTypeObject *tp = (PyTypeObject *)callable;
if (tp->tp_vectorcall == NULL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(tp);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_BUILTIN_O: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_O functions */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (total_args != 1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyCFunction_CheckExact(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (PyCFunction_GET_FLAGS(callable) != METH_O) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// CPython promises to check all non-vectorcall function calls.
if (tstate->c_recursion_remaining <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
PyObject *arg = args[0];
_Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_BUILTIN_FAST: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_FASTCALL functions, without keywords */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (!PyCFunction_CheckExact(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
/* res = func(self, args, nargs) */
res = ((PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_BUILTIN_FAST_WITH_KEYWORDS: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (!PyCFunction_CheckExact(callable)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_LEN: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* len(o) */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (total_args != 1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyInterpreterState *interp = tstate->interp;
if (callable != interp->callable_cache.len) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
JUMP_TO_ERROR();
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
if (res == NULL) {
GOTO_ERROR(error);
}
Py_DECREF(callable);
Py_DECREF(arg);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_ISINSTANCE: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* isinstance(o, o2) */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (total_args != 2) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyInterpreterState *interp = tstate->interp;
if (callable != interp->callable_cache.isinstance) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyObject *cls = args[1];
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
JUMP_TO_ERROR();
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
if (res == NULL) {
GOTO_ERROR(error);
}
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_METHOD_DESCRIPTOR_O: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
if (total_args != 2) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != METH_O) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// CPython promises to check all non-vectorcall function calls.
if (tstate->c_recursion_remaining <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyObject *arg = args[1];
PyObject *self = args[0];
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = args[0];
if (!Py_IS_TYPE(self, d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
res = cfunc(self, args + 1, nargs, NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_METHOD_DESCRIPTOR_NOARGS: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
assert(oparg == 0 || oparg == 1);
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (total_args != 1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyMethodDef *meth = method->d_method;
PyObject *self = args[0];
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (meth->ml_flags != METH_NOARGS) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
// CPython promises to check all non-vectorcall function calls.
if (tstate->c_recursion_remaining <= 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
case _CALL_METHOD_DESCRIPTOR_FAST: {
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
oparg = CURRENT_OPARG();
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
/* Builtin METH_FASTCALL methods, without keywords */
if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyMethodDef *meth = method->d_method;
if (meth->ml_flags != METH_FASTCALL) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
PyObject *self = args[0];
if (!Py_IS_TYPE(self, method->d_common.d_type)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(CALL, hit);
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
break;
}
/* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */
/* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
/* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */
/* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _MAKE_FUNCTION: {
PyObject *codeobj;
PyObject *func;
codeobj = stack_pointer[-1];
PyFunctionObject *func_obj = (PyFunctionObject *)
PyFunction_New(codeobj, GLOBALS());
Py_DECREF(codeobj);
if (func_obj == NULL) {
JUMP_TO_ERROR();
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
func = (PyObject *)func_obj;
stack_pointer[-1] = func;
break;
}
case _SET_FUNCTION_ATTRIBUTE: {
PyObject *func;
PyObject *attr;
oparg = CURRENT_OPARG();
func = stack_pointer[-1];
attr = stack_pointer[-2];
assert(PyFunction_Check(func));
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
case MAKE_FUNCTION_CLOSURE:
assert(func_obj->func_closure == NULL);
func_obj->func_closure = attr;
break;
case MAKE_FUNCTION_ANNOTATIONS:
assert(func_obj->func_annotations == NULL);
func_obj->func_annotations = attr;
break;
case MAKE_FUNCTION_KWDEFAULTS:
assert(PyDict_CheckExact(attr));
assert(func_obj->func_kwdefaults == NULL);
func_obj->func_kwdefaults = attr;
break;
case MAKE_FUNCTION_DEFAULTS:
assert(PyTuple_CheckExact(attr));
assert(func_obj->func_defaults == NULL);
func_obj->func_defaults = attr;
break;
default:
Py_UNREACHABLE();
}
stack_pointer[-2] = func;
stack_pointer += -1;
break;
}
case _RETURN_GENERATOR: {
PyObject *res;
assert(PyFunction_Check(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
JUMP_TO_ERROR();
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
frame->instr_ptr++;
_PyFrame_Copy(frame, gen_frame);
assert(frame->frame_obj == NULL);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
res = (PyObject *)gen;
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
LOAD_IP(frame->return_offset);
LOAD_SP();
LLTRACE_RESUME_FRAME();
stack_pointer[0] = res;
stack_pointer += 1;
break;
}
case _BUILD_SLICE: {
PyObject *step = NULL;
PyObject *stop;
PyObject *start;
PyObject *slice;
oparg = CURRENT_OPARG();
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
slice = PySlice_New(start, stop, step);
Py_DECREF(start);
Py_DECREF(stop);
Py_XDECREF(step);
if (slice == NULL) JUMP_TO_ERROR();
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
break;
}
case _CONVERT_VALUE: {
PyObject *value;
PyObject *result;
oparg = CURRENT_OPARG();
value = stack_pointer[-1];
conversion_func conv_fn;
assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
conv_fn = _PyEval_ConversionFuncs[oparg];
result = conv_fn(value);
Py_DECREF(value);
if (result == NULL) JUMP_TO_ERROR();
stack_pointer[-1] = result;
break;
}
case _FORMAT_SIMPLE: {
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
/* If value is a unicode object, then we know the result
* of format(value) is value itself. */
if (!PyUnicode_CheckExact(value)) {
res = PyObject_Format(value, NULL);
Py_DECREF(value);
if (res == NULL) JUMP_TO_ERROR();
}
else {
res = value;
}
stack_pointer[-1] = res;
break;
}
case _FORMAT_WITH_SPEC: {
PyObject *fmt_spec;
PyObject *value;
PyObject *res;
fmt_spec = stack_pointer[-1];
value = stack_pointer[-2];
res = PyObject_Format(value, fmt_spec);
Py_DECREF(value);
Py_DECREF(fmt_spec);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _COPY: {
PyObject *bottom;
PyObject *top;
oparg = CURRENT_OPARG();
bottom = stack_pointer[-1 - (oparg-1)];
assert(oparg > 0);
top = Py_NewRef(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
break;
}
case _BINARY_OP: {
PyObject *rhs;
PyObject *lhs;
PyObject *res;
oparg = CURRENT_OPARG();
rhs = stack_pointer[-1];
lhs = stack_pointer[-2];
assert(_PyEval_BinaryOps[oparg]);
res = _PyEval_BinaryOps[oparg](lhs, rhs);
Py_DECREF(lhs);
Py_DECREF(rhs);
if (res == NULL) JUMP_TO_ERROR();
stack_pointer[-2] = res;
stack_pointer += -1;
break;
}
case _SWAP: {
PyObject *top;
PyObject *bottom;
oparg = CURRENT_OPARG();
top = stack_pointer[-1];
bottom = stack_pointer[-2 - (oparg-2)];
assert(oparg >= 2);
stack_pointer[-2 - (oparg-2)] = top;
stack_pointer[-1] = bottom;
break;
}
/* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */
/* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */
case _GUARD_IS_TRUE_POP: {
PyObject *flag;
flag = stack_pointer[-1];
stack_pointer += -1;
if (!Py_IsTrue(flag)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(Py_IsTrue(flag));
break;
}
case _GUARD_IS_FALSE_POP: {
PyObject *flag;
flag = stack_pointer[-1];
stack_pointer += -1;
if (!Py_IsFalse(flag)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(Py_IsFalse(flag));
break;
}
case _GUARD_IS_NONE_POP: {
PyObject *val;
val = stack_pointer[-1];
stack_pointer += -1;
if (!Py_IsNone(val)) {
Py_DECREF(val);
if (1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
}
break;
}
case _GUARD_IS_NOT_NONE_POP: {
PyObject *val;
val = stack_pointer[-1];
stack_pointer += -1;
if (Py_IsNone(val)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
Py_DECREF(val);
break;
}
case _JUMP_TO_TOP: {
#ifndef _Py_JIT
next_uop = &current_executor->trace[1];
#endif
break;
}
case _SET_IP: {
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND();
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
break;
}
case _CHECK_STACK_SPACE_OPERAND: {
uint32_t framesize = (uint32_t)CURRENT_OPERAND();
assert(framesize <= INT_MAX);
if (!_PyThreadState_HasStackSpace(tstate, framesize)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (tstate->py_recursion_remaining <= 1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _SAVE_RETURN_OFFSET: {
oparg = CURRENT_OPARG();
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
break;
}
case _EXIT_TRACE: {
EXIT_TO_TRACE();
break;
}
case _CHECK_VALIDITY: {
if (!current_executor->vm_data.valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _LOAD_CONST_INLINE: {
PyObject *value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
value = Py_NewRef(ptr);
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _LOAD_CONST_INLINE_BORROW: {
PyObject *value;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
value = ptr;
stack_pointer[0] = value;
stack_pointer += 1;
break;
}
case _POP_TOP_LOAD_CONST_INLINE_BORROW: {
PyObject *pop;
PyObject *value;
pop = stack_pointer[-1];
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
Py_DECREF(pop);
value = ptr;
stack_pointer[-1] = value;
break;
}
case _LOAD_CONST_INLINE_WITH_NULL: {
PyObject *value;
PyObject *null;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
value = Py_NewRef(ptr);
null = NULL;
stack_pointer[0] = value;
stack_pointer[1] = null;
stack_pointer += 2;
break;
}
case _LOAD_CONST_INLINE_BORROW_WITH_NULL: {
PyObject *value;
PyObject *null;
PyObject *ptr = (PyObject *)CURRENT_OPERAND();
value = ptr;
null = NULL;
stack_pointer[0] = value;
stack_pointer[1] = null;
stack_pointer += 2;
break;
}
case _CHECK_FUNCTION: {
uint32_t func_version = (uint32_t)CURRENT_OPERAND();
assert(PyFunction_Check(frame->f_funcobj));
if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _INTERNAL_INCREMENT_OPT_COUNTER: {
PyObject *opt;
opt = stack_pointer[-1];
_PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)opt;
exe->count++;
stack_pointer += -1;
break;
}
case _COLD_EXIT: {
oparg = CURRENT_OPARG();
_PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
_PyExitData *exit = &previous->exits[oparg];
PyCodeObject *code = _PyFrame_GetCode(frame);
_Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
_Py_BackoffCounter temperature = exit->temperature;
if (!backoff_counter_triggers(temperature)) {
exit->temperature = advance_backoff_counter(temperature);
GOTO_TIER_ONE(target);
}
_PyExecutorObject *executor;
if (target->op.code == ENTER_EXECUTOR) {
executor = code->co_executors->executors[target->op.arg];
Py_INCREF(executor);
}
else {
int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
if (optimized <= 0) {
exit->temperature = restart_backoff_counter(temperature);
if (optimized < 0) {
Py_DECREF(previous);
tstate->previous_executor = Py_None;
GOTO_UNWIND();
}
GOTO_TIER_ONE(target);
}
}
/* We need two references. One to store in exit->executor and
* one to keep the executor alive when executing. */
Py_INCREF(executor);
exit->executor = executor;
GOTO_TIER_TWO(executor);
break;
}
case _DYNAMIC_EXIT: {
oparg = CURRENT_OPARG();
tstate->previous_executor = (PyObject *)current_executor;
_PyExitData *exit = (_PyExitData *)&current_executor->exits[oparg];
_Py_CODEUNIT *target = frame->instr_ptr;
_PyExecutorObject *executor;
if (target->op.code == ENTER_EXECUTOR) {
PyCodeObject *code = (PyCodeObject *)frame->f_executable;
executor = code->co_executors->executors[target->op.arg];
Py_INCREF(executor);
}
else {
if (!backoff_counter_triggers(exit->temperature)) {
exit->temperature = advance_backoff_counter(exit->temperature);
GOTO_TIER_ONE(target);
}
int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
if (optimized <= 0) {
exit->temperature = restart_backoff_counter(exit->temperature);
if (optimized < 0) {
Py_DECREF(current_executor);
tstate->previous_executor = Py_None;
GOTO_UNWIND();
}
GOTO_TIER_ONE(target);
}
else {
exit->temperature = initial_temperature_backoff_counter();
}
}
GOTO_TIER_TWO(executor);
break;
}
case _START_EXECUTOR: {
PyObject *executor = (PyObject *)CURRENT_OPERAND();
Py_DECREF(tstate->previous_executor);
tstate->previous_executor = NULL;
#ifndef _Py_JIT
current_executor = (_PyExecutorObject*)executor;
#endif
if (!((_PyExecutorObject *)executor)->vm_data.valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _FATAL_ERROR: {
assert(0);
Py_FatalError("Fatal error uop executed.");
break;
}
case _CHECK_VALIDITY_AND_SET_IP: {
PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND();
if (!current_executor->vm_data.valid) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr;
break;
}
case _DEOPT: {
EXIT_TO_TIER1();
break;
}
case _ERROR_POP_N: {
oparg = CURRENT_OPARG();
uint32_t target = (uint32_t)CURRENT_OPERAND();
frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target;
stack_pointer += -oparg;
GOTO_UNWIND();
break;
}
case _TIER2_RESUME_CHECK: {
#if defined(__EMSCRIPTEN__)
if (_Py_emscripten_signal_clock == 0) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
if (eval_breaker & _PY_EVAL_EVENTS_MASK) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version));
break;
}
#undef TIER_TWO
Reference in a new issue View git blame Copy permalink