mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-10 04:45:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9
cpython/Python/generated_cases.c.h
Jelle Zijlstra a0559849ac
[3.13] gh-119821: Support non-dict globals in LOAD_FROM_DICT_OR_GLOBALS (#119822) (#119889) 
dSupport non-dict globals in LOAD_FROM_DICT_OR_GLOBALS

The implementation basically copies LOAD_GLOBAL. Possibly it could be deduplicated,
but that seems like it may get hairy since the two operations have different operands.

This is important to fix in 3.14 for PEP 649, but it's a bug in earlier versions too,
and we should backport to 3.13 and 3.12 if possible.

(cherry picked from commit 80a4e38994)
2024-05-31 21:56:26 -07:00

6254 lines
242 KiB
C
Generated
Raw Blame History

// This file is generated by Tools/cases_generator/tier1_generator.py
// from:
// Python/bytecodes.c
// Do not edit!
#ifdef TIER_TWO
#error "This file is for Tier 1 only"
#endif
#define TIER_ONE 1
TARGET(BEFORE_ASYNC_WITH) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BEFORE_ASYNC_WITH);
PyObject *mgr;
PyObject *exit;
PyObject *res;
mgr = stack_pointer[-1];
PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__));
if (enter == NULL) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object does not support the "
"asynchronous context manager protocol",
Py_TYPE(mgr)->tp_name);
}
goto error;
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__));
if (exit == NULL) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object does not support the "
"asynchronous context manager protocol "
"(missed __aexit__ method)",
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
goto error;
}
Py_DECREF(mgr);
res = PyObject_CallNoArgs(enter);
Py_DECREF(enter);
if (res == NULL) {
Py_DECREF(exit);
if (true) goto pop_1_error;
}
stack_pointer[-1] = exit;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(BEFORE_WITH) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BEFORE_WITH);
PyObject *mgr;
PyObject *exit;
PyObject *res;
mgr = stack_pointer[-1];
/* pop the context manager, push its __exit__ and the
* value returned from calling its __enter__
*/
PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__));
if (enter == NULL) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object does not support the "
"context manager protocol",
Py_TYPE(mgr)->tp_name);
}
goto error;
}
exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__));
if (exit == NULL) {
if (!_PyErr_Occurred(tstate)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object does not support the "
"context manager protocol "
"(missed __exit__ method)",
Py_TYPE(mgr)->tp_name);
}
Py_DECREF(enter);
goto error;
}
Py_DECREF(mgr);
res = PyObject_CallNoArgs(enter);
Py_DECREF(enter);
if (res == NULL) {
Py_DECREF(exit);
if (true) goto pop_1_error;
}
stack_pointer[-1] = exit;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(BINARY_OP) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP);
PREDICTED(BINARY_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *rhs;
PyObject *lhs;
PyObject *res;
// _SPECIALIZE_BINARY_OP
rhs = stack_pointer[-1];
lhs = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
DISPATCH_SAME_OPARG();
}
STAT_INC(BINARY_OP, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
assert(NB_ADD <= oparg);
assert(oparg <= NB_INPLACE_XOR);
}
// _BINARY_OP
{
assert(_PyEval_BinaryOps[oparg]);
res = _PyEval_BinaryOps[oparg](lhs, rhs);
Py_DECREF(lhs);
Py_DECREF(rhs);
if (res == NULL) goto pop_2_error;
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_ADD_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_FLOAT
{
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left)->ob_fval +
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_ADD_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_INT
{
STAT_INC(BINARY_OP, hit);
res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_ADD_UNICODE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_ADD_UNICODE
{
STAT_INC(BINARY_OP, hit);
res = PyUnicode_Concat(left, right);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (res == NULL) goto pop_2_error;
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_INPLACE_ADD_UNICODE
{
assert(next_instr->op.code == STORE_FAST);
PyObject **target_local = &GETLOCAL(next_instr->op.arg);
DEOPT_IF(*target_local != left, BINARY_OP);
STAT_INC(BINARY_OP, hit);
/* Handle `left = left + right` or `left += right` for str.
*
* When possible, extend `left` in place rather than
* allocating a new PyUnicodeObject. This attempts to avoid
* quadratic behavior when one neglects to use str.join().
*
* If `left` has only two references remaining (one from
* the stack, one in the locals), DECREFing `left` leaves
* only the locals reference, so PyUnicode_Append knows
* that the string is safe to mutate.
*/
assert(Py_REFCNT(left) >= 2);
_Py_DECREF_NO_DEALLOC(left);
PyUnicode_Append(target_local, right);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (*target_local == NULL) goto pop_2_error;
// The STORE_FAST is already done.
assert(next_instr->op.code == STORE_FAST);
SKIP_OVER(1);
}
stack_pointer += -2;
DISPATCH();
}
TARGET(BINARY_OP_MULTIPLY_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_MULTIPLY_FLOAT
{
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left)->ob_fval *
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_MULTIPLY_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_MULTIPLY_INT
{
STAT_INC(BINARY_OP, hit);
res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_SUBTRACT_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_SUBTRACT_FLOAT
{
STAT_INC(BINARY_OP, hit);
double dres =
((PyFloatObject *)left)->ob_fval -
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_OP_SUBTRACT_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
}
/* Skip 1 cache entry */
// _BINARY_OP_SUBTRACT_INT
{
STAT_INC(BINARY_OP, hit);
res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_SLICE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BINARY_SLICE);
PyObject *stop;
PyObject *start;
PyObject *container;
PyObject *res;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
// Can't use ERROR_IF() here, because we haven't
// DECREF'ed container yet, and we still own slice.
if (slice == NULL) {
res = NULL;
}
else {
res = PyObject_GetItem(container, slice);
Py_DECREF(slice);
}
Py_DECREF(container);
if (res == NULL) goto pop_3_error;
stack_pointer[-3] = res;
stack_pointer += -2;
DISPATCH();
}
TARGET(BINARY_SUBSCR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR);
PREDICTED(BINARY_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *sub;
PyObject *container;
PyObject *res;
// _SPECIALIZE_BINARY_SUBSCR
sub = stack_pointer[-1];
container = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_BinarySubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(BINARY_SUBSCR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _BINARY_SUBSCR
{
res = PyObject_GetItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
if (res == NULL) goto pop_2_error;
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_SUBSCR_DICT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *dict;
PyObject *res;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
dict = stack_pointer[-2];
DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
int rc = PyDict_GetItemRef(dict, sub, &res);
if (rc == 0) {
_PyErr_SetKeyError(sub);
}
Py_DECREF(dict);
Py_DECREF(sub);
if (rc <= 0) goto pop_2_error;
// not found or error
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_SUBSCR_GETITEM) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *container;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
container = stack_pointer[-2];
DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR);
PyTypeObject *tp = Py_TYPE(container);
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR);
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *cached = ht->_spec_cache.getitem;
DEOPT_IF(cached == NULL, BINARY_SUBSCR);
assert(PyFunction_Check(cached));
PyFunctionObject *getitem = (PyFunctionObject *)cached;
uint32_t cached_version = ht->_spec_cache.getitem_version;
DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR);
PyCodeObject *code = (PyCodeObject *)getitem->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
Py_INCREF(getitem);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);
STACK_SHRINK(2);
new_frame->localsplus[0] = container;
new_frame->localsplus[1] = sub;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
TARGET(BINARY_SUBSCR_LIST_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *list;
PyObject *res;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
list = stack_pointer[-2];
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
// Deopt unless 0 <= sub < PyList_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
res = PyList_GET_ITEM(list, index);
assert(res != NULL);
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_SUBSCR_STR_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *str;
PyObject *res;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
str = stack_pointer[-2];
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR);
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR);
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(str);
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BINARY_SUBSCR_TUPLE_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *tuple;
PyObject *res;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
tuple = stack_pointer[-2];
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
// Deopt unless 0 <= sub < PyTuple_Size(list)
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
res = PyTuple_GET_ITEM(tuple, index);
assert(res != NULL);
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(tuple);
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(BUILD_CONST_KEY_MAP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_CONST_KEY_MAP);
PyObject *keys;
PyObject **values;
PyObject *map;
keys = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
assert(PyTuple_CheckExact(keys));
assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg);
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(values[_i]);
}
Py_DECREF(keys);
if (map == NULL) { stack_pointer += -1 - oparg; goto error; }
stack_pointer[-1 - oparg] = map;
stack_pointer += -oparg;
DISPATCH();
}
TARGET(BUILD_LIST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_LIST);
PyObject **values;
PyObject *list;
values = &stack_pointer[-oparg];
list = _PyList_FromArraySteal(values, oparg);
if (list == NULL) { stack_pointer += -oparg; goto error; }
stack_pointer[-oparg] = list;
stack_pointer += 1 - oparg;
DISPATCH();
}
TARGET(BUILD_MAP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_MAP);
PyObject **values;
PyObject *map;
values = &stack_pointer[-oparg*2];
map = _PyDict_FromItems(
values, 2,
values+1, 2,
oparg);
for (int _i = oparg*2; --_i >= 0;) {
Py_DECREF(values[_i]);
}
if (map == NULL) { stack_pointer += -oparg*2; goto error; }
stack_pointer[-oparg*2] = map;
stack_pointer += 1 - oparg*2;
DISPATCH();
}
TARGET(BUILD_SET) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SET);
PyObject **values;
PyObject *set;
values = &stack_pointer[-oparg];
set = PySet_New(NULL);
if (set == NULL)
goto error;
int err = 0;
for (int i = 0; i < oparg; i++) {
PyObject *item = values[i];
if (err == 0)
err = PySet_Add(set, item);
Py_DECREF(item);
}
if (err != 0) {
Py_DECREF(set);
if (true) { stack_pointer += -oparg; goto error; }
}
stack_pointer[-oparg] = set;
stack_pointer += 1 - oparg;
DISPATCH();
}
TARGET(BUILD_SLICE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_SLICE);
PyObject *step = NULL;
PyObject *stop;
PyObject *start;
PyObject *slice;
if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; }
stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)];
start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)];
slice = PySlice_New(start, stop, step);
Py_DECREF(start);
Py_DECREF(stop);
Py_XDECREF(step);
if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; }
stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice;
stack_pointer += -1 - ((oparg == 3) ? 1 : 0);
DISPATCH();
}
TARGET(BUILD_STRING) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_STRING);
PyObject **pieces;
PyObject *str;
pieces = &stack_pointer[-oparg];
str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(pieces[_i]);
}
if (str == NULL) { stack_pointer += -oparg; goto error; }
stack_pointer[-oparg] = str;
stack_pointer += 1 - oparg;
DISPATCH();
}
TARGET(BUILD_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(BUILD_TUPLE);
PyObject **values;
PyObject *tup;
values = &stack_pointer[-oparg];
tup = _PyTuple_FromArraySteal(values, oparg);
if (tup == NULL) { stack_pointer += -oparg; goto error; }
stack_pointer[-oparg] = tup;
stack_pointer += 1 - oparg;
DISPATCH();
}
TARGET(CACHE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CACHE);
assert(0 && "Executing a cache.");
Py_FatalError("Executing a cache.");
DISPATCH();
}
TARGET(CALL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL);
PREDICTED(CALL);
_Py_CODEUNIT *this_instr = next_instr - 4;
(void)this_instr;
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
// _SPECIALIZE_CALL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
DISPATCH_SAME_OPARG();
}
STAT_INC(CALL, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 2 cache entries */
// _CALL
{
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
else if (Py_TYPE(callable) == &PyMethod_Type) {
args--;
total_args++;
PyObject *self = ((PyMethodObject *)callable)->im_self;
args[0] = Py_NewRef(self);
PyObject *method = ((PyMethodObject *)callable)->im_func;
args[-1] = Py_NewRef(method);
Py_DECREF(callable);
callable = method;
}
// Check if the call can be inlined or not
if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)callable, locals,
args, total_args, NULL
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
goto error;
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
res = PyObject_Vectorcall(
callable, args,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
if (opcode == INSTRUMENTED_CALL) {
PyObject *arg = total_args == 0 ?
&_PyInstrumentation_MISSING : args[0];
if (res == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, callable, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, callable, arg);
if (err < 0) {
Py_CLEAR(res);
}
}
}
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_ALLOC_AND_ENTER_INIT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *null;
PyObject *callable;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* This instruction does the following:
* 1. Creates the object (by calling ``object.__new__``)
* 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
* 3. Pushes the frame for ``__init__`` to the frame stack
* */
_PyCallCache *cache = (_PyCallCache *)&this_instr[1];
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
DEOPT_IF(code->co_argcount != oparg+1, CALL);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
STAT_INC(CALL, hit);
PyObject *self = _PyType_NewManagedObject(tp);
if (self == NULL) {
goto error;
}
Py_DECREF(tp);
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
Py_INCREF(self);
shim->localsplus[0] = self;
Py_INCREF(init);
_PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
/* Copy self followed by args to __init__ frame */
init_frame->localsplus[0] = self;
for (int i = 0; i < oparg; i++) {
init_frame->localsplus[i+1] = args[i];
}
frame->return_offset = (uint16_t)(next_instr - this_instr);
STACK_SHRINK(oparg+2);
_PyFrame_SetStackPointer(frame, stack_pointer);
/* Link frames */
init_frame->previous = shim;
shim->previous = frame;
frame = tstate->current_frame = init_frame;
CALL_STAT_INC(inlined_py_calls);
/* Account for pushing the extra frame.
* We don't check recursion depth here,
* as it will be checked after start_frame */
tstate->py_recursion_remaining--;
goto start_frame;
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *null;
PyObject *callable;
PyObject *func;
PyObject *self;
PyObject *self_or_null;
PyObject **args;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_CALL_BOUND_METHOD_EXACT_ARGS
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
}
// _INIT_CALL_BOUND_METHOD_EXACT_ARGS
{
STAT_INC(CALL, hit);
self = Py_NewRef(((PyMethodObject *)callable)->im_self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS
func = Py_NewRef(((PyMethodObject *)callable)->im_func);
stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization
Py_DECREF(callable);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = self;
callable = func;
{
uint32_t func_version = read_u32(&this_instr[2].cache);
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
}
// _CHECK_STACK_SPACE
{
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
stack_pointer += -2 - oparg;
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_BOUND_METHOD_GENERAL) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *null;
PyObject *callable;
PyObject *method;
PyObject *self;
PyObject **args;
PyObject *self_or_null;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_METHOD_VERSION
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL);
PyObject *func = ((PyMethodObject *)callable)->im_func;
DEOPT_IF(!PyFunction_Check(func), CALL);
DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL);
DEOPT_IF(null != NULL, CALL);
}
// _EXPAND_METHOD
{
assert(null == NULL);
assert(Py_TYPE(callable) == &PyMethod_Type);
self = ((PyMethodObject *)callable)->im_self;
Py_INCREF(self);
stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL
method = ((PyMethodObject *)callable)->im_func;
assert(PyFunction_Check(method));
Py_INCREF(method);
Py_DECREF(callable);
}
// _PY_FRAME_GENERAL
args = &stack_pointer[-oparg];
self_or_null = self;
callable = method;
{
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
assert(Py_TYPE(callable) == &PyFunction_Type);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)callable, locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
stack_pointer += -2 - oparg;
if (new_frame == NULL) {
goto error;
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_BUILTIN_CLASS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_CLASS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_CLASS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
STAT_INC(CALL, hit);
res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL);
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(tp);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_FAST
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_FASTCALL functions, without keywords */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
/* res = func(self, args, nargs) */
res = ((PyCFunctionFast)(void(*)(void))cfunc)(
PyCFunction_GET_SELF(callable),
args,
total_args);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_FAST_WITH_KEYWORDS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL);
STAT_INC(CALL, hit);
/* res = func(self, args, nargs, kwnames) */
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))
PyCFunction_GET_FUNCTION(callable);
res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_BUILTIN_O) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_BUILTIN_O);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_BUILTIN_O
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
/* Builtin METH_O functions */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
PyObject *arg = args[0];
_Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(arg);
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_FUNCTION_EX);
PREDICTED(CALL_FUNCTION_EX);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
PyObject *kwargs = NULL;
PyObject *callargs;
PyObject *func;
PyObject *result;
if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1)]; }
callargs = stack_pointer[-1 - (oparg & 1)];
func = stack_pointer[-3 - (oparg & 1)];
// DICT_MERGE is called before this opcode if there are kwargs.
// It converts all dict subtypes in kwargs into regular dicts.
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
goto error;
}
PyObject *tuple = PySequence_Tuple(callargs);
if (tuple == NULL) {
goto error;
}
Py_SETREF(callargs, tuple);
}
assert(PyTuple_CheckExact(callargs));
EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) {
PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, func, arg);
if (err) goto error;
result = PyObject_Call(func, callargs, kwargs);
if (!PyFunction_Check(func) && !PyMethod_Check(func)) {
if (result == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, func, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, func, arg);
if (err < 0) {
Py_CLEAR(result);
}
}
}
}
else {
if (Py_TYPE(func) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
assert(PyTuple_CheckExact(callargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
(PyFunctionObject *)func, locals,
nargs, callargs, kwargs);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
DISPATCH_INLINED(new_frame);
}
result = PyObject_Call(func, callargs, kwargs);
}
Py_DECREF(func);
Py_DECREF(callargs);
Py_XDECREF(kwargs);
assert(PEEK(2 + (oparg & 1)) == NULL);
if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; }
stack_pointer[-3 - (oparg & 1)] = result;
stack_pointer += -2 - (oparg & 1);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_INTRINSIC_1) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_1);
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
assert(oparg <= MAX_INTRINSIC_1);
res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);
Py_DECREF(value);
if (res == NULL) goto pop_1_error;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(CALL_INTRINSIC_2) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_INTRINSIC_2);
PyObject *value1;
PyObject *value2;
PyObject *res;
value1 = stack_pointer[-1];
value2 = stack_pointer[-2];
assert(oparg <= MAX_INTRINSIC_2);
res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);
Py_DECREF(value2);
Py_DECREF(value1);
if (res == NULL) goto pop_2_error;
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(CALL_ISINSTANCE) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_ISINSTANCE);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* isinstance(o, o2) */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(total_args != 2, CALL);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable != interp->callable_cache.isinstance, CALL);
STAT_INC(CALL, hit);
PyObject *cls = args[1];
PyObject *inst = args[0];
int retval = PyObject_IsInstance(inst, cls);
if (retval < 0) {
goto error;
}
res = PyBool_FromLong(retval);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
if (res == NULL) {
GOTO_ERROR(error);
}
Py_DECREF(inst);
Py_DECREF(cls);
Py_DECREF(callable);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
DISPATCH();
}
TARGET(CALL_KW) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CALL_KW);
PREDICTED(CALL_KW);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
PyObject *kwnames;
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
kwnames = stack_pointer[-1];
args = &stack_pointer[-1 - oparg];
self_or_null = stack_pointer[-2 - oparg];
callable = stack_pointer[-3 - oparg];
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
args--;
total_args++;
PyObject *self = ((PyMethodObject *)callable)->im_self;
args[0] = Py_NewRef(self);
PyObject *method = ((PyMethodObject *)callable)->im_func;
args[-1] = Py_NewRef(method);
Py_DECREF(callable);
callable = method;
}
int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames);
// Check if the call can be inlined or not
if (Py_TYPE(callable) == &PyFunction_Type &&
tstate->interp->eval_frame == NULL &&
((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
{
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)callable, locals,
args, positional_args, kwnames
);
Py_DECREF(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 3);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
if (new_frame == NULL) {
goto error;
}
assert(next_instr - this_instr == 1);
frame->return_offset = 1;
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
res = PyObject_Vectorcall(
callable, args,
positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
kwnames);
if (opcode == INSTRUMENTED_CALL_KW) {
PyObject *arg = total_args == 0 ?
&_PyInstrumentation_MISSING : args[0];
if (res == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, callable, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, callable, arg);
if (err < 0) {
Py_CLEAR(res);
}
}
}
Py_DECREF(kwnames);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
if (res == NULL) { stack_pointer += -3 - oparg; goto error; }
stack_pointer[-3 - oparg] = res;
stack_pointer += -2 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_LEN) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LEN);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
/* len(o) */
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable != interp->callable_cache.len, CALL);
STAT_INC(CALL, hit);
PyObject *arg = args[0];
Py_ssize_t len_i = PyObject_Length(arg);
if (len_i < 0) {
goto error;
}
res = PyLong_FromSsize_t(len_i);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
if (res == NULL) {
GOTO_ERROR(error);
}
Py_DECREF(callable);
Py_DECREF(arg);
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
DISPATCH();
}
TARGET(CALL_LIST_APPEND) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_LIST_APPEND);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *arg;
PyObject *self;
PyObject *callable;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
arg = stack_pointer[-1];
self = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
PyInterpreterState *interp = tstate->interp;
DEOPT_IF(callable != interp->callable_cache.list_append, CALL);
assert(self != NULL);
DEOPT_IF(!PyList_Check(self), CALL);
STAT_INC(CALL, hit);
if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) {
goto pop_1_error; // Since arg is DECREF'ed already
}
Py_DECREF(self);
Py_DECREF(callable);
STACK_SHRINK(3);
// Skip POP_TOP
assert(next_instr->op.code == POP_TOP);
SKIP_OVER(1);
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_FAST
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
/* Builtin METH_FASTCALL methods, without keywords */
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunctionFast cfunc =
(PyCFunctionFast)(void(*)(void))meth->ml_meth;
int nargs = total_args - 1;
res = cfunc(self, args + 1, nargs);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Clear the stack of the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL);
PyTypeObject *d_type = method->d_common.d_type;
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL);
STAT_INC(CALL, hit);
int nargs = total_args - 1;
PyCFunctionFastWithKeywords cfunc =
(PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
res = cfunc(self, args + 1, nargs, NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
/* Free the arguments. */
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_NOARGS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
assert(oparg == 0 || oparg == 1);
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
DEOPT_IF(total_args != 1, CALL);
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, NULL);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_METHOD_DESCRIPTOR_O) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject **args;
PyObject *self_or_null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_METHOD_DESCRIPTOR_O
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
PyMethodDescrObject *method = (PyMethodDescrObject *)callable;
DEOPT_IF(total_args != 2, CALL);
DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL);
PyMethodDef *meth = method->d_method;
DEOPT_IF(meth->ml_flags != METH_O, CALL);
// CPython promises to check all non-vectorcall function calls.
DEOPT_IF(tstate->c_recursion_remaining <= 0, CALL);
PyObject *arg = args[1];
PyObject *self = args[0];
DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL);
STAT_INC(CALL, hit);
PyCFunction cfunc = meth->ml_meth;
_Py_EnterRecursiveCallTstateUnchecked(tstate);
res = _PyCFunction_TrampolineCall(cfunc, self, arg);
_Py_LeaveRecursiveCallTstate(tstate);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(self);
Py_DECREF(arg);
Py_DECREF(callable);
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_NON_PY_GENERAL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_NON_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *callable;
PyObject **args;
PyObject *self_or_null;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CHECK_IS_NOT_PY_CALLABLE
callable = stack_pointer[-2 - oparg];
{
DEOPT_IF(PyFunction_Check(callable), CALL);
DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL);
}
// _CALL_NON_PY_GENERAL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
#if TIER_ONE
assert(opcode != INSTRUMENTED_CALL);
#endif
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
/* Callable is not a normal Python function */
res = PyObject_Vectorcall(
callable, args,
total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
NULL);
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
Py_DECREF(callable);
for (int i = 0; i < total_args; i++) {
Py_DECREF(args[i]);
}
if (res == NULL) { stack_pointer += -2 - oparg; goto error; }
}
// _CHECK_PERIODIC
{
}
stack_pointer[-2 - oparg] = res;
stack_pointer += -1 - oparg;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_PY_EXACT_ARGS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_EXACT_ARGS);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *self_or_null;
PyObject *callable;
PyObject **args;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_FUNCTION_EXACT_ARGS
self_or_null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL);
}
// _CHECK_STACK_SPACE
{
PyFunctionObject *func = (PyFunctionObject *)callable;
PyCodeObject *code = (PyCodeObject *)func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL);
DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL);
}
// _INIT_CALL_PY_EXACT_ARGS
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
int has_self = (self_or_null != NULL);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable;
new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
PyObject **first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
first_non_self_local[i] = args[i];
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
stack_pointer += -2 - oparg;
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_PY_GENERAL) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *callable;
PyObject **args;
PyObject *self_or_null;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, CALL);
}
// _CHECK_FUNCTION_VERSION
callable = stack_pointer[-2 - oparg];
{
uint32_t func_version = read_u32(&this_instr[2].cache);
DEOPT_IF(!PyFunction_Check(callable), CALL);
PyFunctionObject *func = (PyFunctionObject *)callable;
DEOPT_IF(func->func_version != func_version, CALL);
}
// _PY_FRAME_GENERAL
args = &stack_pointer[-oparg];
self_or_null = stack_pointer[-1 - oparg];
{
// oparg counts all of the args, but *not* self:
int total_args = oparg;
if (self_or_null != NULL) {
args--;
total_args++;
}
assert(Py_TYPE(callable) == &PyFunction_Type);
int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)callable, locals,
args, total_args, NULL
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
stack_pointer += -2 - oparg;
if (new_frame == NULL) {
goto error;
}
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_STR_1) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_STR_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *arg;
PyObject *null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_STR_1
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
{
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
STAT_INC(CALL, hit);
res = PyObject_Str(arg);
Py_DECREF(arg);
if (res == NULL) goto pop_3_error;
}
// _CHECK_PERIODIC
{
}
stack_pointer[-3] = res;
stack_pointer += -2;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_TUPLE_1) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TUPLE_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *arg;
PyObject *null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
// _CALL_TUPLE_1
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
{
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL);
STAT_INC(CALL, hit);
res = PySequence_Tuple(arg);
Py_DECREF(arg);
if (res == NULL) goto pop_3_error;
}
// _CHECK_PERIODIC
{
}
stack_pointer[-3] = res;
stack_pointer += -2;
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(CALL_TYPE_1) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(CALL_TYPE_1);
static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
PyObject *arg;
PyObject *null;
PyObject *callable;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
arg = stack_pointer[-1];
null = stack_pointer[-2];
callable = stack_pointer[-3];
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
STAT_INC(CALL, hit);
res = Py_NewRef(Py_TYPE(arg));
Py_DECREF(arg);
stack_pointer[-3] = res;
stack_pointer += -2;
DISPATCH();
}
TARGET(CHECK_EG_MATCH) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EG_MATCH);
PyObject *match_type;
PyObject *exc_value;
PyObject *rest;
PyObject *match;
match_type = stack_pointer[-1];
exc_value = stack_pointer[-2];
if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (true) goto pop_2_error;
}
match = NULL;
rest = NULL;
int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
&match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (res < 0) goto pop_2_error;
assert((match == NULL) == (rest == NULL));
if (match == NULL) goto pop_2_error;
if (!Py_IsNone(match)) {
PyErr_SetHandledException(match);
}
stack_pointer[-2] = rest;
stack_pointer[-1] = match;
DISPATCH();
}
TARGET(CHECK_EXC_MATCH) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CHECK_EXC_MATCH);
PyObject *right;
PyObject *left;
PyObject *b;
right = stack_pointer[-1];
left = stack_pointer[-2];
assert(PyExceptionInstance_Check(left));
if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
Py_DECREF(right);
if (true) goto pop_1_error;
}
int res = PyErr_GivenExceptionMatches(left, right);
Py_DECREF(right);
b = res ? Py_True : Py_False;
stack_pointer[-1] = b;
DISPATCH();
}
TARGET(CLEANUP_THROW) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(CLEANUP_THROW);
PyObject *exc_value;
PyObject *last_sent_val;
PyObject *sub_iter;
PyObject *none;
PyObject *value;
exc_value = stack_pointer[-1];
last_sent_val = stack_pointer[-2];
sub_iter = stack_pointer[-3];
assert(throwflag);
assert(exc_value && PyExceptionInstance_Check(exc_value));
if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) {
value = Py_NewRef(((PyStopIterationObject *)exc_value)->value);
Py_DECREF(sub_iter);
Py_DECREF(last_sent_val);
Py_DECREF(exc_value);
none = Py_None;
}
else {
_PyErr_SetRaisedException(tstate, Py_NewRef(exc_value));
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
stack_pointer[-3] = none;
stack_pointer[-2] = value;
stack_pointer += -1;
DISPATCH();
}
TARGET(COMPARE_OP) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP);
PREDICTED(COMPARE_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *right;
PyObject *left;
PyObject *res;
// _SPECIALIZE_COMPARE_OP
right = stack_pointer[-1];
left = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(COMPARE_OP, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _COMPARE_OP
{
assert((oparg >> 5) <= Py_GE);
res = PyObject_RichCompare(left, right, oparg >> 5);
Py_DECREF(left);
Py_DECREF(right);
if (res == NULL) goto pop_2_error;
if (oparg & 16) {
int res_bool = PyObject_IsTrue(res);
Py_DECREF(res);
if (res_bool < 0) goto pop_2_error;
res = res_bool ? Py_True : Py_False;
}
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(COMPARE_OP_FLOAT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_FLOAT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_FLOAT
{
STAT_INC(COMPARE_OP, hit);
double dleft = PyFloat_AS_DOUBLE(left);
double dright = PyFloat_AS_DOUBLE(right);
// 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg
int sign_ish = COMPARISON_BIT(dleft, dright);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(COMPARE_OP_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_INT
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_INT
{
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP);
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 &&
_PyLong_DigitCount((PyLongObject *)right) <= 1);
Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left);
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(COMPARE_OP_STR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *res;
// _GUARD_BOTH_UNICODE
right = stack_pointer[-1];
left = stack_pointer[-2];
{
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
}
/* Skip 1 cache entry */
// _COMPARE_OP_STR
{
STAT_INC(COMPARE_OP, hit);
int eq = _PyUnicode_Equal(left, right);
assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(eq == 0 || eq == 1);
assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS);
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
}
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(CONTAINS_OP) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP);
PREDICTED(CONTAINS_OP);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *right;
PyObject *left;
PyObject *b;
// _SPECIALIZE_CONTAINS_OP
right = stack_pointer[-1];
left = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ContainsOp(right, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(CONTAINS_OP, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _CONTAINS_OP
{
int res = PySequence_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? Py_True : Py_False;
}
stack_pointer[-2] = b;
stack_pointer += -1;
DISPATCH();
}
TARGET(CONTAINS_OP_DICT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_DICT);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *b;
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP);
STAT_INC(CONTAINS_OP, hit);
int res = PyDict_Contains(right, left);
Py_DECREF(left);
Py_DECREF(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
DISPATCH();
}
TARGET(CONTAINS_OP_SET) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CONTAINS_OP_SET);
static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size");
PyObject *right;
PyObject *left;
PyObject *b;
/* Skip 1 cache entry */
right = stack_pointer[-1];
left = stack_pointer[-2];
DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP);
STAT_INC(CONTAINS_OP, hit);
// Note: both set and frozenset use the same seq_contains method!
int res = _PySet_Contains((PySetObject *)right, left);
Py_DECREF(left);
Py_DECREF(right);
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
DISPATCH();
}
TARGET(CONVERT_VALUE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CONVERT_VALUE);
PyObject *value;
PyObject *result;
value = stack_pointer[-1];
conversion_func conv_fn;
assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
conv_fn = _PyEval_ConversionFuncs[oparg];
result = conv_fn(value);
Py_DECREF(value);
if (result == NULL) goto pop_1_error;
stack_pointer[-1] = result;
DISPATCH();
}
TARGET(COPY) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY);
PyObject *bottom;
PyObject *top;
bottom = stack_pointer[-1 - (oparg-1)];
assert(oparg > 0);
top = Py_NewRef(bottom);
stack_pointer[0] = top;
stack_pointer += 1;
DISPATCH();
}
TARGET(COPY_FREE_VARS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(COPY_FREE_VARS);
/* Copy closure variables to free variables */
PyCodeObject *co = _PyFrame_GetCode(frame);
assert(PyFunction_Check(frame->f_funcobj));
PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure;
assert(oparg == co->co_nfreevars);
int offset = co->co_nlocalsplus - oparg;
for (int i = 0; i < oparg; ++i) {
PyObject *o = PyTuple_GET_ITEM(closure, i);
frame->localsplus[offset + i] = Py_NewRef(o);
}
DISPATCH();
}
TARGET(DELETE_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_ATTR);
PyObject *owner;
owner = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_DelAttr(owner, name);
Py_DECREF(owner);
if (err) goto pop_1_error;
stack_pointer += -1;
DISPATCH();
}
TARGET(DELETE_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_DEREF);
PyObject *cell = GETLOCAL(oparg);
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL);
if (oldobj == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
Py_DECREF(oldobj);
DISPATCH();
}
TARGET(DELETE_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_FAST);
PyObject *v = GETLOCAL(oparg);
if (v == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
SETLOCAL(oparg, NULL);
DISPATCH();
}
TARGET(DELETE_GLOBAL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_GLOBAL);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_Pop(GLOBALS(), name, NULL);
// Can't use ERROR_IF here.
if (err < 0) {
goto error;
}
if (err == 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
}
DISPATCH();
}
TARGET(DELETE_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_NAME);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
int err;
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals when deleting %R", name);
goto error;
}
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
if (err != 0) {
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG,
name);
goto error;
}
DISPATCH();
}
TARGET(DELETE_SUBSCR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DELETE_SUBSCR);
PyObject *sub;
PyObject *container;
sub = stack_pointer[-1];
container = stack_pointer[-2];
/* del container[sub] */
int err = PyObject_DelItem(container, sub);
Py_DECREF(container);
Py_DECREF(sub);
if (err) goto pop_2_error;
stack_pointer += -2;
DISPATCH();
}
TARGET(DICT_MERGE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_MERGE);
PyObject *update;
PyObject *dict;
PyObject *callable;
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
callable = stack_pointer[-5 - (oparg - 1)];
if (_PyDict_MergeEx(dict, update, 2) < 0) {
_PyEval_FormatKwargsError(tstate, callable, update);
Py_DECREF(update);
if (true) goto pop_1_error;
}
Py_DECREF(update);
stack_pointer += -1;
DISPATCH();
}
TARGET(DICT_UPDATE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(DICT_UPDATE);
PyObject *update;
PyObject *dict;
update = stack_pointer[-1];
dict = stack_pointer[-2 - (oparg - 1)];
if (PyDict_Update(dict, update) < 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.200s' object is not a mapping",
Py_TYPE(update)->tp_name);
}
Py_DECREF(update);
if (true) goto pop_1_error;
}
Py_DECREF(update);
stack_pointer += -1;
DISPATCH();
}
TARGET(END_ASYNC_FOR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(END_ASYNC_FOR);
PyObject *exc;
PyObject *awaitable;
exc = stack_pointer[-1];
awaitable = stack_pointer[-2];
assert(exc && PyExceptionInstance_Check(exc));
if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) {
Py_DECREF(awaitable);
Py_DECREF(exc);
}
else {
Py_INCREF(exc);
_PyErr_SetRaisedException(tstate, exc);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
stack_pointer += -2;
DISPATCH();
}
TARGET(END_FOR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_FOR);
PyObject *value;
value = stack_pointer[-1];
Py_DECREF(value);
stack_pointer += -1;
DISPATCH();
}
TARGET(END_SEND) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(END_SEND);
PyObject *value;
PyObject *receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
Py_DECREF(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
DISPATCH();
}
TARGET(ENTER_EXECUTOR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(ENTER_EXECUTOR);
#ifdef _Py_TIER2
PyCodeObject *code = _PyFrame_GetCode(frame);
_PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
assert(executor->vm_data.index == INSTR_OFFSET() - 1);
assert(executor->vm_data.code == code);
assert(executor->vm_data.valid);
assert(tstate->previous_executor == NULL);
/* If the eval breaker is set then stay in tier 1.
* This avoids any potentially infinite loops
* involving _RESUME_CHECK */
if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
opcode = executor->vm_data.opcode;
oparg = (oparg & ~255) | executor->vm_data.oparg;
next_instr = this_instr;
if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) {
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
DISPATCH_GOTO();
}
tstate->previous_executor = Py_None;
Py_INCREF(executor);
GOTO_TIER_TWO(executor);
#else
Py_FatalError("ENTER_EXECUTOR is not supported in this build");
#endif /* _Py_TIER2 */
DISPATCH();
}
TARGET(EXIT_INIT_CHECK) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXIT_INIT_CHECK);
PyObject *should_be_none;
should_be_none = stack_pointer[-1];
assert(STACK_LEVEL() == 2);
if (should_be_none != Py_None) {
PyErr_Format(PyExc_TypeError,
"__init__() should return None, not '%.200s'",
Py_TYPE(should_be_none)->tp_name);
goto error;
}
stack_pointer += -1;
DISPATCH();
}
TARGET(EXTENDED_ARG) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXTENDED_ARG);
assert(oparg);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
PRE_DISPATCH_GOTO();
DISPATCH_GOTO();
}
TARGET(FORMAT_SIMPLE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_SIMPLE);
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
/* If value is a unicode object, then we know the result
* of format(value) is value itself. */
if (!PyUnicode_CheckExact(value)) {
res = PyObject_Format(value, NULL);
Py_DECREF(value);
if (res == NULL) goto pop_1_error;
}
else {
res = value;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(FORMAT_WITH_SPEC) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(FORMAT_WITH_SPEC);
PyObject *fmt_spec;
PyObject *value;
PyObject *res;
fmt_spec = stack_pointer[-1];
value = stack_pointer[-2];
res = PyObject_Format(value, fmt_spec);
Py_DECREF(value);
Py_DECREF(fmt_spec);
if (res == NULL) goto pop_2_error;
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(FOR_ITER) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER);
PREDICTED(FOR_ITER);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *iter;
PyObject *next;
// _SPECIALIZE_FOR_ITER
iter = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ForIter(iter, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(FOR_ITER, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _FOR_ITER
{
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
next = (*Py_TYPE(iter)->tp_iternext)(iter);
if (next == NULL) {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */
JUMPBY(oparg + 2);
DISPATCH();
}
// Common case: no jump, leave it to the code generator
}
stack_pointer[0] = next;
stack_pointer += 1;
DISPATCH();
}
TARGET(FOR_ITER_GEN) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_GEN);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
_PyInterpreterFrame *gen_frame;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, FOR_ITER);
}
// _FOR_ITER_GEN_FRAME
iter = stack_pointer[-1];
{
PyGenObject *gen = (PyGenObject *)iter;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
STAT_INC(FOR_ITER, hit);
gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
_PyFrame_StackPush(gen_frame, Py_None);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
}
// _PUSH_FRAME
new_frame = gen_frame;
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(FOR_ITER_LIST) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_LIST);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
/* Skip 1 cache entry */
// _ITER_CHECK_LIST
iter = stack_pointer[-1];
{
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
}
// _ITER_JUMP_LIST
{
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
STAT_INC(FOR_ITER, hit);
PyListObject *seq = it->it_seq;
if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) {
it->it_index = -1;
#ifndef Py_GIL_DISABLED
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
#endif
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
DISPATCH();
}
}
// _ITER_NEXT_LIST
{
_PyListIterObject *it = (_PyListIterObject *)iter;
assert(Py_TYPE(iter) == &PyListIter_Type);
PyListObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyList_GET_SIZE(seq));
next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++));
}
stack_pointer[0] = next;
stack_pointer += 1;
DISPATCH();
}
TARGET(FOR_ITER_RANGE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_RANGE);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
/* Skip 1 cache entry */
// _ITER_CHECK_RANGE
iter = stack_pointer[-1];
{
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
}
// _ITER_JUMP_RANGE
{
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
assert(Py_TYPE(r) == &PyRangeIter_Type);
STAT_INC(FOR_ITER, hit);
if (r->len <= 0) {
STACK_SHRINK(1);
Py_DECREF(r);
// Jump over END_FOR and POP_TOP instructions.
JUMPBY(oparg + 2);
DISPATCH();
}
}
// _ITER_NEXT_RANGE
{
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
assert(Py_TYPE(r) == &PyRangeIter_Type);
assert(r->len > 0);
long value = r->start;
r->start = value + r->step;
r->len--;
next = PyLong_FromLong(value);
if (next == NULL) goto error;
}
stack_pointer[0] = next;
stack_pointer += 1;
DISPATCH();
}
TARGET(FOR_ITER_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(FOR_ITER_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
PyObject *iter;
PyObject *next;
/* Skip 1 cache entry */
// _ITER_CHECK_TUPLE
iter = stack_pointer[-1];
{
DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER);
}
// _ITER_JUMP_TUPLE
{
_PyTupleIterObject *it = (_PyTupleIterObject *)iter;
assert(Py_TYPE(iter) == &PyTupleIter_Type);
STAT_INC(FOR_ITER, hit);
PyTupleObject *seq = it->it_seq;
if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) {
if (seq != NULL) {
it->it_seq = NULL;
Py_DECREF(seq);
}
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */
JUMPBY(oparg + 2);
DISPATCH();
}
}
// _ITER_NEXT_TUPLE
{
_PyTupleIterObject *it = (_PyTupleIterObject *)iter;
assert(Py_TYPE(iter) == &PyTupleIter_Type);
PyTupleObject *seq = it->it_seq;
assert(seq);
assert(it->it_index < PyTuple_GET_SIZE(seq));
next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++));
}
stack_pointer[0] = next;
stack_pointer += 1;
DISPATCH();
}
TARGET(GET_AITER) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AITER);
PyObject *obj;
PyObject *iter;
obj = stack_pointer[-1];
unaryfunc getter = NULL;
PyTypeObject *type = Py_TYPE(obj);
if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
}
if (getter == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
Py_DECREF(obj);
if (true) goto pop_1_error;
}
iter = (*getter)(obj);
Py_DECREF(obj);
if (iter == NULL) goto pop_1_error;
if (Py_TYPE(iter)->tp_as_async == NULL ||
Py_TYPE(iter)->tp_as_async->am_anext == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' received an object from __aiter__ "
"that does not implement __anext__: %.100s",
Py_TYPE(iter)->tp_name);
Py_DECREF(iter);
if (true) goto pop_1_error;
}
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(GET_ANEXT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ANEXT);
PyObject *aiter;
PyObject *awaitable;
aiter = stack_pointer[-1];
unaryfunc getter = NULL;
PyObject *next_iter = NULL;
PyTypeObject *type = Py_TYPE(aiter);
if (PyAsyncGen_CheckExact(aiter)) {
awaitable = type->tp_as_async->am_anext(aiter);
if (awaitable == NULL) {
goto error;
}
} else {
if (type->tp_as_async != NULL){
getter = type->tp_as_async->am_anext;
}
if (getter != NULL) {
next_iter = (*getter)(aiter);
if (next_iter == NULL) {
goto error;
}
}
else {
_PyErr_Format(tstate, PyExc_TypeError,
"'async for' requires an iterator with "
"__anext__ method, got %.100s",
type->tp_name);
goto error;
}
awaitable = _PyCoro_GetAwaitableIter(next_iter);
if (awaitable == NULL) {
_PyErr_FormatFromCause(
PyExc_TypeError,
"'async for' received an invalid object "
"from __anext__: %.100s",
Py_TYPE(next_iter)->tp_name);
Py_DECREF(next_iter);
goto error;
} else {
Py_DECREF(next_iter);
}
}
stack_pointer[0] = awaitable;
stack_pointer += 1;
DISPATCH();
}
TARGET(GET_AWAITABLE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_AWAITABLE);
PyObject *iterable;
PyObject *iter;
iterable = stack_pointer[-1];
iter = _PyCoro_GetAwaitableIter(iterable);
if (iter == NULL) {
_PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
}
Py_DECREF(iterable);
if (iter != NULL && PyCoro_CheckExact(iter)) {
PyObject *yf = _PyGen_yf((PyGenObject*)iter);
if (yf != NULL) {
/* `iter` is a coroutine object that is being
awaited, `yf` is a pointer to the current awaitable
being awaited on. */
Py_DECREF(yf);
Py_CLEAR(iter);
_PyErr_SetString(tstate, PyExc_RuntimeError,
"coroutine is being awaited already");
/* The code below jumps to `error` if `iter` is NULL. */
}
}
if (iter == NULL) goto pop_1_error;
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(GET_ITER) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_ITER);
PyObject *iterable;
PyObject *iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
iter = PyObject_GetIter(iterable);
Py_DECREF(iterable);
if (iter == NULL) goto pop_1_error;
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(GET_LEN) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_LEN);
PyObject *obj;
PyObject *len_o;
obj = stack_pointer[-1];
// PUSH(len(TOS))
Py_ssize_t len_i = PyObject_Length(obj);
if (len_i < 0) goto error;
len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) goto error;
stack_pointer[0] = len_o;
stack_pointer += 1;
DISPATCH();
}
TARGET(GET_YIELD_FROM_ITER) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(GET_YIELD_FROM_ITER);
PyObject *iterable;
PyObject *iter;
iterable = stack_pointer[-1];
/* before: [obj]; after [getiter(obj)] */
if (PyCoro_CheckExact(iterable)) {
/* `iterable` is a coroutine */
if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
/* and it is used in a 'yield from' expression of a
regular generator. */
_PyErr_SetString(tstate, PyExc_TypeError,
"cannot 'yield from' a coroutine object "
"in a non-coroutine generator");
goto error;
}
iter = iterable;
}
else if (PyGen_CheckExact(iterable)) {
iter = iterable;
}
else {
/* `iterable` is not a generator. */
iter = PyObject_GetIter(iterable);
if (iter == NULL) {
goto error;
}
Py_DECREF(iterable);
}
stack_pointer[-1] = iter;
DISPATCH();
}
TARGET(IMPORT_FROM) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_FROM);
PyObject *from;
PyObject *res;
from = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
res = import_from(tstate, from, name);
if (res == NULL) goto error;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(IMPORT_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IMPORT_NAME);
PyObject *fromlist;
PyObject *level;
PyObject *res;
fromlist = stack_pointer[-1];
level = stack_pointer[-2];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
res = import_name(tstate, frame, name, fromlist, level);
Py_DECREF(level);
Py_DECREF(fromlist);
if (res == NULL) goto pop_2_error;
stack_pointer[-2] = res;
stack_pointer += -1;
DISPATCH();
}
TARGET(INSTRUMENTED_CALL) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 4;
INSTRUCTION_STATS(INSTRUMENTED_CALL);
/* Skip 3 cache entries */
int is_meth = PEEK(oparg + 1) != NULL;
int total_args = oparg + is_meth;
PyObject *function = PEEK(oparg + 2);
PyObject *arg = total_args == 0 ?
&_PyInstrumentation_MISSING : PEEK(total_args);
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
if (err) goto error;
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(CALL);
}
TARGET(INSTRUMENTED_CALL_FUNCTION_EX) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
}
TARGET(INSTRUMENTED_CALL_KW) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_CALL_KW);
int is_meth = PEEK(oparg + 2) != NULL;
int total_args = oparg + is_meth;
PyObject *function = PEEK(oparg + 3);
PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING
: PEEK(total_args + 1);
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg);
if (err) goto error;
GO_TO_INSTRUCTION(CALL_KW);
}
TARGET(INSTRUMENTED_END_FOR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_FOR);
PyObject *value;
PyObject *receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
/* Need to create a fake StopIteration error here,
* to conform to PEP 380 */
if (PyGen_Check(receiver)) {
if (monitor_stop_iteration(tstate, frame, this_instr, value)) {
goto error;
}
}
Py_DECREF(value);
stack_pointer += -1;
DISPATCH();
}
TARGET(INSTRUMENTED_END_SEND) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_END_SEND);
PyObject *value;
PyObject *receiver;
value = stack_pointer[-1];
receiver = stack_pointer[-2];
if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
if (monitor_stop_iteration(tstate, frame, this_instr, value)) {
goto error;
}
}
Py_DECREF(receiver);
stack_pointer[-2] = value;
stack_pointer += -1;
DISPATCH();
}
TARGET(INSTRUMENTED_FOR_ITER) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER);
/* Skip 1 cache entry */
_Py_CODEUNIT *target;
PyObject *iter = TOP();
PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
if (next != NULL) {
PUSH(next);
target = next_instr;
}
else {
if (_PyErr_Occurred(tstate)) {
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
goto error;
}
monitor_raise(tstate, frame, this_instr);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
assert(next_instr[oparg].op.code == END_FOR ||
next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
STACK_SHRINK(1);
Py_DECREF(iter);
/* Skip END_FOR and POP_TOP */
target = next_instr + oparg + 2;
}
INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_INSTRUCTION) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION);
int next_opcode = _Py_call_instrumentation_instruction(
tstate, frame, this_instr);
if (next_opcode < 0) goto error;
next_instr = this_instr;
if (_PyOpcode_Caches[next_opcode]) {
PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter);
}
assert(next_opcode > 0 && next_opcode < 256);
opcode = next_opcode;
DISPATCH_GOTO();
}
TARGET(INSTRUMENTED_JUMP_BACKWARD) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD);
/* Skip 1 cache entry */
CHECK_EVAL_BREAKER();
INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP);
DISPATCH();
}
TARGET(INSTRUMENTED_JUMP_FORWARD) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD);
INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP);
DISPATCH();
}
TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR);
/* Skip 1 cache entry */
// cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
// don't want to specialize instrumented instructions
PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
}
TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE);
/* Skip 1 cache entry */
PyObject *cond = POP();
assert(PyBool_Check(cond));
int flag = Py_IsFalse(cond);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE);
/* Skip 1 cache entry */
PyObject *value = POP();
int flag = Py_IsNone(value);
int offset;
if (flag) {
offset = oparg;
}
else {
Py_DECREF(value);
offset = 0;
}
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE);
/* Skip 1 cache entry */
PyObject *value = POP();
int offset;
int nflag = Py_IsNone(value);
if (nflag) {
offset = 0;
}
else {
Py_DECREF(value);
offset = oparg;
}
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | !nflag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE);
/* Skip 1 cache entry */
PyObject *cond = POP();
assert(PyBool_Check(cond));
int flag = Py_IsTrue(cond);
int offset = flag * oparg;
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
DISPATCH();
}
TARGET(INSTRUMENTED_RESUME) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RESUME);
uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
if (code_version != global_version && tstate->tracing == 0) {
if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {
goto error;
}
next_instr = this_instr;
}
else {
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
CHECK_EVAL_BREAKER();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_call_instrumentation(
tstate, oparg > 0, frame, this_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err) goto error;
if (frame->instr_ptr != this_instr) {
/* Instrumentation has jumped */
next_instr = frame->instr_ptr;
DISPATCH();
}
}
DISPATCH();
}
TARGET(INSTRUMENTED_RETURN_CONST) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST);
PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) goto error;
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_LeaveRecursiveCallPy(tstate);
assert(frame != &entry_frame);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, retval);
LOAD_IP(frame->return_offset);
goto resume_frame;
}
TARGET(INSTRUMENTED_RETURN_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE);
PyObject *retval;
retval = stack_pointer[-1];
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_RETURN,
frame, this_instr, retval);
if (err) goto error;
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_LeaveRecursiveCallPy(tstate);
assert(frame != &entry_frame);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, retval);
LOAD_IP(frame->return_offset);
goto resume_frame;
}
TARGET(INSTRUMENTED_YIELD_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE);
PyObject *retval;
retval = stack_pointer[-1];
assert(frame != &entry_frame);
frame->instr_ptr = next_instr;
PyGenObject *gen = _PyFrame_GetGenerator(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
gen->gi_frame_state = FRAME_SUSPENDED + oparg;
_PyFrame_SetStackPointer(frame, stack_pointer - 1);
int err = _Py_call_instrumentation_arg(
tstate, PY_MONITORING_EVENT_PY_YIELD,
frame, this_instr, retval);
if (err) goto error;
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
_PyFrame_StackPush(frame, retval);
/* We don't know which of these is relevant here, so keep them equal */
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
goto resume_frame;
}
TARGET(INTERPRETER_EXIT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(INTERPRETER_EXIT);
PyObject *retval;
retval = stack_pointer[-1];
assert(frame == &entry_frame);
assert(_PyFrame_IsIncomplete(frame));
/* Restore previous frame and return. */
tstate->current_frame = frame->previous;
assert(!_PyErr_Occurred(tstate));
tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;
return retval;
}
TARGET(IS_OP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(IS_OP);
PyObject *right;
PyObject *left;
PyObject *b;
right = stack_pointer[-1];
left = stack_pointer[-2];
int res = Py_Is(left, right) ^ oparg;
Py_DECREF(left);
Py_DECREF(right);
b = res ? Py_True : Py_False;
stack_pointer[-2] = b;
stack_pointer += -1;
DISPATCH();
}
TARGET(JUMP_BACKWARD) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(JUMP_BACKWARD);
/* Skip 1 cache entry */
CHECK_EVAL_BREAKER();
assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg);
#ifdef _Py_TIER2
#if ENABLE_SPECIALIZATION
_Py_BackoffCounter counter = this_instr[1].counter;
if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
_Py_CODEUNIT *start = this_instr;
/* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
while (oparg > 255) {
oparg >>= 8;
start--;
}
_PyExecutorObject *executor;
int optimized = _PyOptimizer_Optimize(frame, start, stack_pointer, &executor);
if (optimized < 0) goto error;
if (optimized) {
assert(tstate->previous_executor == NULL);
tstate->previous_executor = Py_None;
GOTO_TIER_TWO(executor);
}
else {
this_instr[1].counter = restart_backoff_counter(counter);
}
}
else {
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
}
#endif /* ENABLE_SPECIALIZATION */
#endif /* _Py_TIER2 */
DISPATCH();
}
TARGET(JUMP_BACKWARD_NO_INTERRUPT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT);
/* This bytecode is used in the `yield from` or `await` loop.
* If there is an interrupt, we want it handled in the innermost
* generator or coroutine, so we deliberately do not check it here.
* (see bpo-30039).
*/
JUMPBY(-oparg);
DISPATCH();
}
TARGET(JUMP_FORWARD) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(JUMP_FORWARD);
JUMPBY(oparg);
DISPATCH();
}
TARGET(LIST_APPEND) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_APPEND);
PyObject *v;
PyObject *list;
v = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
stack_pointer += -1;
DISPATCH();
}
TARGET(LIST_EXTEND) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LIST_EXTEND);
PyObject *iterable;
PyObject *list;
iterable = stack_pointer[-1];
list = stack_pointer[-2 - (oparg-1)];
PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
if (none_val == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
(Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
{
_PyErr_Clear(tstate);
_PyErr_Format(tstate, PyExc_TypeError,
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
Py_DECREF(iterable);
if (true) goto pop_1_error;
}
assert(Py_IsNone(none_val));
Py_DECREF(iterable);
stack_pointer += -1;
DISPATCH();
}
TARGET(LOAD_ASSERTION_ERROR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_ASSERTION_ERROR);
PyObject *value;
value = Py_NewRef(PyExc_AssertionError);
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR);
PREDICTED(LOAD_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 10;
(void)this_instr;
PyObject *owner;
PyObject *attr;
PyObject *self_or_null = NULL;
// _SPECIALIZE_LOAD_ATTR
owner = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_Py_Specialize_LoadAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_ATTR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 8 cache entries */
// _LOAD_ATTR
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
if (oparg & 1) {
/* Designed to work in tandem with CALL, pushes two values. */
attr = NULL;
if (_PyObject_GetMethod(owner, name, &attr)) {
/* We can bypass temporary bound method object.
meth is unbound method and obj is self.
meth | self | arg1 | ... | argN
*/
assert(attr != NULL); // No errors on this branch
self_or_null = owner; // Transfer ownership
}
else {
/* meth is not an unbound method (but a regular attr, or
something was returned by a descriptor protocol). Set
the second element of the stack to NULL, to signal
CALL that it's not a method call.
meth | NULL | arg1 | ... | argN
*/
Py_DECREF(owner);
if (attr == NULL) goto pop_1_error;
self_or_null = NULL;
}
}
else {
/* Classic, pushes one value. */
attr = PyObject_GetAttr(owner, name);
Py_DECREF(owner);
if (attr == NULL) goto pop_1_error;
}
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = self_or_null;
stack_pointer += (oparg & 1);
DISPATCH();
}
TARGET(LOAD_ATTR_CLASS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_CLASS);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_CLASS
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
DEOPT_IF(!PyType_Check(owner), LOAD_ATTR);
assert(type_version != 0);
DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_CLASS
{
PyObject *descr = read_obj(&this_instr[6].cache);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = Py_NewRef(descr);
null = NULL;
Py_DECREF(owner);
}
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
DISPATCH();
}
TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *getattribute = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(getattribute, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)getattribute;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
Py_INCREF(f);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
new_frame->localsplus[1] = Py_NewRef(name);
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
TARGET(LOAD_ATTR_INSTANCE_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&this_instr[4].cache);
attr = _PyObject_InlineValues(owner)->values[index];
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_LAZY_DICT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *self = NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
/* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = Py_NewRef(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_NO_DICT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *self = NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_METHOD_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
assert(Py_TYPE(owner)->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR));
attr = Py_NewRef(descr);
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_ATTR_METHOD_WITH_VALUES) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *self = NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_METHOD_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert(oparg & 1);
/* Cached method object */
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
attr = Py_NewRef(descr);
assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR));
self = owner;
}
stack_pointer[-1] = attr;
stack_pointer[0] = self;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_ATTR_MODULE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _CHECK_ATTR_MODULE
owner = stack_pointer[-1];
{
uint32_t dict_version = read_u32(&this_instr[2].cache);
DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
DEOPT_IF(dict->ma_keys->dk_version != dict_version, LOAD_ATTR);
}
// _LOAD_ATTR_MODULE
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE);
assert(index < dict->ma_keys->dk_nentries);
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index;
attr = ep->me_value;
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
DISPATCH();
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_NONDESCRIPTOR_NO_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
assert(Py_TYPE(owner)->tp_dictoffset == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
Py_DECREF(owner);
attr = Py_NewRef(descr);
}
stack_pointer[-1] = attr;
DISPATCH();
}
TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
uint32_t keys_version = read_u32(&this_instr[4].cache);
PyTypeObject *owner_cls = Py_TYPE(owner);
PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls;
DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR);
}
// _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES
{
PyObject *descr = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
STAT_INC(LOAD_ATTR, hit);
assert(descr != NULL);
Py_DECREF(owner);
attr = Py_NewRef(descr);
}
stack_pointer[-1] = attr;
DISPATCH();
}
TARGET(LOAD_ATTR_PROPERTY) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *fget = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
TARGET(LOAD_ATTR_SLOT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _LOAD_ATTR_SLOT
{
uint16_t index = read_u16(&this_instr[4].cache);
char *addr = (char *)owner + index;
attr = *(PyObject **)addr;
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
DISPATCH();
}
TARGET(LOAD_ATTR_WITH_HINT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 10;
INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
PyObject *owner;
PyObject *attr;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
// _CHECK_ATTR_WITH_HINT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(dict == NULL, LOAD_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
attr = ep->me_value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, LOAD_ATTR);
attr = ep->me_value;
}
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
null = NULL;
Py_DECREF(owner);
}
/* Skip 5 cache entries */
stack_pointer[-1] = attr;
if (oparg & 1) stack_pointer[0] = null;
stack_pointer += (oparg & 1);
DISPATCH();
}
TARGET(LOAD_BUILD_CLASS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_BUILD_CLASS);
PyObject *bc;
if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error;
if (bc == NULL) {
_PyErr_SetString(tstate, PyExc_NameError,
"__build_class__ not found");
if (true) goto error;
}
stack_pointer[0] = bc;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_CONST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_CONST);
PyObject *value;
value = GETITEM(FRAME_CO_CONSTS, oparg);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_DEREF);
PyObject *value;
PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
value = PyCell_GetRef(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) goto error;
}
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST);
PyObject *value;
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_FAST_AND_CLEAR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR);
PyObject *value;
value = GETLOCAL(oparg);
// do not use SETLOCAL here, it decrefs the old value
GETLOCAL(oparg) = NULL;
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_FAST_CHECK) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_CHECK);
PyObject *value;
value = GETLOCAL(oparg);
if (value == NULL) {
_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
if (1) goto error;
}
Py_INCREF(value);
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_FAST_LOAD_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST);
PyObject *value1;
PyObject *value2;
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
value1 = GETLOCAL(oparg1);
value2 = GETLOCAL(oparg2);
Py_INCREF(value1);
Py_INCREF(value2);
stack_pointer[0] = value1;
stack_pointer[1] = value2;
stack_pointer += 2;
DISPATCH();
}
TARGET(LOAD_FROM_DICT_OR_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF);
PyObject *class_dict;
PyObject *value;
class_dict = stack_pointer[-1];
PyObject *name;
assert(class_dict);
assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus);
name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg);
if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) {
goto error;
}
if (!value) {
PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
value = PyCell_GetRef(cell);
if (value == NULL) {
_PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
}
Py_DECREF(class_dict);
stack_pointer[-1] = value;
DISPATCH();
}
TARGET(LOAD_FROM_DICT_OR_GLOBALS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS);
PyObject *mod_or_class_dict;
PyObject *v;
mod_or_class_dict = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
goto error;
}
if (v == NULL) {
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
if (v == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
goto error;
}
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto pop_1_error;
if (v == NULL) {
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto pop_1_error;
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) goto pop_1_error;
}
}
}
}
Py_DECREF(mod_or_class_dict);
stack_pointer[-1] = v;
DISPATCH();
}
TARGET(LOAD_GLOBAL) {
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL);
PREDICTED(LOAD_GLOBAL);
_Py_CODEUNIT *this_instr = next_instr - 5;
(void)this_instr;
PyObject *res;
PyObject *null = NULL;
// _SPECIALIZE_LOAD_GLOBAL
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_GLOBAL, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 1 cache entry */
/* Skip 1 cache entry */
/* Skip 1 cache entry */
// _LOAD_GLOBAL
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (PyDict_CheckExact(GLOBALS())
&& PyDict_CheckExact(BUILTINS()))
{
res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
(PyDictObject *)BUILTINS(),
name);
if (res == NULL) {
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
_PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
}
if (true) goto error;
}
}
else {
/* Slow-path if globals or builtins is not a dict */
/* namespace 1: globals */
if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error;
if (res == NULL) {
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error;
if (res == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) goto error;
}
}
}
null = NULL;
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
DISPATCH();
}
TARGET(LOAD_GLOBAL_BUILTIN) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyObject *res;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _GUARD_BUILTINS_VERSION
{
uint16_t version = read_u16(&this_instr[3].cache);
PyDictObject *dict = (PyDictObject *)BUILTINS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
// _LOAD_GLOBAL_BUILTINS
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *bdict = (PyDictObject *)BUILTINS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
DISPATCH();
}
TARGET(LOAD_GLOBAL_MODULE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(LOAD_GLOBAL_MODULE);
static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
PyObject *res;
PyObject *null = NULL;
/* Skip 1 cache entry */
// _GUARD_GLOBALS_VERSION
{
uint16_t version = read_u16(&this_instr[2].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL);
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(dict->ma_keys));
}
/* Skip 1 cache entry */
// _LOAD_GLOBAL_MODULE
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictObject *dict = (PyDictObject *)GLOBALS();
PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys);
res = entries[index].me_value;
DEOPT_IF(res == NULL, LOAD_GLOBAL);
Py_INCREF(res);
STAT_INC(LOAD_GLOBAL, hit);
null = NULL;
}
stack_pointer[0] = res;
if (oparg & 1) stack_pointer[1] = null;
stack_pointer += 1 + (oparg & 1);
DISPATCH();
}
TARGET(LOAD_LOCALS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_LOCALS);
PyObject *locals;
locals = LOCALS();
if (locals == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) goto error;
}
Py_INCREF(locals);
stack_pointer[0] = locals;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(LOAD_NAME);
PyObject *v;
PyObject *mod_or_class_dict = LOCALS();
if (mod_or_class_dict == NULL) {
_PyErr_SetString(tstate, PyExc_SystemError,
"no locals found");
if (true) goto error;
}
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) {
goto error;
}
if (v == NULL) {
if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) {
goto error;
}
if (v == NULL) {
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) {
goto error;
}
if (v == NULL) {
_PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
}
}
}
stack_pointer[0] = v;
stack_pointer += 1;
DISPATCH();
}
TARGET(LOAD_SUPER_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR);
PREDICTED(LOAD_SUPER_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *class;
PyObject *global_super;
PyObject *self;
PyObject *attr;
PyObject *null = NULL;
// _SPECIALIZE_LOAD_SUPER_ATTR
class = stack_pointer[-2];
global_super = stack_pointer[-3];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
DISPATCH_SAME_OPARG();
}
STAT_INC(LOAD_SUPER_ATTR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _LOAD_SUPER_ATTR
self = stack_pointer[-1];
{
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, global_super, arg);
if (err) goto pop_3_error;
}
// we make no attempt to optimize here; specializations should
// handle any case whose performance we care about
PyObject *stack[] = {class, self};
PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
if (super == NULL) {
_Py_call_instrumentation_exc2(
tstate, PY_MONITORING_EVENT_C_RAISE,
frame, this_instr, global_super, arg);
}
else {
int err = _Py_call_instrumentation_2args(
tstate, PY_MONITORING_EVENT_C_RETURN,
frame, this_instr, global_super, arg);
if (err < 0) {
Py_CLEAR(super);
}
}
}
Py_DECREF(global_super);
Py_DECREF(class);
Py_DECREF(self);
if (super == NULL) goto pop_3_error;
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
attr = PyObject_GetAttr(super, name);
Py_DECREF(super);
if (attr == NULL) goto pop_3_error;
null = NULL;
}
stack_pointer[-3] = attr;
if (oparg & 1) stack_pointer[-2] = null;
stack_pointer += -2 + (oparg & 1);
DISPATCH();
}
TARGET(LOAD_SUPER_ATTR_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
PyObject *self;
PyObject *class;
PyObject *global_super;
PyObject *attr;
/* Skip 1 cache entry */
self = stack_pointer[-1];
class = stack_pointer[-2];
global_super = stack_pointer[-3];
assert(!(oparg & 1));
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
Py_DECREF(global_super);
Py_DECREF(class);
Py_DECREF(self);
if (attr == NULL) goto pop_3_error;
stack_pointer[-3] = attr;
stack_pointer += -2;
DISPATCH();
}
TARGET(LOAD_SUPER_ATTR_METHOD) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD);
static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
PyObject *self;
PyObject *class;
PyObject *global_super;
PyObject *attr;
PyObject *self_or_null;
/* Skip 1 cache entry */
self = stack_pointer[-1];
class = stack_pointer[-2];
global_super = stack_pointer[-3];
assert(oparg & 1);
DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR);
DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR);
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
PyTypeObject *cls = (PyTypeObject *)class;
int method_found = 0;
attr = _PySuper_Lookup(cls, self, name,
Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL);
Py_DECREF(global_super);
Py_DECREF(class);
if (attr == NULL) {
Py_DECREF(self);
if (true) goto pop_3_error;
}
if (method_found) {
self_or_null = self; // transfer ownership
} else {
Py_DECREF(self);
self_or_null = NULL;
}
stack_pointer[-3] = attr;
stack_pointer[-2] = self_or_null;
stack_pointer += -1;
DISPATCH();
}
TARGET(MAKE_CELL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_CELL);
// "initial" is probably NULL but not if it's an arg (or set
// via the f_locals proxy before MAKE_CELL has run).
PyObject *initial = GETLOCAL(oparg);
PyObject *cell = PyCell_New(initial);
if (cell == NULL) {
goto error;
}
SETLOCAL(oparg, cell);
DISPATCH();
}
TARGET(MAKE_FUNCTION) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAKE_FUNCTION);
PyObject *codeobj;
PyObject *func;
codeobj = stack_pointer[-1];
PyFunctionObject *func_obj = (PyFunctionObject *)
PyFunction_New(codeobj, GLOBALS());
Py_DECREF(codeobj);
if (func_obj == NULL) {
goto error;
}
_PyFunction_SetVersion(
func_obj, ((PyCodeObject *)codeobj)->co_version);
func = (PyObject *)func_obj;
stack_pointer[-1] = func;
DISPATCH();
}
TARGET(MAP_ADD) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MAP_ADD);
PyObject *value;
PyObject *key;
PyObject *dict;
value = stack_pointer[-1];
key = stack_pointer[-2];
dict = stack_pointer[-3 - (oparg - 1)];
assert(PyDict_CheckExact(dict));
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
stack_pointer += -2;
DISPATCH();
}
TARGET(MATCH_CLASS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_CLASS);
PyObject *names;
PyObject *type;
PyObject *subject;
PyObject *attrs;
names = stack_pointer[-1];
type = stack_pointer[-2];
subject = stack_pointer[-3];
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
assert(PyTuple_CheckExact(names));
attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
Py_DECREF(subject);
Py_DECREF(type);
Py_DECREF(names);
if (attrs) {
assert(PyTuple_CheckExact(attrs)); // Success!
}
else {
if (_PyErr_Occurred(tstate)) goto pop_3_error;
// Error!
attrs = Py_None; // Failure!
}
stack_pointer[-3] = attrs;
stack_pointer += -2;
DISPATCH();
}
TARGET(MATCH_KEYS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_KEYS);
PyObject *keys;
PyObject *subject;
PyObject *values_or_none;
keys = stack_pointer[-1];
subject = stack_pointer[-2];
// On successful match, PUSH(values). Otherwise, PUSH(None).
values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
if (values_or_none == NULL) goto error;
stack_pointer[0] = values_or_none;
stack_pointer += 1;
DISPATCH();
}
TARGET(MATCH_MAPPING) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_MAPPING);
PyObject *subject;
PyObject *res;
subject = stack_pointer[-1];
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
res = match ? Py_True : Py_False;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(MATCH_SEQUENCE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(MATCH_SEQUENCE);
PyObject *subject;
PyObject *res;
subject = stack_pointer[-1];
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
res = match ? Py_True : Py_False;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(NOP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(NOP);
DISPATCH();
}
TARGET(POP_EXCEPT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_EXCEPT);
PyObject *exc_value;
exc_value = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value);
stack_pointer += -1;
DISPATCH();
}
TARGET(POP_JUMP_IF_FALSE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
PyObject *cond;
/* Skip 1 cache entry */
cond = stack_pointer[-1];
assert(PyBool_Check(cond));
int flag = Py_IsFalse(cond);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
stack_pointer += -1;
DISPATCH();
}
TARGET(POP_JUMP_IF_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NONE);
PyObject *value;
PyObject *b;
PyObject *cond;
/* Skip 1 cache entry */
// _IS_NONE
value = stack_pointer[-1];
{
if (Py_IsNone(value)) {
b = Py_True;
}
else {
b = Py_False;
Py_DECREF(value);
}
}
// _POP_JUMP_IF_TRUE
cond = b;
{
assert(PyBool_Check(cond));
int flag = Py_IsTrue(cond);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
}
stack_pointer += -1;
DISPATCH();
}
TARGET(POP_JUMP_IF_NOT_NONE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE);
PyObject *value;
PyObject *b;
PyObject *cond;
/* Skip 1 cache entry */
// _IS_NONE
value = stack_pointer[-1];
{
if (Py_IsNone(value)) {
b = Py_True;
}
else {
b = Py_False;
Py_DECREF(value);
}
}
// _POP_JUMP_IF_FALSE
cond = b;
{
assert(PyBool_Check(cond));
int flag = Py_IsFalse(cond);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
}
stack_pointer += -1;
DISPATCH();
}
TARGET(POP_JUMP_IF_TRUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 2;
INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
PyObject *cond;
/* Skip 1 cache entry */
cond = stack_pointer[-1];
assert(PyBool_Check(cond));
int flag = Py_IsTrue(cond);
#if ENABLE_SPECIALIZATION
this_instr[1].cache = (this_instr[1].cache << 1) | flag;
#endif
JUMPBY(oparg * flag);
stack_pointer += -1;
DISPATCH();
}
TARGET(POP_TOP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(POP_TOP);
PyObject *value;
value = stack_pointer[-1];
Py_DECREF(value);
stack_pointer += -1;
DISPATCH();
}
TARGET(PUSH_EXC_INFO) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_EXC_INFO);
PyObject *new_exc;
PyObject *prev_exc;
new_exc = stack_pointer[-1];
_PyErr_StackItem *exc_info = tstate->exc_info;
if (exc_info->exc_value != NULL) {
prev_exc = exc_info->exc_value;
}
else {
prev_exc = Py_None;
}
assert(PyExceptionInstance_Check(new_exc));
exc_info->exc_value = Py_NewRef(new_exc);
stack_pointer[-1] = prev_exc;
stack_pointer[0] = new_exc;
stack_pointer += 1;
DISPATCH();
}
TARGET(PUSH_NULL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(PUSH_NULL);
PyObject *res;
res = NULL;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(RAISE_VARARGS) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(RAISE_VARARGS);
PyObject **args;
args = &stack_pointer[-oparg];
PyObject *cause = NULL, *exc = NULL;
switch (oparg) {
case 2:
cause = args[1];
/* fall through */
case 1:
exc = args[0];
/* fall through */
case 0:
if (do_raise(tstate, exc, cause)) {
assert(oparg == 0);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
break;
default:
_PyErr_SetString(tstate, PyExc_SystemError,
"bad RAISE_VARARGS oparg");
break;
}
if (true) { stack_pointer += -oparg; goto error; }
}
TARGET(RERAISE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
(void)this_instr;
next_instr += 1;
INSTRUCTION_STATS(RERAISE);
PyObject *exc;
PyObject **values;
exc = stack_pointer[-1];
values = &stack_pointer[-1 - oparg];
assert(oparg >= 0 && oparg <= 2);
if (oparg) {
PyObject *lasti = values[0];
if (PyLong_Check(lasti)) {
frame->instr_ptr = _PyCode_CODE(_PyFrame_GetCode(frame)) + PyLong_AsLong(lasti);
assert(!_PyErr_Occurred(tstate));
}
else {
assert(PyLong_Check(lasti));
_PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
goto error;
}
}
assert(exc && PyExceptionInstance_Check(exc));
Py_INCREF(exc);
_PyErr_SetRaisedException(tstate, exc);
monitor_reraise(tstate, frame, this_instr);
goto exception_unwind;
}
TARGET(RESERVED) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESERVED);
assert(0 && "Executing RESERVED instruction.");
Py_FatalError("Executing RESERVED instruction.");
DISPATCH();
}
TARGET(RESUME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME);
PREDICTED(RESUME);
_Py_CODEUNIT *this_instr = next_instr - 1;
(void)this_instr;
assert(frame == tstate->current_frame);
if (tstate->tracing == 0) {
uintptr_t global_version =
_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
~_PY_EVAL_EVENTS_MASK;
PyCodeObject* code = _PyFrame_GetCode(frame);
uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);
assert((code_version & 255) == 0);
if (code_version != global_version) {
int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
if (err) goto error;
next_instr = this_instr;
DISPATCH();
}
assert(this_instr->op.code == RESUME ||
this_instr->op.code == RESUME_CHECK ||
this_instr->op.code == INSTRUMENTED_RESUME ||
this_instr->op.code == ENTER_EXECUTOR);
if (this_instr->op.code == RESUME) {
#if ENABLE_SPECIALIZATION
FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
#endif /* ENABLE_SPECIALIZATION */
}
}
if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
CHECK_EVAL_BREAKER();
}
DISPATCH();
}
TARGET(RESUME_CHECK) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESUME_CHECK);
static_assert(0 == 0, "incorrect cache size");
#if defined(__EMSCRIPTEN__)
DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME);
_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
DEOPT_IF(eval_breaker != version, RESUME);
DISPATCH();
}
TARGET(RETURN_CONST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_CONST);
PyObject *value;
PyObject *retval;
// _LOAD_CONST
{
value = GETITEM(FRAME_CO_CONSTS, oparg);
Py_INCREF(value);
}
// _POP_FRAME
retval = value;
{
#if TIER_ONE
assert(frame != &entry_frame);
#endif
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, retval);
LOAD_SP();
LOAD_IP(frame->return_offset);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(RETURN_GENERATOR) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_GENERATOR);
PyObject *res;
assert(PyFunction_Check(frame->f_funcobj));
PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj;
PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func);
if (gen == NULL) {
goto error;
}
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
frame->instr_ptr++;
_PyFrame_Copy(frame, gen_frame);
assert(frame->frame_obj == NULL);
gen->gi_frame_state = FRAME_CREATED;
gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
_Py_LeaveRecursiveCallPy(tstate);
res = (PyObject *)gen;
_PyInterpreterFrame *prev = frame->previous;
_PyThreadState_PopFrame(tstate, frame);
frame = tstate->current_frame = prev;
LOAD_IP(frame->return_offset);
LOAD_SP();
LLTRACE_RESUME_FRAME();
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(RETURN_VALUE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RETURN_VALUE);
PyObject *retval;
retval = stack_pointer[-1];
#if TIER_ONE
assert(frame != &entry_frame);
#endif
stack_pointer += -1;
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(EMPTY());
_Py_LeaveRecursiveCallPy(tstate);
// GH-99729: We need to unlink the frame *before* clearing it:
_PyInterpreterFrame *dying = frame;
frame = tstate->current_frame = dying->previous;
_PyEval_FrameClearAndPop(tstate, dying);
_PyFrame_StackPush(frame, retval);
LOAD_SP();
LOAD_IP(frame->return_offset);
LLTRACE_RESUME_FRAME();
DISPATCH();
}
TARGET(SEND) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND);
PREDICTED(SEND);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *receiver;
PyObject *v;
PyObject *retval;
// _SPECIALIZE_SEND
receiver = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_Send(receiver, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(SEND, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _SEND
v = stack_pointer[-1];
{
assert(frame != &entry_frame);
if ((tstate->interp->eval_frame == NULL) &&
(Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) &&
((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING)
{
PyGenObject *gen = (PyGenObject *)receiver;
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
STACK_SHRINK(1);
_PyFrame_StackPush(gen_frame, v);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
DISPATCH_INLINED(gen_frame);
}
if (Py_IsNone(v) && PyIter_Check(receiver)) {
retval = Py_TYPE(receiver)->tp_iternext(receiver);
}
else {
retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
}
if (retval == NULL) {
if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
) {
monitor_raise(tstate, frame, this_instr);
}
if (_PyGen_FetchStopIterationValue(&retval) == 0) {
assert(retval != NULL);
JUMPBY(oparg);
}
else {
goto error;
}
}
Py_DECREF(v);
}
stack_pointer[-1] = retval;
DISPATCH();
}
TARGET(SEND_GEN) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(SEND_GEN);
static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
PyObject *v;
PyObject *receiver;
/* Skip 1 cache entry */
v = stack_pointer[-1];
receiver = stack_pointer[-2];
DEOPT_IF(tstate->interp->eval_frame, SEND);
PyGenObject *gen = (PyGenObject *)receiver;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND);
STAT_INC(SEND, hit);
_PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
STACK_SHRINK(1);
_PyFrame_StackPush(gen_frame, v);
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
DISPATCH_INLINED(gen_frame);
}
TARGET(SETUP_ANNOTATIONS) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SETUP_ANNOTATIONS);
int err;
PyObject *ann_dict;
if (LOCALS() == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when setting up annotations");
if (true) goto error;
}
/* check if __annotations__ in locals()... */
if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error;
if (ann_dict == NULL) {
ann_dict = PyDict_New();
if (ann_dict == NULL) goto error;
err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__),
ann_dict);
Py_DECREF(ann_dict);
if (err) goto error;
}
else {
Py_DECREF(ann_dict);
}
DISPATCH();
}
TARGET(SET_ADD) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_ADD);
PyObject *v;
PyObject *set;
v = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
int err = PySet_Add(set, v);
Py_DECREF(v);
if (err) goto pop_1_error;
stack_pointer += -1;
DISPATCH();
}
TARGET(SET_FUNCTION_ATTRIBUTE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE);
PyObject *func;
PyObject *attr;
func = stack_pointer[-1];
attr = stack_pointer[-2];
assert(PyFunction_Check(func));
PyFunctionObject *func_obj = (PyFunctionObject *)func;
switch(oparg) {
case MAKE_FUNCTION_CLOSURE:
assert(func_obj->func_closure == NULL);
func_obj->func_closure = attr;
break;
case MAKE_FUNCTION_ANNOTATIONS:
assert(func_obj->func_annotations == NULL);
func_obj->func_annotations = attr;
break;
case MAKE_FUNCTION_KWDEFAULTS:
assert(PyDict_CheckExact(attr));
assert(func_obj->func_kwdefaults == NULL);
func_obj->func_kwdefaults = attr;
break;
case MAKE_FUNCTION_DEFAULTS:
assert(PyTuple_CheckExact(attr));
assert(func_obj->func_defaults == NULL);
func_obj->func_defaults = attr;
break;
default:
Py_UNREACHABLE();
}
stack_pointer[-2] = func;
stack_pointer += -1;
DISPATCH();
}
TARGET(SET_UPDATE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SET_UPDATE);
PyObject *iterable;
PyObject *set;
iterable = stack_pointer[-1];
set = stack_pointer[-2 - (oparg-1)];
int err = _PySet_Update(set, iterable);
Py_DECREF(iterable);
if (err < 0) goto pop_1_error;
stack_pointer += -1;
DISPATCH();
}
TARGET(STORE_ATTR) {
frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR);
PREDICTED(STORE_ATTR);
_Py_CODEUNIT *this_instr = next_instr - 5;
(void)this_instr;
PyObject *owner;
PyObject *v;
// _SPECIALIZE_STORE_ATTR
owner = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr;
_Py_Specialize_StoreAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_ATTR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 3 cache entries */
// _STORE_ATTR
v = stack_pointer[-2];
{
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_SetAttr(owner, name, v);
Py_DECREF(v);
Py_DECREF(owner);
if (err) goto pop_2_error;
}
stack_pointer += -2;
DISPATCH();
}
TARGET(STORE_ATTR_INSTANCE_VALUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _GUARD_DORV_NO_DICT
{
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(_PyObject_GetManagedDict(owner), STORE_ATTR);
DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
STAT_INC(STORE_ATTR, hit);
assert(_PyObject_GetManagedDict(owner) == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
_PyDictValues_AddToInsertionOrder(values, index);
}
else {
Py_DECREF(old_value);
}
Py_DECREF(owner);
}
stack_pointer += -2;
DISPATCH();
}
TARGET(STORE_ATTR_SLOT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_SLOT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *value;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _STORE_ATTR_SLOT
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
char *addr = (char *)owner + index;
STAT_INC(STORE_ATTR, hit);
PyObject *old_value = *(PyObject **)addr;
*(PyObject **)addr = value;
Py_XDECREF(old_value);
Py_DECREF(owner);
}
stack_pointer += -2;
DISPATCH();
}
TARGET(STORE_ATTR_WITH_HINT) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 5;
INSTRUCTION_STATS(STORE_ATTR_WITH_HINT);
static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
PyObject *owner;
PyObject *value;
/* Skip 1 cache entry */
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint16_t hint = read_u16(&this_instr[4].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = _PyObject_GetManagedDict(owner);
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
PyObject *old_value;
uint64_t new_version;
if (DK_IS_UNICODE(dict->ma_keys)) {
PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
else {
PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
DEOPT_IF(ep->me_key != name, STORE_ATTR);
old_value = ep->me_value;
DEOPT_IF(old_value == NULL, STORE_ATTR);
new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value);
ep->me_value = value;
}
Py_DECREF(old_value);
STAT_INC(STORE_ATTR, hit);
/* Ensure dict is GC tracked if it needs to be */
if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
_PyObject_GC_TRACK(dict);
}
/* PEP 509 */
dict->ma_version_tag = new_version;
Py_DECREF(owner);
stack_pointer += -2;
DISPATCH();
}
TARGET(STORE_DEREF) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_DEREF);
PyObject *v;
v = stack_pointer[-1];
PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg);
PyCell_SetTakeRef(cell, v);
stack_pointer += -1;
DISPATCH();
}
TARGET(STORE_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST);
PyObject *value;
value = stack_pointer[-1];
SETLOCAL(oparg, value);
stack_pointer += -1;
DISPATCH();
}
TARGET(STORE_FAST_LOAD_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_LOAD_FAST);
PyObject *value1;
PyObject *value2;
value1 = stack_pointer[-1];
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
value2 = GETLOCAL(oparg2);
Py_INCREF(value2);
stack_pointer[-1] = value2;
DISPATCH();
}
TARGET(STORE_FAST_STORE_FAST) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_FAST_STORE_FAST);
PyObject *value1;
PyObject *value2;
value1 = stack_pointer[-1];
value2 = stack_pointer[-2];
uint32_t oparg1 = oparg >> 4;
uint32_t oparg2 = oparg & 15;
SETLOCAL(oparg1, value1);
SETLOCAL(oparg2, value2);
stack_pointer += -2;
DISPATCH();
}
TARGET(STORE_GLOBAL) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_GLOBAL);
PyObject *v;
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_SetItem(GLOBALS(), name, v);
Py_DECREF(v);
if (err) goto pop_1_error;
stack_pointer += -1;
DISPATCH();
}
TARGET(STORE_NAME) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_NAME);
PyObject *v;
v = stack_pointer[-1];
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
PyObject *ns = LOCALS();
int err;
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
Py_DECREF(v);
if (true) goto pop_1_error;
}
if (PyDict_CheckExact(ns))
err = PyDict_SetItem(ns, name, v);
else
err = PyObject_SetItem(ns, name, v);
Py_DECREF(v);
if (err) goto pop_1_error;
stack_pointer += -1;
DISPATCH();
}
TARGET(STORE_SLICE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(STORE_SLICE);
PyObject *stop;
PyObject *start;
PyObject *container;
PyObject *v;
stop = stack_pointer[-1];
start = stack_pointer[-2];
container = stack_pointer[-3];
v = stack_pointer[-4];
PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
int err;
if (slice == NULL) {
err = 1;
}
else {
err = PyObject_SetItem(container, slice, v);
Py_DECREF(slice);
}
Py_DECREF(v);
Py_DECREF(container);
if (err) goto pop_4_error;
stack_pointer += -4;
DISPATCH();
}
TARGET(STORE_SUBSCR) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR);
PREDICTED(STORE_SUBSCR);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *sub;
PyObject *container;
PyObject *v;
// _SPECIALIZE_STORE_SUBSCR
sub = stack_pointer[-1];
container = stack_pointer[-2];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(STORE_SUBSCR, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
// _STORE_SUBSCR
v = stack_pointer[-3];
{
/* container[sub] = v */
int err = PyObject_SetItem(container, sub, v);
Py_DECREF(v);
Py_DECREF(container);
Py_DECREF(sub);
if (err) goto pop_3_error;
}
stack_pointer += -3;
DISPATCH();
}
TARGET(STORE_SUBSCR_DICT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *dict;
PyObject *value;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
dict = stack_pointer[-2];
value = stack_pointer[-3];
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
if (err) goto pop_3_error;
stack_pointer += -3;
DISPATCH();
}
TARGET(STORE_SUBSCR_LIST_INT) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
PyObject *sub;
PyObject *list;
PyObject *value;
/* Skip 1 cache entry */
sub = stack_pointer[-1];
list = stack_pointer[-2];
value = stack_pointer[-3];
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
// Ensure nonnegative, zero-or-one-digit ints.
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR);
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
// Ensure index < len(list)
DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
PyList_SET_ITEM(list, index, value);
assert(old_value != NULL);
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
stack_pointer += -3;
DISPATCH();
}
TARGET(SWAP) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(SWAP);
PyObject *top;
PyObject *bottom;
top = stack_pointer[-1];
bottom = stack_pointer[-2 - (oparg-2)];
assert(oparg >= 2);
stack_pointer[-2 - (oparg-2)] = top;
stack_pointer[-1] = bottom;
DISPATCH();
}
TARGET(TO_BOOL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL);
PREDICTED(TO_BOOL);
_Py_CODEUNIT *this_instr = next_instr - 4;
(void)this_instr;
PyObject *value;
PyObject *res;
// _SPECIALIZE_TO_BOOL
value = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_ToBool(value, next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(TO_BOOL, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
}
/* Skip 2 cache entries */
// _TO_BOOL
{
int err = PyObject_IsTrue(value);
Py_DECREF(value);
if (err < 0) goto pop_1_error;
res = err ? Py_True : Py_False;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_ALWAYS_TRUE) {
_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *owner;
PyObject *value;
PyObject *res;
/* Skip 1 cache entry */
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL);
}
// _REPLACE_WITH_TRUE
value = owner;
{
Py_DECREF(value);
res = Py_True;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_BOOL) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_BOOL);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
DEOPT_IF(!PyBool_Check(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
DISPATCH();
}
TARGET(TO_BOOL_INT) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_INT);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
if (_PyLong_IsZero((PyLongObject *)value)) {
assert(_Py_IsImmortal(value));
res = Py_False;
}
else {
Py_DECREF(value);
res = Py_True;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_LIST) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_LIST);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
DEOPT_IF(!PyList_CheckExact(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
Py_DECREF(value);
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_NONE) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_NONE);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
// This one is a bit weird, because we expect *some* failures:
DEOPT_IF(!Py_IsNone(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
res = Py_False;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(TO_BOOL_STR) {
frame->instr_ptr = next_instr;
next_instr += 4;
INSTRUCTION_STATS(TO_BOOL_STR);
static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
PyObject *value;
PyObject *res;
/* Skip 1 cache entry */
/* Skip 2 cache entries */
value = stack_pointer[-1];
DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
if (value == &_Py_STR(empty)) {
assert(_Py_IsImmortal(value));
res = Py_False;
}
else {
assert(Py_SIZE(value));
Py_DECREF(value);
res = Py_True;
}
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNARY_INVERT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_INVERT);
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
res = PyNumber_Invert(value);
Py_DECREF(value);
if (res == NULL) goto pop_1_error;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNARY_NEGATIVE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NEGATIVE);
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
res = PyNumber_Negative(value);
Py_DECREF(value);
if (res == NULL) goto pop_1_error;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNARY_NOT) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNARY_NOT);
PyObject *value;
PyObject *res;
value = stack_pointer[-1];
assert(PyBool_Check(value));
res = Py_IsFalse(value) ? Py_True : Py_False;
stack_pointer[-1] = res;
DISPATCH();
}
TARGET(UNPACK_EX) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(UNPACK_EX);
PyObject *seq;
seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
PyObject **top = stack_pointer + totalargs - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
stack_pointer += (oparg >> 8) + (oparg & 0xFF);
DISPATCH();
}
TARGET(UNPACK_SEQUENCE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE);
PREDICTED(UNPACK_SEQUENCE);
_Py_CODEUNIT *this_instr = next_instr - 2;
(void)this_instr;
PyObject *seq;
// _SPECIALIZE_UNPACK_SEQUENCE
seq = stack_pointer[-1];
{
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(UNPACK_SEQUENCE, deferred);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */
(void)seq;
(void)counter;
}
// _UNPACK_SEQUENCE
{
PyObject **top = stack_pointer + oparg - 1;
int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
}
stack_pointer += -1 + oparg;
DISPATCH();
}
TARGET(UNPACK_SEQUENCE_LIST) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
PyObject **values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyList_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
*values++ = Py_NewRef(items[i]);
}
Py_DECREF(seq);
stack_pointer += -1 + oparg;
DISPATCH();
}
TARGET(UNPACK_SEQUENCE_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
PyObject **values;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
values = &stack_pointer[-1];
DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
PyObject **items = _PyTuple_ITEMS(seq);
for (int i = oparg; --i >= 0; ) {
*values++ = Py_NewRef(items[i]);
}
Py_DECREF(seq);
stack_pointer += -1 + oparg;
DISPATCH();
}
TARGET(UNPACK_SEQUENCE_TWO_TUPLE) {
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE);
static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
PyObject *seq;
PyObject *val1;
PyObject *val0;
/* Skip 1 cache entry */
seq = stack_pointer[-1];
assert(oparg == 2);
DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE);
DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE);
STAT_INC(UNPACK_SEQUENCE, hit);
val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
Py_DECREF(seq);
stack_pointer[-1] = val1;
stack_pointer[0] = val0;
stack_pointer += 1;
DISPATCH();
}
TARGET(WITH_EXCEPT_START) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(WITH_EXCEPT_START);
PyObject *val;
PyObject *lasti;
PyObject *exit_func;
PyObject *res;
val = stack_pointer[-1];
lasti = stack_pointer[-3];
exit_func = stack_pointer[-4];
/* At the top of the stack are 4 values:
- val: TOP = exc_info()
- unused: SECOND = previous exception
- lasti: THIRD = lasti of exception in exc_info()
- exit_func: FOURTH = the context.__exit__ bound method
We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
Then we push the __exit__ return value.
*/
PyObject *exc, *tb;
assert(val && PyExceptionInstance_Check(val));
exc = PyExceptionInstance_Class(val);
tb = PyException_GetTraceback(val);
if (tb == NULL) {
tb = Py_None;
}
else {
Py_DECREF(tb);
}
assert(PyLong_Check(lasti));
(void)lasti; // Shut up compiler warning if asserts are off
PyObject *stack[4] = {NULL, exc, val, tb};
res = PyObject_Vectorcall(exit_func, stack + 1,
3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
if (res == NULL) goto error;
stack_pointer[0] = res;
stack_pointer += 1;
DISPATCH();
}
TARGET(YIELD_VALUE) {
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(YIELD_VALUE);
PyObject *retval;
PyObject *value;
retval = stack_pointer[-1];
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
// or throw() call.
#if TIER_ONE
assert(frame != &entry_frame);
#endif
frame->instr_ptr++;
PyGenObject *gen = _PyFrame_GetGenerator(frame);
assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
assert(oparg == 0 || oparg == 1);
gen->gi_frame_state = FRAME_SUSPENDED + oparg;
stack_pointer += -1;
_PyFrame_SetStackPointer(frame, stack_pointer);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
_PyInterpreterFrame *gen_frame = frame;
frame = tstate->current_frame = frame->previous;
gen_frame->previous = NULL;
/* We don't know which of these is relevant here, so keep them equal */
assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
#if TIER_ONE
assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE ||
frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT ||
_PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR);
#endif
LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND);
LOAD_SP();
value = retval;
LLTRACE_RESUME_FRAME();
stack_pointer[0] = value;
stack_pointer += 1;
DISPATCH();
}
#undef TIER_ONE
Reference in a new issue View git blame Copy permalink