mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

GH-131798: Allow the JIT to remove more int/float/str guards (GH-131800)

Browse source
This commit is contained in:
Brandt Bucher 2025-04-01 15:10:15 -07:00 • committed by GitHub
parent cd69d55f64
commit 1a9d4a1fb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 756 additions and 703 deletions
Download patch file Download diff file Expand all files Collapse all files

86
Python/bytecodes.c
View file

@ -524,6 +524,12 @@ dummy_func(
res = PyStackRef_False;
}
op(_GUARD_NOS_UNICODE, (nos, tos -- nos, tos)) {
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
EXIT_IF(!PyUnicode_CheckExact(o));
}
op(_GUARD_TOS_UNICODE, (value -- value)) {
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
EXIT_IF(!PyUnicode_CheckExact(value_o));
@ -581,14 +587,8 @@ dummy_func(
BINARY_OP_EXTEND,
};
op(_GUARD_BOTH_INT, (left, right -- left, right)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
EXIT_IF(!PyLong_CheckExact(left_o));
EXIT_IF(!PyLong_CheckExact(right_o));
}
op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
op(_GUARD_NOS_INT, (left, right -- left, right)) {
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
EXIT_IF(!PyLong_CheckExact(left_o));
}
@ -644,20 +644,14 @@ dummy_func(
}
macro(BINARY_OP_MULTIPLY_INT) =
_GUARD_BOTH_INT + unused/5 + _BINARY_OP_MULTIPLY_INT;
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT;
macro(BINARY_OP_ADD_INT) =
_GUARD_BOTH_INT + unused/5 + _BINARY_OP_ADD_INT;
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT;
macro(BINARY_OP_SUBTRACT_INT) =
_GUARD_BOTH_INT + unused/5 + _BINARY_OP_SUBTRACT_INT;
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT;
op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
EXIT_IF(!PyFloat_CheckExact(left_o));
EXIT_IF(!PyFloat_CheckExact(right_o));
}
op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {
op(_GUARD_NOS_FLOAT, (left, right -- left, right)) {
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
EXIT_IF(!PyFloat_CheckExact(left_o));
}
@ -713,19 +707,11 @@ dummy_func(
}
macro(BINARY_OP_MULTIPLY_FLOAT) =
_GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT;
_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT;
macro(BINARY_OP_ADD_FLOAT) =
_GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT;
_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT;
macro(BINARY_OP_SUBTRACT_FLOAT) =
_GUARD_BOTH_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT;
op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
EXIT_IF(!PyUnicode_CheckExact(left_o));
EXIT_IF(!PyUnicode_CheckExact(right_o));
}
_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT;
pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
@ -743,7 +729,7 @@ dummy_func(
}
macro(BINARY_OP_ADD_UNICODE) =
_GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE;
_GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE;
// This is a subtle one. It's a super-instruction for
// BINARY_OP_ADD_UNICODE followed by STORE_FAST
@ -822,7 +808,7 @@ dummy_func(
unused/1 + _GUARD_BINARY_OP_EXTEND + rewind/-4 + _BINARY_OP_EXTEND;
macro(BINARY_OP_INPLACE_ADD_UNICODE) =
_GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE;
_GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE;
specializing op(_SPECIALIZE_BINARY_SLICE, (container, start, stop -- container, start, stop)) {
// Placeholder until we implement BINARY_SLICE specialization
@ -875,11 +861,14 @@ dummy_func(
macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE;
inst(BINARY_OP_SUBSCR_LIST_INT, (unused/5, list_st, sub_st -- res)) {
macro(BINARY_OP_SUBSCR_LIST_INT) =
_GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_LIST_INT;
op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
DEOPT_IF(!PyLong_CheckExact(sub));
assert(PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
// Deopt unless 0 <= sub < PyList_Size(list)
@ -901,12 +890,15 @@ dummy_func(
DECREF_INPUTS();
}
inst(BINARY_OP_SUBSCR_STR_INT, (unused/5, str_st, sub_st -- res)) {
macro(BINARY_OP_SUBSCR_STR_INT) =
_GUARD_TOS_INT + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_SUBSCR_STR_INT;
op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
DEOPT_IF(!PyLong_CheckExact(sub));
DEOPT_IF(!PyUnicode_CheckExact(str));
assert(PyLong_CheckExact(sub));
assert(PyUnicode_CheckExact(str));
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index);
@ -921,11 +913,14 @@ dummy_func(
res = PyStackRef_FromPyObjectImmortal(res_o);
}
inst(BINARY_OP_SUBSCR_TUPLE_INT, (unused/5, tuple_st, sub_st -- res)) {
macro(BINARY_OP_SUBSCR_TUPLE_INT) =
_GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT;
op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
DEOPT_IF(!PyLong_CheckExact(sub));
assert(PyLong_CheckExact(sub));
DEOPT_IF(!PyTuple_CheckExact(tuple));
// Deopt unless 0 <= sub < PyTuple_Size(list)
@ -1025,11 +1020,14 @@ dummy_func(
macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR;
inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list_st, sub_st -- )) {
macro(STORE_SUBSCR_LIST_INT) =
_GUARD_TOS_INT + unused/1 + _STORE_SUBSCR_LIST_INT;
op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
DEOPT_IF(!PyLong_CheckExact(sub));
assert(PyLong_CheckExact(sub));
DEOPT_IF(!PyList_CheckExact(list));
// Ensure nonnegative, zero-or-one-digit ints.
@ -2584,13 +2582,13 @@ dummy_func(
macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP;
macro(COMPARE_OP_FLOAT) =
_GUARD_BOTH_FLOAT + unused/1 + _COMPARE_OP_FLOAT;
_GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/1 + _COMPARE_OP_FLOAT;
macro(COMPARE_OP_INT) =
_GUARD_BOTH_INT + unused/1 + _COMPARE_OP_INT;
_GUARD_TOS_INT + _GUARD_NOS_INT + unused/1 + _COMPARE_OP_INT;
macro(COMPARE_OP_STR) =
_GUARD_BOTH_UNICODE + unused/1 + _COMPARE_OP_STR;
_GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/1 + _COMPARE_OP_STR;
op(_COMPARE_OP_FLOAT, (left, right -- res)) {
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);

95
Python/executor_cases.c.h generated
View file

@ -760,6 +760,20 @@
break;
}
case _GUARD_NOS_UNICODE: {
_PyStackRef tos;
_PyStackRef nos;
tos = stack_pointer[-1];
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_TOS_UNICODE: {
_PyStackRef value;
value = stack_pointer[-1];
@ -834,27 +848,12 @@
break;
}
case _GUARD_BOTH_INT: {
case _GUARD_NOS_INT: {
_PyStackRef right;
_PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyLong_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyLong_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_NOS_INT: {
_PyStackRef left;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
@ -958,27 +957,12 @@
break;
}
case _GUARD_BOTH_FLOAT: {
case _GUARD_NOS_FLOAT: {
_PyStackRef right;
_PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyFloat_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyFloat_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _GUARD_NOS_FLOAT: {
_PyStackRef left;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
@ -1079,24 +1063,6 @@
break;
}
case _GUARD_BOTH_UNICODE: {
_PyStackRef right;
_PyStackRef left;
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyUnicode_CheckExact(left_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyUnicode_CheckExact(right_o)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
break;
}
case _BINARY_OP_ADD_UNICODE: {
_PyStackRef right;
_PyStackRef left;
@ -1332,10 +1298,7 @@
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -1390,14 +1353,8 @@
str_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!PyUnicode_CheckExact(str)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyLong_CheckExact(sub));
assert(PyUnicode_CheckExact(str));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -1436,10 +1393,7 @@
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyLong_CheckExact(sub));
if (!PyTuple_CheckExact(tuple)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
@ -1646,10 +1600,7 @@
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();

635
Python/generated_cases.c.h generated
View file

@ -92,21 +92,27 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
// _GUARD_TOS_FLOAT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyFloat_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyFloat_CheckExact(right_o)) {
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -145,21 +151,27 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_INT
// _GUARD_TOS_INT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyLong_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyLong_CheckExact(right_o)) {
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -200,21 +212,29 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_UNICODE
// _GUARD_TOS_UNICODE
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyUnicode_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyUnicode_CheckExact(right_o)) {
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -223,6 +243,8 @@
/* Skip 5 cache entries */
// _BINARY_OP_ADD_UNICODE
{
right = tos;
left = nos;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
assert(PyUnicode_CheckExact(left_o));
@ -317,20 +339,28 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
// _GUARD_BOTH_UNICODE
// _GUARD_TOS_UNICODE
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyUnicode_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyUnicode_CheckExact(right_o)) {
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -339,6 +369,8 @@
/* Skip 5 cache entries */
// _BINARY_OP_INPLACE_ADD_UNICODE
{
right = tos;
left = nos;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
assert(PyUnicode_CheckExact(left_o));
assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right)));
@ -405,21 +437,27 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
// _GUARD_TOS_FLOAT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyFloat_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyFloat_CheckExact(right_o)) {
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -458,21 +496,27 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_INT
// _GUARD_TOS_INT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyLong_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyLong_CheckExact(right_o)) {
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -655,66 +699,76 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef list_st;
_PyStackRef sub_st;
_PyStackRef res;
// _GUARD_TOS_INT
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
sub_st = stack_pointer[-1];
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
// _BINARY_OP_SUBSCR_LIST_INT
{
sub_st = value;
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
// Deopt unless 0 <= sub < PyList_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
res = PyStackRef_FromPyObjectSteal(res_o);
#else
if (index >= PyList_GET_SIZE(list)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(BINARY_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = list_st;
list_st = res;
stack_pointer[-2] = list_st;
PyStackRef_CLOSE(tmp);
tmp = sub_st;
sub_st = PyStackRef_NULL;
stack_pointer[-1] = sub_st;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
}
if (!PyList_CheckExact(list)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
// Deopt unless 0 <= sub < PyList_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
res = PyStackRef_FromPyObjectSteal(res_o);
#else
if (index >= PyList_GET_SIZE(list)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
res = PyStackRef_FromPyObjectNew(res_o);
#endif
STAT_INC(BINARY_OP, hit);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = list_st;
list_st = res;
stack_pointer[-2] = list_st;
PyStackRef_CLOSE(tmp);
tmp = sub_st;
sub_st = PyStackRef_NULL;
stack_pointer[-1] = sub_st;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
DISPATCH();
}
@ -729,51 +783,71 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef str_st;
_PyStackRef sub_st;
_PyStackRef res;
// _GUARD_TOS_INT
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
sub_st = stack_pointer[-1];
str_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
if (!PyLong_CheckExact(sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
// _BINARY_OP_SUBSCR_STR_INT
{
sub_st = tos;
str_st = nos;
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
assert(PyLong_CheckExact(sub));
assert(PyUnicode_CheckExact(str));
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (PyUnicode_GET_LENGTH(str) <= index) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(str_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
res = PyStackRef_FromPyObjectImmortal(res_o);
}
if (!PyUnicode_CheckExact(str)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (PyUnicode_GET_LENGTH(str) <= index) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(str_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
res = PyStackRef_FromPyObjectImmortal(res_o);
stack_pointer[0] = res;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
@ -791,49 +865,59 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef tuple_st;
_PyStackRef sub_st;
_PyStackRef res;
// _GUARD_TOS_INT
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
}
/* Skip 5 cache entries */
sub_st = stack_pointer[-1];
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
if (!PyLong_CheckExact(sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
// _BINARY_OP_SUBSCR_TUPLE_INT
{
sub_st = value;
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
assert(PyLong_CheckExact(sub));
if (!PyTuple_CheckExact(tuple)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
// Deopt unless 0 <= sub < PyTuple_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (index >= PyTuple_GET_SIZE(tuple)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
res = PyStackRef_FromPyObjectNew(res_o);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = tuple_st;
tuple_st = res;
stack_pointer[-1] = tuple_st;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
if (!PyTuple_CheckExact(tuple)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
// Deopt unless 0 <= sub < PyTuple_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (index >= PyTuple_GET_SIZE(tuple)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
res = PyStackRef_FromPyObjectNew(res_o);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyStackRef tmp = tuple_st;
tuple_st = res;
stack_pointer[-1] = tuple_st;
PyStackRef_CLOSE(tmp);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH();
}
@ -848,21 +932,27 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
// _GUARD_TOS_FLOAT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyFloat_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyFloat_CheckExact(right_o)) {
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -901,21 +991,27 @@
next_instr += 6;
INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_INT
// _GUARD_TOS_INT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyLong_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyLong_CheckExact(right_o)) {
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(BINARY_OP);
assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
JUMP_TO_PREDICTED(BINARY_OP);
@ -4513,21 +4609,27 @@
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_FLOAT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_FLOAT
// _GUARD_TOS_FLOAT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyFloat_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyFloat_CheckExact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
}
if (!PyFloat_CheckExact(right_o)) {
}
// _GUARD_NOS_FLOAT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyFloat_CheckExact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
@ -4565,21 +4667,27 @@
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_INT);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef value;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_INT
// _GUARD_TOS_INT
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyLong_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
}
if (!PyLong_CheckExact(right_o)) {
}
// _GUARD_NOS_INT
{
right = value;
left = stack_pointer[-2];
(void)right;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
if (!PyLong_CheckExact(left_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
@ -4629,21 +4737,29 @@
next_instr += 2;
INSTRUCTION_STATS(COMPARE_OP_STR);
static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
_PyStackRef value;
_PyStackRef nos;
_PyStackRef tos;
_PyStackRef left;
_PyStackRef right;
_PyStackRef res;
// _GUARD_BOTH_UNICODE
// _GUARD_TOS_UNICODE
{
right = stack_pointer[-1];
left = stack_pointer[-2];
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
if (!PyUnicode_CheckExact(left_o)) {
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyUnicode_CheckExact(value_o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
}
if (!PyUnicode_CheckExact(right_o)) {
}
// _GUARD_NOS_UNICODE
{
tos = value;
nos = stack_pointer[-2];
(void)tos;
PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
if (!PyUnicode_CheckExact(o)) {
UPDATE_MISS_STATS(COMPARE_OP);
assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP));
JUMP_TO_PREDICTED(COMPARE_OP);
@ -4652,6 +4768,8 @@
/* Skip 1 cache entry */
// _COMPARE_OP_STR
{
right = tos;
left = nos;
PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(COMPARE_OP, hit);
@ -11346,56 +11464,65 @@
_PyStackRef value;
_PyStackRef list_st;
_PyStackRef sub_st;
/* Skip 1 cache entry */
sub_st = stack_pointer[-1];
list_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
if (!PyList_CheckExact(list)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
// Ensure nonnegative, zero-or-one-digit ints.
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (!LOCK_OBJECT(list)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
// Ensure index < len(list)
if (index >= PyList_GET_SIZE(list)) {
UNLOCK_OBJECT(list);
if (true) {
// _GUARD_TOS_INT
{
value = stack_pointer[-1];
PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
if (!PyLong_CheckExact(value_o)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
}
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index],
/* Skip 1 cache entry */
// _STORE_SUBSCR_LIST_INT
{
sub_st = value;
list_st = stack_pointer[-2];
value = stack_pointer[-3];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
assert(PyLong_CheckExact(sub));
if (!PyList_CheckExact(list)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
// Ensure nonnegative, zero-or-one-digit ints.
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (!LOCK_OBJECT(list)) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
// Ensure index < len(list)
if (index >= PyList_GET_SIZE(list)) {
UNLOCK_OBJECT(list);
if (true) {
UPDATE_MISS_STATS(STORE_SUBSCR);
assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR));
JUMP_TO_PREDICTED(STORE_SUBSCR);
}
}
STAT_INC(STORE_SUBSCR, hit);
PyObject *old_value = PyList_GET_ITEM(list, index);
FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index],
PyStackRef_AsPyObjectSteal(value));
assert(old_value != NULL);
UNLOCK_OBJECT(list); // unlock before decrefs!
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(list_st);
Py_DECREF(old_value);
stack_pointer = _PyFrame_GetStackPointer(frame);
assert(old_value != NULL);
UNLOCK_OBJECT(list); // unlock before decrefs!
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(list_st);
Py_DECREF(old_value);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
DISPATCH();
}

61
Python/optimizer_bytecodes.c
View file

@ -104,22 +104,18 @@ dummy_func(void) {
res = sym_new_null(ctx);
}
op(_GUARD_BOTH_INT, (left, right -- left, right)) {
if (sym_matches_type(left, &PyLong_Type)) {
if (sym_matches_type(right, &PyLong_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0);
}
op(_GUARD_TOS_INT, (tos -- tos)) {
if (sym_matches_type(tos, &PyLong_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
if (sym_matches_type(right, &PyLong_Type)) {
REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0);
}
sym_set_type(tos, &PyLong_Type);
}
op(_GUARD_NOS_INT, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyLong_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(left, &PyLong_Type);
sym_set_type(right, &PyLong_Type);
sym_set_type(nos, &PyLong_Type);
}
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
@ -145,32 +141,18 @@ dummy_func(void) {
}
}
op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {
if (sym_matches_type(left, &PyFloat_Type)) {
if (sym_matches_type(right, &PyFloat_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0);
}
op(_GUARD_TOS_FLOAT, (tos -- tos)) {
if (sym_matches_type(tos, &PyFloat_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
if (sym_matches_type(right, &PyFloat_Type)) {
REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0);
}
}
sym_set_type(left, &PyFloat_Type);
sym_set_type(right, &PyFloat_Type);
sym_set_type(tos, &PyFloat_Type);
}
op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) {
if (sym_matches_type(left, &PyUnicode_Type) &&
sym_matches_type(right, &PyUnicode_Type)) {
REPLACE_OP(this_instr, _NOP, 0 ,0);
op(_GUARD_NOS_FLOAT, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyFloat_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(left, &PyUnicode_Type);
sym_set_type(right, &PyUnicode_Type);
sym_set_type(nos, &PyFloat_Type);
}
op(_BINARY_OP, (left, right -- res)) {
@ -418,6 +400,13 @@ dummy_func(void) {
}
}
op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyUnicode_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyUnicode_Type);
}
op(_GUARD_TOS_UNICODE, (value -- value)) {
if (sym_matches_type(value, &PyUnicode_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);

94
Python/optimizer_cases.c.h generated
View file

@ -217,6 +217,16 @@
break;
}
case _GUARD_NOS_UNICODE: {
JitOptSymbol *nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyUnicode_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyUnicode_Type);
break;
}
case _GUARD_TOS_UNICODE: {
JitOptSymbol *value;
value = stack_pointer[-1];
@ -252,34 +262,23 @@
break;
}
case _GUARD_BOTH_INT: {
JitOptSymbol *right;
JitOptSymbol *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (sym_matches_type(left, &PyLong_Type)) {
if (sym_matches_type(right, &PyLong_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0);
}
}
else {
if (sym_matches_type(right, &PyLong_Type)) {
REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0);
}
}
sym_set_type(left, &PyLong_Type);
sym_set_type(right, &PyLong_Type);
break;
}
case _GUARD_NOS_INT: {
JitOptSymbol *nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyLong_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyLong_Type);
break;
}
case _GUARD_TOS_INT: {
JitOptSymbol *tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyLong_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyLong_Type);
break;
}
@ -376,34 +375,23 @@
break;
}
case _GUARD_BOTH_FLOAT: {
JitOptSymbol *right;
JitOptSymbol *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (sym_matches_type(left, &PyFloat_Type)) {
if (sym_matches_type(right, &PyFloat_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
else {
REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0);
}
}
else {
if (sym_matches_type(right, &PyFloat_Type)) {
REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0);
}
}
sym_set_type(left, &PyFloat_Type);
sym_set_type(right, &PyFloat_Type);
break;
}
case _GUARD_NOS_FLOAT: {
JitOptSymbol *nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyFloat_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(nos, &PyFloat_Type);
break;
}
case _GUARD_TOS_FLOAT: {
JitOptSymbol *tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyFloat_Type)) {
REPLACE_OP(this_instr, _NOP, 0, 0);
}
sym_set_type(tos, &PyFloat_Type);
break;
}
@ -503,20 +491,6 @@
break;
}
case _GUARD_BOTH_UNICODE: {
JitOptSymbol *right;
JitOptSymbol *left;
right = stack_pointer[-1];
left = stack_pointer[-2];
if (sym_matches_type(left, &PyUnicode_Type) &&
sym_matches_type(right, &PyUnicode_Type)) {
REPLACE_OP(this_instr, _NOP, 0 ,0);
}
sym_set_type(left, &PyUnicode_Type);
sym_set_type(right, &PyUnicode_Type);
break;
}
case _BINARY_OP_ADD_UNICODE: {
JitOptSymbol *right;
JitOptSymbol *left;