mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-24 11:44:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-46841: Use inline caching for COMPARE_OP (GH-31622)

Browse source
This commit is contained in:
Brandt Bucher 2022-03-01 05:53:13 -08:00 • committed by GitHub
parent df9f759755
commit 7820a5897e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 179 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

33
Python/ceval.c
View file

@ -3671,8 +3671,10 @@ handle_eval_breaker:
SET_TOP(res);
Py_DECREF(left);
Py_DECREF(right);
if (res == NULL)
if (res == NULL) {
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_IF_TRUE);
DISPATCH();
@ -3680,18 +3682,17 @@ handle_eval_breaker:
TARGET(COMPARE_OP_ADAPTIVE) {
assert(cframe.use_tracing == 0);
SpecializedCacheEntry *cache = GET_CACHE();
if (cache->adaptive.counter == 0) {
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (cache->counter == 0) {
PyObject *right = TOP();
PyObject *left = SECOND();
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, cache);
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH();
}
else {
STAT_INC(COMPARE_OP, deferred);
cache->adaptive.counter--;
oparg = cache->adaptive.original_oparg;
cache->counter--;
JUMP_TO_INSTRUCTION(COMPARE_OP);
}
}
@ -3699,8 +3700,8 @@ handle_eval_breaker:
TARGET(COMPARE_OP_FLOAT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
SpecializedCacheEntry *caches = GET_CACHE();
int when_to_jump_mask = caches[0].adaptive.index;
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
@ -3711,6 +3712,7 @@ handle_eval_breaker:
DEOPT_IF(isnan(dleft), COMPARE_OP);
DEOPT_IF(isnan(dright), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
Py_DECREF(left);
@ -3731,8 +3733,8 @@ handle_eval_breaker:
TARGET(COMPARE_OP_INT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
SpecializedCacheEntry *caches = GET_CACHE();
int when_to_jump_mask = caches[0].adaptive.index;
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
@ -3744,6 +3746,7 @@ handle_eval_breaker:
Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
int sign = (ileft > iright) - (ileft < iright);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
Py_DECREF(left);
@ -3764,8 +3767,8 @@ handle_eval_breaker:
TARGET(COMPARE_OP_STR_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
SpecializedCacheEntry *caches = GET_CACHE();
int invert = caches[0].adaptive.index;
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int invert = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
@ -3775,8 +3778,8 @@ handle_eval_breaker:
if (res < 0) {
goto error;
}
assert(caches[0].adaptive.original_oparg == Py_EQ ||
caches[0].adaptive.original_oparg == Py_NE);
assert(oparg == Py_EQ || oparg == Py_NE);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
STACK_SHRINK(2);
@ -5601,7 +5604,7 @@ MISS_WITH_CACHE(LOAD_METHOD)
MISS_WITH_CACHE(PRECALL)
MISS_WITH_CACHE(CALL)
MISS_WITH_INLINE_CACHE(BINARY_OP)
MISS_WITH_CACHE(COMPARE_OP)
MISS_WITH_INLINE_CACHE(COMPARE_OP)
MISS_WITH_CACHE(BINARY_SUBSCR)
MISS_WITH_INLINE_CACHE(UNPACK_SEQUENCE)
MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)

36
Python/specialize.c
View file

@ -65,7 +65,6 @@ static uint8_t cache_requirements[256] = {
[CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
[STORE_ATTR] = 1, // _PyAdaptiveEntry
[COMPARE_OP] = 1, /* _PyAdaptiveEntry */
};
Py_ssize_t _Py_QuickenedCount = 0;
@ -2057,26 +2056,27 @@ static int compare_masks[] = {
};
void
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
_Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
int oparg)
{
_PyAdaptiveEntry *adaptive = &cache->adaptive;
int op = adaptive->original_oparg;
int next_opcode = _Py_OPCODE(instr[1]);
assert(_PyOpcode_InlineCacheEntries[COMPARE_OP] ==
INLINE_CACHE_ENTRIES_COMPARE_OP);
_PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) {
// Can't ever combine, so don't don't bother being adaptive (unless
// we're collecting stats, where it's more important to get accurate hit
// counts for the unadaptive version and each of the different failure
// types):
#ifndef Py_STATS
*instr = _Py_MAKECODEUNIT(COMPARE_OP, adaptive->original_oparg);
*instr = _Py_MAKECODEUNIT(COMPARE_OP, oparg);
return;
#endif
SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP);
goto failure;
}
assert(op <= Py_GE);
int when_to_jump_mask = compare_masks[op];
assert(oparg <= Py_GE);
int when_to_jump_mask = compare_masks[oparg];
if (next_opcode == POP_JUMP_IF_FALSE) {
when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
}
@ -2085,14 +2085,14 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
goto failure;
}
if (PyFloat_CheckExact(lhs)) {
*instr = _Py_MAKECODEUNIT(COMPARE_OP_FLOAT_JUMP, _Py_OPARG(*instr));
adaptive->index = when_to_jump_mask;
*instr = _Py_MAKECODEUNIT(COMPARE_OP_FLOAT_JUMP, oparg);
cache->mask = when_to_jump_mask;
goto success;
}
if (PyLong_CheckExact(lhs)) {
if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) {
*instr = _Py_MAKECODEUNIT(COMPARE_OP_INT_JUMP, _Py_OPARG(*instr));
adaptive->index = when_to_jump_mask;
*instr = _Py_MAKECODEUNIT(COMPARE_OP_INT_JUMP, oparg);
cache->mask = when_to_jump_mask;
goto success;
}
else {
@ -2101,24 +2101,24 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
}
}
if (PyUnicode_CheckExact(lhs)) {
if (op != Py_EQ && op != Py_NE) {
if (oparg != Py_EQ && oparg != Py_NE) {
SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
goto failure;
}
else {
*instr = _Py_MAKECODEUNIT(COMPARE_OP_STR_JUMP, _Py_OPARG(*instr));
adaptive->index = (when_to_jump_mask & 2) == 0;
*instr = _Py_MAKECODEUNIT(COMPARE_OP_STR_JUMP, oparg);
cache->mask = (when_to_jump_mask & 2) == 0;
goto success;
}
}
SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
failure:
STAT_INC(COMPARE_OP, failure);
cache_backoff(adaptive);
cache->counter = ADAPTIVE_CACHE_BACKOFF;
return;
success:
STAT_INC(COMPARE_OP, success);
adaptive->counter = initial_counter_value();
cache->counter = initial_counter_value();
}
#ifdef Py_STATS