bpo-47120: make POP_JUMP_IF_TRUE/FALSE/NONE/NOT_NONE relative (GH-32400)

This commit is contained in:
Irit Katriel 2022-04-11 10:40:24 +01:00 committed by GitHub
parent 98ff4a6877
commit dd207a6ac5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 357 additions and 135 deletions

View file

@ -3646,8 +3646,6 @@ handle_eval_breaker:
goto error;
}
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_IF_TRUE);
DISPATCH();
}
@ -3670,7 +3668,7 @@ handle_eval_breaker:
TARGET(COMPARE_OP_FLOAT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
// Combined: COMPARE_OP (float ? float) + POP_JUMP_(direction)_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
@ -3688,22 +3686,31 @@ handle_eval_breaker:
STACK_SHRINK(2);
Py_DECREF(left);
Py_DECREF(right);
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
int jump = (9 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
NOTRACE_DISPATCH();
}
else if (jump >= 8) {
assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE);
JUMPBY(1 - oparg);
CHECK_EVAL_BREAKER();
}
else {
JUMPTO(oparg);
CHECK_EVAL_BREAKER();
NOTRACE_DISPATCH();
assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_FORWARD_IF_FALSE);
JUMPBY(1 + oparg);
}
NOTRACE_DISPATCH();
}
TARGET(COMPARE_OP_INT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
// Combined: COMPARE_OP (int ? int) + POP_JUMP_(direction)_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
@ -3722,24 +3729,33 @@ handle_eval_breaker:
STACK_SHRINK(2);
Py_DECREF(left);
Py_DECREF(right);
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
int jump = (9 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
NOTRACE_DISPATCH();
}
else if (jump >= 8) {
assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE);
JUMPBY(1 - oparg);
CHECK_EVAL_BREAKER();
}
else {
JUMPTO(oparg);
CHECK_EVAL_BREAKER();
NOTRACE_DISPATCH();
assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_FORWARD_IF_FALSE);
JUMPBY(1 + oparg);
}
NOTRACE_DISPATCH();
}
TARGET(COMPARE_OP_STR_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_(direction)_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int invert = cache->mask;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
@ -3752,22 +3768,31 @@ handle_eval_breaker:
assert(oparg == Py_EQ || oparg == Py_NE);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
STACK_SHRINK(2);
Py_DECREF(left);
Py_DECREF(right);
assert(res == 0 || res == 1);
assert(invert == 0 || invert == 1);
int jump = res ^ invert;
int sign = 1 - res;
int jump = (9 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
NOTRACE_DISPATCH();
}
else if (jump >= 8) {
assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE);
JUMPBY(1 - oparg);
CHECK_EVAL_BREAKER();
}
else {
JUMPTO(oparg);
CHECK_EVAL_BREAKER();
NOTRACE_DISPATCH();
assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_FORWARD_IF_FALSE);
JUMPBY(1 + oparg);
}
NOTRACE_DISPATCH();
}
TARGET(IS_OP) {
@ -3779,8 +3804,6 @@ handle_eval_breaker:
SET_TOP(b);
Py_DECREF(left);
Py_DECREF(right);
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_IF_TRUE);
DISPATCH();
}
@ -3796,8 +3819,6 @@ handle_eval_breaker:
PyObject *b = (res^oparg) ? Py_True : Py_False;
Py_INCREF(b);
PUSH(b);
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_IF_TRUE);
DISPATCH();
}
@ -3915,26 +3936,25 @@ handle_eval_breaker:
JUMP_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
}
TARGET(POP_JUMP_IF_FALSE) {
PREDICTED(POP_JUMP_IF_FALSE);
TARGET(POP_JUMP_BACKWARD_IF_FALSE) {
PREDICTED(POP_JUMP_BACKWARD_IF_FALSE);
PyObject *cond = POP();
int err;
if (Py_IsTrue(cond)) {
Py_DECREF(cond);
DISPATCH();
}
if (Py_IsFalse(cond)) {
Py_DECREF(cond);
JUMPTO(oparg);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
err = PyObject_IsTrue(cond);
int err = PyObject_IsTrue(cond);
Py_DECREF(cond);
if (err > 0)
;
else if (err == 0) {
JUMPTO(oparg);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
}
else
@ -3942,24 +3962,46 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_IF_TRUE) {
PREDICTED(POP_JUMP_IF_TRUE);
TARGET(POP_JUMP_FORWARD_IF_FALSE) {
PREDICTED(POP_JUMP_FORWARD_IF_FALSE);
PyObject *cond = POP();
if (Py_IsTrue(cond)) {
Py_DECREF(cond);
}
else if (Py_IsFalse(cond)) {
Py_DECREF(cond);
JUMPBY(oparg);
}
else {
int err = PyObject_IsTrue(cond);
Py_DECREF(cond);
if (err > 0)
;
else if (err == 0) {
JUMPBY(oparg);
}
else
goto error;
}
DISPATCH();
}
TARGET(POP_JUMP_BACKWARD_IF_TRUE) {
PyObject *cond = POP();
int err;
if (Py_IsFalse(cond)) {
Py_DECREF(cond);
DISPATCH();
}
if (Py_IsTrue(cond)) {
Py_DECREF(cond);
JUMPTO(oparg);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
err = PyObject_IsTrue(cond);
int err = PyObject_IsTrue(cond);
Py_DECREF(cond);
if (err > 0) {
JUMPTO(oparg);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
}
else if (err == 0)
@ -3969,11 +4011,34 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_IF_NOT_NONE) {
TARGET(POP_JUMP_FORWARD_IF_TRUE) {
PyObject *cond = POP();
if (Py_IsFalse(cond)) {
Py_DECREF(cond);
}
else if (Py_IsTrue(cond)) {
Py_DECREF(cond);
JUMPBY(oparg);
}
else {
int err = PyObject_IsTrue(cond);
Py_DECREF(cond);
if (err > 0) {
JUMPBY(oparg);
}
else if (err == 0)
;
else
goto error;
}
DISPATCH();
}
TARGET(POP_JUMP_BACKWARD_IF_NOT_NONE) {
PyObject *value = POP();
if (!Py_IsNone(value)) {
Py_DECREF(value);
JUMPTO(oparg);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
@ -3981,11 +4046,20 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_IF_NONE) {
TARGET(POP_JUMP_FORWARD_IF_NOT_NONE) {
PyObject *value = POP();
if (!Py_IsNone(value)) {
JUMPBY(oparg);
}
Py_DECREF(value);
DISPATCH();
}
TARGET(POP_JUMP_BACKWARD_IF_NONE) {
PyObject *value = POP();
if (Py_IsNone(value)) {
Py_DECREF(value);
JUMPTO(oparg);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
@ -3993,6 +4067,15 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_FORWARD_IF_NONE) {
PyObject *value = POP();
if (Py_IsNone(value)) {
JUMPBY(oparg);
}
Py_DECREF(value);
DISPATCH();
}
TARGET(JUMP_IF_FALSE_OR_POP) {
PyObject *cond = TOP();
int err;
@ -4108,7 +4191,8 @@ handle_eval_breaker:
PyObject *res = match ? Py_True : Py_False;
Py_INCREF(res);
PUSH(res);
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_FORWARD_IF_FALSE);
PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
DISPATCH();
}
@ -4118,7 +4202,8 @@ handle_eval_breaker:
PyObject *res = match ? Py_True : Py_False;
Py_INCREF(res);
PUSH(res);
PREDICT(POP_JUMP_IF_FALSE);
PREDICT(POP_JUMP_FORWARD_IF_FALSE);
PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
DISPATCH();
}

View file

@ -78,8 +78,12 @@
#define POP_BLOCK -4
#define JUMP -5
#define JUMP_NO_INTERRUPT -6
#define POP_JUMP_IF_FALSE -7
#define POP_JUMP_IF_TRUE -8
#define POP_JUMP_IF_NONE -9
#define POP_JUMP_IF_NOT_NONE -10
#define MIN_VIRTUAL_OPCODE -6
#define MIN_VIRTUAL_OPCODE -10
#define MAX_ALLOWED_OPCODE 254
#define IS_WITHIN_OPCODE_RANGE(opcode) \
@ -87,11 +91,36 @@
#define IS_VIRTUAL_OPCODE(opcode) ((opcode) < 0)
#define IS_VIRTUAL_JUMP_OPCODE(opcode) \
((opcode) == JUMP || \
(opcode) == JUMP_NO_INTERRUPT || \
(opcode) == POP_JUMP_IF_NONE || \
(opcode) == POP_JUMP_IF_NOT_NONE || \
(opcode) == POP_JUMP_IF_FALSE || \
(opcode) == POP_JUMP_IF_TRUE)
/* opcodes which are not emitted in codegen stage, only by the assembler */
#define IS_ASSEMBLER_OPCODE(opcode) \
((opcode) == JUMP_FORWARD || \
(opcode) == JUMP_BACKWARD || \
(opcode) == JUMP_BACKWARD_NO_INTERRUPT)
(opcode) == JUMP_BACKWARD_NO_INTERRUPT || \
(opcode) == POP_JUMP_FORWARD_IF_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_NONE || \
(opcode) == POP_JUMP_FORWARD_IF_NOT_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_NOT_NONE || \
(opcode) == POP_JUMP_FORWARD_IF_TRUE || \
(opcode) == POP_JUMP_BACKWARD_IF_TRUE || \
(opcode) == POP_JUMP_FORWARD_IF_FALSE || \
(opcode) == POP_JUMP_BACKWARD_IF_FALSE)
#define IS_BACKWARDS_JUMP_OPCODE(opcode) \
((opcode) == JUMP_BACKWARD || \
(opcode) == JUMP_BACKWARD_NO_INTERRUPT || \
(opcode) == POP_JUMP_BACKWARD_IF_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_NOT_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_TRUE || \
(opcode) == POP_JUMP_BACKWARD_IF_FALSE)
#define IS_TOP_LEVEL_AWAIT(c) ( \
@ -156,7 +185,7 @@ is_block_push(struct instr *instr)
static inline int
is_jump(struct instr *i)
{
return i->i_opcode == JUMP ||
return IS_VIRTUAL_JUMP_OPCODE(i->i_opcode) ||
is_bit_set_in_table(_PyOpcode_Jump, i->i_opcode);
}
@ -1027,10 +1056,18 @@ stack_effect(int opcode, int oparg, int jump)
case JUMP_IF_FALSE_OR_POP:
return jump ? 0 : -1;
case POP_JUMP_IF_FALSE:
case POP_JUMP_IF_TRUE:
case POP_JUMP_BACKWARD_IF_NONE:
case POP_JUMP_FORWARD_IF_NONE:
case POP_JUMP_IF_NONE:
case POP_JUMP_BACKWARD_IF_NOT_NONE:
case POP_JUMP_FORWARD_IF_NOT_NONE:
case POP_JUMP_IF_NOT_NONE:
case POP_JUMP_FORWARD_IF_FALSE:
case POP_JUMP_BACKWARD_IF_FALSE:
case POP_JUMP_IF_FALSE:
case POP_JUMP_FORWARD_IF_TRUE:
case POP_JUMP_BACKWARD_IF_TRUE:
case POP_JUMP_IF_TRUE:
return -1;
case LOAD_GLOBAL:
@ -7609,14 +7646,33 @@ normalize_jumps(struct assembler *a)
}
struct instr *last = &b->b_instr[b->b_iused-1];
assert(!IS_ASSEMBLER_OPCODE(last->i_opcode));
if (last->i_opcode == JUMP) {
if (is_jump(last)) {
bool is_forward = last->i_target->b_visited == 0;
last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD;
}
if (last->i_opcode == JUMP_NO_INTERRUPT) {
bool is_forward = last->i_target->b_visited == 0;
last->i_opcode = is_forward ?
JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT;
switch(last->i_opcode) {
case JUMP:
last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD;
break;
case JUMP_NO_INTERRUPT:
last->i_opcode = is_forward ?
JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT;
break;
case POP_JUMP_IF_NOT_NONE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_NOT_NONE : POP_JUMP_BACKWARD_IF_NOT_NONE;
break;
case POP_JUMP_IF_NONE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_NONE : POP_JUMP_BACKWARD_IF_NONE;
break;
case POP_JUMP_IF_FALSE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_FALSE : POP_JUMP_BACKWARD_IF_FALSE;
break;
case POP_JUMP_IF_TRUE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_TRUE : POP_JUMP_BACKWARD_IF_TRUE;
break;
}
}
}
}
@ -7652,16 +7708,17 @@ assemble_jump_offsets(struct assembler *a, struct compiler *c)
instr->i_oparg = instr->i_target->b_offset;
if (is_relative_jump(instr)) {
if (instr->i_oparg < bsize) {
assert(instr->i_opcode == JUMP_BACKWARD ||
instr->i_opcode == JUMP_BACKWARD_NO_INTERRUPT);
assert(IS_BACKWARDS_JUMP_OPCODE(instr->i_opcode));
instr->i_oparg = bsize - instr->i_oparg;
}
else {
assert(instr->i_opcode != JUMP_BACKWARD);
assert(instr->i_opcode != JUMP_BACKWARD_NO_INTERRUPT);
assert(!IS_BACKWARDS_JUMP_OPCODE(instr->i_opcode));
instr->i_oparg -= bsize;
}
}
else {
assert(!IS_BACKWARDS_JUMP_OPCODE(instr->i_opcode));
}
if (instr_size(instr) != isize) {
extended_arg_recompile = 1;
}
@ -8644,7 +8701,7 @@ apply_static_swaps(basicblock *block, int i)
static bool
jump_thread(struct instr *inst, struct instr *target, int opcode)
{
assert(!IS_VIRTUAL_OPCODE(opcode) || opcode == JUMP);
assert(!IS_VIRTUAL_OPCODE(opcode) || IS_VIRTUAL_JUMP_OPCODE(opcode));
assert(is_jump(inst));
assert(is_jump(target));
// bpo-45773: If inst->i_target == target->i_target, then nothing actually

View file

@ -113,8 +113,8 @@ static void *opcode_targets[256] = {
&&TARGET_JUMP_IF_FALSE_OR_POP,
&&TARGET_JUMP_IF_TRUE_OR_POP,
&&TARGET_PRECALL_NO_KW_STR_1,
&&TARGET_POP_JUMP_IF_FALSE,
&&TARGET_POP_JUMP_IF_TRUE,
&&TARGET_POP_JUMP_FORWARD_IF_FALSE,
&&TARGET_POP_JUMP_FORWARD_IF_TRUE,
&&TARGET_LOAD_GLOBAL,
&&TARGET_IS_OP,
&&TARGET_CONTAINS_OP,
@ -127,8 +127,8 @@ static void *opcode_targets[256] = {
&&TARGET_STORE_FAST,
&&TARGET_DELETE_FAST,
&&TARGET_PRECALL_NO_KW_TYPE_1,
&&TARGET_POP_JUMP_IF_NOT_NONE,
&&TARGET_POP_JUMP_IF_NONE,
&&TARGET_POP_JUMP_FORWARD_IF_NOT_NONE,
&&TARGET_POP_JUMP_FORWARD_IF_NONE,
&&TARGET_RAISE_VARARGS,
&&TARGET_GET_AWAITABLE,
&&TARGET_MAKE_FUNCTION,
@ -172,6 +172,10 @@ static void *opcode_targets[256] = {
&&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&TARGET_POP_JUMP_BACKWARD_IF_NOT_NONE,
&&TARGET_POP_JUMP_BACKWARD_IF_NONE,
&&TARGET_POP_JUMP_BACKWARD_IF_FALSE,
&&TARGET_POP_JUMP_BACKWARD_IF_TRUE,
&&TARGET_UNPACK_SEQUENCE_LIST,
&&TARGET_UNPACK_SEQUENCE_TUPLE,
&&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
@ -250,9 +254,5 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_DO_TRACING
};

View file

@ -1883,7 +1883,10 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
_PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) {
if (next_opcode != POP_JUMP_FORWARD_IF_FALSE &&
next_opcode != POP_JUMP_BACKWARD_IF_FALSE &&
next_opcode != POP_JUMP_FORWARD_IF_TRUE &&
next_opcode != POP_JUMP_BACKWARD_IF_TRUE) {
// Can't ever combine, so don't don't bother being adaptive (unless
// we're collecting stats, where it's more important to get accurate hit
// counts for the unadaptive version and each of the different failure
@ -1901,9 +1904,14 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
}
assert(oparg <= Py_GE);
int when_to_jump_mask = compare_masks[oparg];
if (next_opcode == POP_JUMP_IF_FALSE) {
if (next_opcode == POP_JUMP_FORWARD_IF_FALSE ||
next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
}
if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE ||
next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
when_to_jump_mask <<= 3;
}
if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
goto failure;
@ -1931,7 +1939,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
}
else {
_Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP);
cache->mask = (when_to_jump_mask & 2) == 0;
cache->mask = when_to_jump_mask;
goto success;
}
}