mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-30 13:38:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-93554: Conditional jump opcodes only jump forward (GH-96318)

Browse source
This commit is contained in:
Irit Katriel 2022-09-01 21:36:47 +01:00 • committed by GitHub
parent a91f25577c
commit 4c72517cad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 298 additions and 448 deletions
Download patch file Download diff file Expand all files Collapse all files

147
Python/ceval.c
View file

@ -3307,7 +3307,7 @@ handle_eval_breaker:
TARGET(COMPARE_OP_FLOAT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (float ? float) + POP_JUMP_(direction)_IF_(true/false)
// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
@ -3325,23 +3325,12 @@ handle_eval_breaker:
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
int jump = (9 << (sign + 1)) & when_to_jump_mask;
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
}
else if (jump >= 8) {
assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE);
JUMPBY(1 - oparg);
CHECK_EVAL_BREAKER();
}
else {
assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_FORWARD_IF_FALSE);
JUMPBY(1 + oparg);
}
NOTRACE_DISPATCH();
@ -3349,7 +3338,7 @@ handle_eval_breaker:
TARGET(COMPARE_OP_INT_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (int ? int) + POP_JUMP_(direction)_IF_(true/false)
// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
PyObject *right = TOP();
@ -3368,23 +3357,12 @@ handle_eval_breaker:
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
int jump = (9 << (sign + 1)) & when_to_jump_mask;
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
int jump = (1 << (sign + 1)) & when_to_jump_mask;
if (!jump) {
next_instr++;
}
else if (jump >= 8) {
assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE);
JUMPBY(1 - oparg);
CHECK_EVAL_BREAKER();
}
else {
assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_FORWARD_IF_FALSE);
JUMPBY(1 + oparg);
}
NOTRACE_DISPATCH();
@ -3392,9 +3370,9 @@ handle_eval_breaker:
TARGET(COMPARE_OP_STR_JUMP) {
assert(cframe.use_tracing == 0);
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_(direction)_IF_(true/false)
// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
int when_to_jump_mask = cache->mask;
int invert = cache->mask;
PyObject *right = TOP();
PyObject *left = SECOND();
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
@ -3407,28 +3385,17 @@ handle_eval_breaker:
assert(oparg == Py_EQ || oparg == Py_NE);
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE);
STACK_SHRINK(2);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(res == 0 || res == 1);
int sign = 1 - res;
int jump = (9 << (sign + 1)) & when_to_jump_mask;
assert(invert == 0 || invert == 1);
int jump = res ^ invert;
if (!jump) {
next_instr++;
}
else if (jump >= 8) {
assert(opcode == POP_JUMP_BACKWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE);
JUMPBY(1 - oparg);
CHECK_EVAL_BREAKER();
}
else {
assert(opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_FORWARD_IF_FALSE);
JUMPBY(1 + oparg);
}
NOTRACE_DISPATCH();
@ -3575,34 +3542,8 @@ handle_eval_breaker:
JUMP_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
}
TARGET(POP_JUMP_BACKWARD_IF_FALSE) {
PREDICTED(POP_JUMP_BACKWARD_IF_FALSE);
PyObject *cond = POP();
if (Py_IsTrue(cond)) {
_Py_DECREF_NO_DEALLOC(cond);
DISPATCH();
}
if (Py_IsFalse(cond)) {
_Py_DECREF_NO_DEALLOC(cond);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
int err = PyObject_IsTrue(cond);
Py_DECREF(cond);
if (err > 0)
;
else if (err == 0) {
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
}
else
goto error;
DISPATCH();
}
TARGET(POP_JUMP_FORWARD_IF_FALSE) {
PREDICTED(POP_JUMP_FORWARD_IF_FALSE);
TARGET(POP_JUMP_IF_FALSE) {
PREDICTED(POP_JUMP_IF_FALSE);
PyObject *cond = POP();
if (Py_IsTrue(cond)) {
_Py_DECREF_NO_DEALLOC(cond);
@ -3625,32 +3566,7 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_BACKWARD_IF_TRUE) {
PyObject *cond = POP();
if (Py_IsFalse(cond)) {
_Py_DECREF_NO_DEALLOC(cond);
DISPATCH();
}
if (Py_IsTrue(cond)) {
_Py_DECREF_NO_DEALLOC(cond);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
int err = PyObject_IsTrue(cond);
Py_DECREF(cond);
if (err > 0) {
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
}
else if (err == 0)
;
else
goto error;
DISPATCH();
}
TARGET(POP_JUMP_FORWARD_IF_TRUE) {
TARGET(POP_JUMP_IF_TRUE) {
PyObject *cond = POP();
if (Py_IsFalse(cond)) {
_Py_DECREF_NO_DEALLOC(cond);
@ -3673,19 +3589,7 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_BACKWARD_IF_NOT_NONE) {
PyObject *value = POP();
if (!Py_IsNone(value)) {
Py_DECREF(value);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
_Py_DECREF_NO_DEALLOC(value);
DISPATCH();
}
TARGET(POP_JUMP_FORWARD_IF_NOT_NONE) {
TARGET(POP_JUMP_IF_NOT_NONE) {
PyObject *value = POP();
if (!Py_IsNone(value)) {
JUMPBY(oparg);
@ -3694,20 +3598,7 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(POP_JUMP_BACKWARD_IF_NONE) {
PyObject *value = POP();
if (Py_IsNone(value)) {
_Py_DECREF_NO_DEALLOC(value);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
}
else {
Py_DECREF(value);
}
DISPATCH();
}
TARGET(POP_JUMP_FORWARD_IF_NONE) {
TARGET(POP_JUMP_IF_NONE) {
PyObject *value = POP();
if (Py_IsNone(value)) {
_Py_DECREF_NO_DEALLOC(value);
@ -3834,8 +3725,7 @@ handle_eval_breaker:
PyObject *res = match ? Py_True : Py_False;
Py_INCREF(res);
PUSH(res);
PREDICT(POP_JUMP_FORWARD_IF_FALSE);
PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
PREDICT(POP_JUMP_IF_FALSE);
DISPATCH();
}
@ -3845,8 +3735,7 @@ handle_eval_breaker:
PyObject *res = match ? Py_True : Py_False;
Py_INCREF(res);
PUSH(res);
PREDICT(POP_JUMP_FORWARD_IF_FALSE);
PREDICT(POP_JUMP_BACKWARD_IF_FALSE);
PREDICT(POP_JUMP_IF_FALSE);
DISPATCH();
}

154
Python/compile.c
View file

@ -96,23 +96,11 @@
#define IS_ASSEMBLER_OPCODE(opcode) \
((opcode) == JUMP_FORWARD || \
(opcode) == JUMP_BACKWARD || \
(opcode) == JUMP_BACKWARD_NO_INTERRUPT || \
(opcode) == POP_JUMP_FORWARD_IF_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_NONE || \
(opcode) == POP_JUMP_FORWARD_IF_NOT_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_NOT_NONE || \
(opcode) == POP_JUMP_FORWARD_IF_TRUE || \
(opcode) == POP_JUMP_BACKWARD_IF_TRUE || \
(opcode) == POP_JUMP_FORWARD_IF_FALSE || \
(opcode) == POP_JUMP_BACKWARD_IF_FALSE)
(opcode) == JUMP_BACKWARD_NO_INTERRUPT)
#define IS_BACKWARDS_JUMP_OPCODE(opcode) \
((opcode) == JUMP_BACKWARD || \
(opcode) == JUMP_BACKWARD_NO_INTERRUPT || \
(opcode) == POP_JUMP_BACKWARD_IF_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_NOT_NONE || \
(opcode) == POP_JUMP_BACKWARD_IF_TRUE || \
(opcode) == POP_JUMP_BACKWARD_IF_FALSE)
(opcode) == JUMP_BACKWARD_NO_INTERRUPT)
#define IS_UNCONDITIONAL_JUMP_OPCODE(opcode) \
((opcode) == JUMP || \
@ -1146,17 +1134,9 @@ stack_effect(int opcode, int oparg, int jump)
case JUMP_IF_FALSE_OR_POP:
return jump ? 0 : -1;
case POP_JUMP_BACKWARD_IF_NONE:
case POP_JUMP_FORWARD_IF_NONE:
case POP_JUMP_IF_NONE:
case POP_JUMP_BACKWARD_IF_NOT_NONE:
case POP_JUMP_FORWARD_IF_NOT_NONE:
case POP_JUMP_IF_NOT_NONE:
case POP_JUMP_FORWARD_IF_FALSE:
case POP_JUMP_BACKWARD_IF_FALSE:
case POP_JUMP_IF_FALSE:
case POP_JUMP_FORWARD_IF_TRUE:
case POP_JUMP_BACKWARD_IF_TRUE:
case POP_JUMP_IF_TRUE:
return -1;
@ -7747,63 +7727,91 @@ assemble_emit(struct assembler *a, struct instr *i)
return 1;
}
static void
normalize_jumps(basicblock *entryblock)
static int
normalize_jumps_in_block(cfg_builder *g, basicblock *b) {
struct instr *last = basicblock_last_instr(b);
if (last == NULL || !is_jump(last)) {
return 0;
}
assert(!IS_ASSEMBLER_OPCODE(last->i_opcode));
bool is_forward = last->i_target->b_visited == 0;
switch(last->i_opcode) {
case JUMP:
last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD;
return 0;
case JUMP_NO_INTERRUPT:
last->i_opcode = is_forward ?
JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT;
return 0;
}
int reversed_opcode = 0;
switch(last->i_opcode) {
case POP_JUMP_IF_NOT_NONE:
reversed_opcode = POP_JUMP_IF_NONE;
break;
case POP_JUMP_IF_NONE:
reversed_opcode = POP_JUMP_IF_NOT_NONE;
break;
case POP_JUMP_IF_FALSE:
reversed_opcode = POP_JUMP_IF_TRUE;
break;
case POP_JUMP_IF_TRUE:
reversed_opcode = POP_JUMP_IF_FALSE;
break;
case JUMP_IF_TRUE_OR_POP:
case JUMP_IF_FALSE_OR_POP:
if (!is_forward) {
/* As far as we can tell, the compiler never emits
* these jumps with a backwards target. If/when this
* exception is raised, we have found a use case for
* a backwards version of this jump (or to replace
* it with the sequence (COPY 1, POP_JUMP_IF_T/F, POP)
*/
PyErr_Format(PyExc_SystemError,
"unexpected %s jumping backwards",
last->i_opcode == JUMP_IF_TRUE_OR_POP ?
"JUMP_IF_TRUE_OR_POP" : "JUMP_IF_FALSE_OR_POP");
}
return 0;
}
if (is_forward) {
return 0;
}
/* transform 'conditional jump T' to
* 'reversed_jump b_next' followed by 'jump_backwards T'
*/
basicblock *target = last->i_target;
basicblock *backwards_jump = cfg_builder_new_block(g);
if (backwards_jump == NULL) {
return -1;
}
basicblock_addop(backwards_jump, JUMP, target->b_label, NO_LOCATION);
backwards_jump->b_instr[0].i_target = target;
last->i_opcode = reversed_opcode;
last->i_target = b->b_next;
backwards_jump->b_cold = b->b_cold;
backwards_jump->b_next = b->b_next;
b->b_next = backwards_jump;
return 0;
}
static int
normalize_jumps(cfg_builder *g)
{
basicblock *entryblock = g->g_entryblock;
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
b->b_visited = 0;
}
for (basicblock *b = entryblock; b != NULL; b = b->b_next) {
b->b_visited = 1;
struct instr *last = basicblock_last_instr(b);
if (last == NULL) {
continue;
}
assert(!IS_ASSEMBLER_OPCODE(last->i_opcode));
if (is_jump(last)) {
bool is_forward = last->i_target->b_visited == 0;
switch(last->i_opcode) {
case JUMP:
last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD;
break;
case JUMP_NO_INTERRUPT:
last->i_opcode = is_forward ?
JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT;
break;
case POP_JUMP_IF_NOT_NONE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_NOT_NONE : POP_JUMP_BACKWARD_IF_NOT_NONE;
break;
case POP_JUMP_IF_NONE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_NONE : POP_JUMP_BACKWARD_IF_NONE;
break;
case POP_JUMP_IF_FALSE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_FALSE : POP_JUMP_BACKWARD_IF_FALSE;
break;
case POP_JUMP_IF_TRUE:
last->i_opcode = is_forward ?
POP_JUMP_FORWARD_IF_TRUE : POP_JUMP_BACKWARD_IF_TRUE;
break;
case JUMP_IF_TRUE_OR_POP:
case JUMP_IF_FALSE_OR_POP:
if (!is_forward) {
/* As far as we can tell, the compiler never emits
* these jumps with a backwards target. If/when this
* exception is raised, we have found a use case for
* a backwards version of this jump (or to replace
* it with the sequence (COPY 1, POP_JUMP_IF_T/F, POP)
*/
PyErr_Format(PyExc_SystemError,
"unexpected %s jumping backwards",
last->i_opcode == JUMP_IF_TRUE_OR_POP ?
"JUMP_IF_TRUE_OR_POP" : "JUMP_IF_FALSE_OR_POP");
}
break;
}
if (normalize_jumps_in_block(g, b) < 0) {
return -1;
}
}
return 0;
}
static void
@ -8638,7 +8646,9 @@ assemble(struct compiler *c, int addNone)
}
/* Order of basic blocks must have been determined by now */
normalize_jumps(g->g_entryblock);
if (normalize_jumps(g) < 0) {
goto error;
}
if (add_checks_for_loads_of_unknown_variables(g->g_entryblock, c) < 0) {
goto error;

16
Python/opcode_targets.h generated
View file

@ -113,8 +113,8 @@ static void *opcode_targets[256] = {
&&TARGET_JUMP_IF_FALSE_OR_POP,
&&TARGET_JUMP_IF_TRUE_OR_POP,
&&TARGET_LOAD_ATTR_METHOD_NO_DICT,
&&TARGET_POP_JUMP_FORWARD_IF_FALSE,
&&TARGET_POP_JUMP_FORWARD_IF_TRUE,
&&TARGET_POP_JUMP_IF_FALSE,
&&TARGET_POP_JUMP_IF_TRUE,
&&TARGET_LOAD_GLOBAL,
&&TARGET_IS_OP,
&&TARGET_CONTAINS_OP,
@ -127,8 +127,8 @@ static void *opcode_targets[256] = {
&&TARGET_STORE_FAST,
&&TARGET_DELETE_FAST,
&&TARGET_LOAD_FAST_CHECK,
&&TARGET_POP_JUMP_FORWARD_IF_NOT_NONE,
&&TARGET_POP_JUMP_FORWARD_IF_NONE,
&&TARGET_POP_JUMP_IF_NOT_NONE,
&&TARGET_POP_JUMP_IF_NONE,
&&TARGET_RAISE_VARARGS,
&&TARGET_GET_AWAITABLE,
&&TARGET_MAKE_FUNCTION,
@ -172,10 +172,6 @@ static void *opcode_targets[256] = {
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&TARGET_POP_JUMP_BACKWARD_IF_NOT_NONE,
&&TARGET_POP_JUMP_BACKWARD_IF_NONE,
&&TARGET_POP_JUMP_BACKWARD_IF_FALSE,
&&TARGET_POP_JUMP_BACKWARD_IF_TRUE,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_STORE_SUBSCR_ADAPTIVE,
&&TARGET_STORE_SUBSCR_DICT,
@ -254,5 +250,9 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_DO_TRACING
};

15
Python/specialize.c
View file

@ -1999,10 +1999,8 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
_PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
if (next_opcode != POP_JUMP_FORWARD_IF_FALSE &&
next_opcode != POP_JUMP_BACKWARD_IF_FALSE &&
next_opcode != POP_JUMP_FORWARD_IF_TRUE &&
next_opcode != POP_JUMP_BACKWARD_IF_TRUE) {
if (next_opcode != POP_JUMP_IF_FALSE &&
next_opcode != POP_JUMP_IF_TRUE) {
// Can't ever combine, so don't don't bother being adaptive (unless
// we're collecting stats, where it's more important to get accurate hit
// counts for the unadaptive version and each of the different failure
@ -2021,14 +2019,9 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
}
assert(oparg <= Py_GE);
int when_to_jump_mask = compare_masks[oparg];
if (next_opcode == POP_JUMP_FORWARD_IF_FALSE ||
next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
if (next_opcode == POP_JUMP_IF_FALSE) {
when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
}
if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE ||
next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
when_to_jump_mask <<= 3;
}
if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
goto failure;
@ -2056,7 +2049,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
}
else {
_Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP);
cache->mask = when_to_jump_mask;
cache->mask = (when_to_jump_mask & 2) == 0;
goto success;
}
}