From 0b50a4f0cdee41a18fb4ba6e75569f9cfaceb39e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 15 Dec 2021 10:30:09 +0000 Subject: [PATCH] bpo-46039: Split yield from in two (GH-30035) * Split YIELD_FROM opcode into SEND and JUMP_ABSOLUTE. * Remove YIELD_FROM opcode. --- Include/internal/pycore_frame.h | 1 + Include/opcode.h | 24 ++++---- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 4 +- .../2021-12-13-17-01-13.bpo-46039.TrCBbF.rst | 2 + Objects/frameobject.c | 8 ++- Objects/genobject.c | 18 +++--- Python/ceval.c | 21 +++---- Python/compile.c | 55 +++++++++++++------ Python/opcode_targets.h | 8 +-- 10 files changed, 91 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-12-13-17-01-13.bpo-46039.TrCBbF.rst diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a55877b55fb..883bef199ba 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -66,6 +66,7 @@ static inline PyObject **_PyFrame_Stackbase(InterpreterFrame *f) { static inline PyObject *_PyFrame_StackPeek(InterpreterFrame *f) { assert(f->stacktop > f->f_code->co_nlocalsplus); + assert(f->localsplus[f->stacktop-1] != NULL); return f->localsplus[f->stacktop-1]; } diff --git a/Include/opcode.h b/Include/opcode.h index 4d14081a6b6..bdabffd9838 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -37,7 +37,6 @@ extern "C" { #define GET_YIELD_FROM_ITER 69 #define PRINT_EXPR 70 #define LOAD_BUILD_CLASS 71 -#define YIELD_FROM 72 #define GET_AWAITABLE 73 #define LOAD_ASSERTION_ERROR 74 #define LIST_TO_TUPLE 82 @@ -81,6 +80,7 @@ extern "C" { #define COPY 120 #define JUMP_IF_NOT_EXC_MATCH 121 #define BINARY_OP 122 +#define SEND 123 #define LOAD_FAST 124 #define STORE_FAST 125 #define DELETE_FAST 126 @@ -154,15 +154,15 @@ extern "C" { #define LOAD_GLOBAL_BUILTIN 65 #define LOAD_METHOD_ADAPTIVE 66 #define LOAD_METHOD_CACHED 67 -#define LOAD_METHOD_CLASS 75 -#define LOAD_METHOD_MODULE 76 -#define LOAD_METHOD_NO_DICT 77 -#define STORE_ATTR_ADAPTIVE 78 -#define STORE_ATTR_INSTANCE_VALUE 79 -#define STORE_ATTR_SLOT 80 -#define STORE_ATTR_WITH_HINT 81 -#define LOAD_FAST__LOAD_FAST 87 -#define STORE_FAST__LOAD_FAST 123 +#define LOAD_METHOD_CLASS 72 +#define LOAD_METHOD_MODULE 75 +#define LOAD_METHOD_NO_DICT 76 +#define STORE_ATTR_ADAPTIVE 77 +#define STORE_ATTR_INSTANCE_VALUE 78 +#define STORE_ATTR_SLOT 79 +#define STORE_ATTR_WITH_HINT 80 +#define LOAD_FAST__LOAD_FAST 81 +#define STORE_FAST__LOAD_FAST 87 #define LOAD_FAST__LOAD_CONST 128 #define LOAD_CONST__LOAD_FAST 131 #define STORE_FAST__STORE_FAST 134 @@ -172,7 +172,7 @@ static uint32_t _PyOpcode_RelativeJump[8] = { 0U, 0U, 536870912U, - 16384U, + 134234112U, 0U, 0U, 0U, @@ -182,7 +182,7 @@ static uint32_t _PyOpcode_Jump[8] = { 0U, 0U, 536870912U, - 2182070272U, + 2316288000U, 0U, 0U, 0U, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index abd0170cac0..0c1078accc8 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -373,6 +373,7 @@ _code_type = type(_write_atomic.__code__) # Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) # Python 3.11a3 3466 (bpo-45292: PEP-654 except*) # Python 3.11a4 3467 (Change CALL_xxx opcodes) +# Python 3.11a4 3468 (Add SEND opcode) # # MAGIC must change whenever the bytecode emitted by the compiler may no @@ -382,7 +383,7 @@ _code_type = type(_write_atomic.__code__) # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3467).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3468).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index 0b64686d605..cb16ef78313 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -93,7 +93,7 @@ def_op('GET_ITER', 68) def_op('GET_YIELD_FROM_ITER', 69) def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) -def_op('YIELD_FROM', 72) + def_op('GET_AWAITABLE', 73) def_op('LOAD_ASSERTION_ERROR', 74) @@ -143,7 +143,7 @@ def_op('RERAISE', 119) def_op('COPY', 120) jabs_op('JUMP_IF_NOT_EXC_MATCH', 121) def_op('BINARY_OP', 122) - +jrel_op('SEND', 123) # Number of bytes to skip def_op('LOAD_FAST', 124) # Local variable number haslocal.append(124) def_op('STORE_FAST', 125) # Local variable number diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-13-17-01-13.bpo-46039.TrCBbF.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-13-17-01-13.bpo-46039.TrCBbF.rst new file mode 100644 index 00000000000..18bdc34d21c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-13-17-01-13.bpo-46039.TrCBbF.rst @@ -0,0 +1,2 @@ +Remove the ``YIELD_FROM`` instruction and replace it with the ``SEND`` +instruction which performs the same operation, but without the loop. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 82931b6e85f..fc62713aa24 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -249,7 +249,13 @@ mark_stacks(PyCodeObject *code_obj, int len) next_stack = pop_value(pop_value(pop_value(next_stack))); stacks[i+1] = next_stack; break; - + case SEND: + j = get_arg(code, i) + i + 1; + assert(j < len); + assert(stacks[j] == UNINITIALIZED || stacks[j] == pop_value(next_stack)); + stacks[j] = pop_value(next_stack); + stacks[i+1] = next_stack; + break; case JUMP_FORWARD: j = get_arg(code, i) + i + 1; assert(j < len); diff --git a/Objects/genobject.c b/Objects/genobject.c index 1b08b43ac22..24a4e94bfc7 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -10,7 +10,7 @@ #include "pycore_frame.h" // InterpreterFrame #include "frameobject.h" // PyFrameObject #include "structmember.h" // PyMemberDef -#include "opcode.h" // YIELD_FROM +#include "opcode.h" // SEND static PyObject *gen_close(PyGenObject *, PyObject *); static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *); @@ -356,14 +356,14 @@ _PyGen_yf(PyGenObject *gen) unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); if (frame->f_lasti < 0) { - /* Return immediately if the frame didn't start yet. YIELD_FROM + /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start - with YIELD_FROM */ - assert(code[0] != YIELD_FROM); + with SEND */ + assert(code[0] != SEND); return NULL; } - if (code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)] != YIELD_FROM) + if (code[frame->f_lasti*sizeof(_Py_CODEUNIT)] != SEND || frame->stacktop < 0) return NULL; yf = _PyFrame_StackPeek(frame); Py_INCREF(yf); @@ -486,9 +486,13 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, ret = _PyFrame_StackPop((InterpreterFrame *)gen->gi_iframe); assert(ret == yf); Py_DECREF(ret); - /* Termination repetition of YIELD_FROM */ + /* Termination repetition of SEND loop */ assert(frame->f_lasti >= 0); - frame->f_lasti += 1; + PyObject *bytecode = gen->gi_code->co_code; + unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND); + int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1]; + frame->f_lasti += jump; if (_PyGen_FetchStopIterationValue(&val) == 0) { ret = gen_send(gen, val); Py_DECREF(val); diff --git a/Python/ceval.c b/Python/ceval.c index 6d2784894f7..79324330d26 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1798,7 +1798,8 @@ check_eval_breaker: if (_Py_atomic_load_relaxed(eval_breaker)) { opcode = _Py_OPCODE(*next_instr); if (opcode != BEFORE_ASYNC_WITH && - opcode != YIELD_FROM) { + opcode != SEND && + _Py_OPCODE(next_instr[-1]) != SEND) { /* Few cases where we skip running signal handlers and other pending calls: - If we're about to enter the 'with:'. It will prevent @@ -2642,8 +2643,9 @@ check_eval_breaker: DISPATCH(); } - TARGET(YIELD_FROM) { + TARGET(SEND) { assert(frame->depth == 0); + assert(STACK_LEVEL() >= 2); PyObject *v = POP(); PyObject *receiver = TOP(); PySendResult gen_status; @@ -2680,17 +2682,13 @@ check_eval_breaker: } if (gen_status == PYGEN_RETURN) { assert (retval != NULL); - Py_DECREF(receiver); SET_TOP(retval); - retval = NULL; + JUMPBY(oparg); DISPATCH(); } assert (gen_status == PYGEN_NEXT); - /* receiver remains on stack, retval is value to be yielded */ - /* and repeat... */ - assert(frame->f_lasti > 0); - frame->f_lasti -= 1; + assert (retval != NULL); frame->f_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); @@ -6770,8 +6768,11 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj, return -1; } if (line != -1 && f->f_trace_lines) { - /* Trace backward edges or if line number has changed */ - if (frame->f_lasti < instr_prev || line != lastline) { + /* Trace backward edges (except in 'yield from') or if line number has changed */ + int trace = line != lastline || + (frame->f_lasti < instr_prev && + _Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND); + if (trace) { result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); } } diff --git a/Python/compile.c b/Python/compile.c index afd9a629c0a..6179ad980aa 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1045,8 +1045,6 @@ stack_effect(int opcode, int oparg, int jump) return 0; case YIELD_VALUE: return 0; - case YIELD_FROM: - return -1; case POP_BLOCK: return 0; case POP_EXCEPT: @@ -1065,7 +1063,8 @@ stack_effect(int opcode, int oparg, int jump) case FOR_ITER: /* -1 at end of iterator, 1 if continue iterating. */ return jump > 0 ? -1 : 1; - + case SEND: + return jump > 0 ? -1 : 0; case STORE_ATTR: return -2; case DELETE_ATTR: @@ -1667,6 +1666,9 @@ compiler_addop_j_noline(struct compiler *c, int opcode, basicblock *b) the ASDL name to synthesize the name of the C type and the visit function. */ +#define ADD_YIELD_FROM(C) \ + RETURN_IF_FALSE(compiler_add_yield_from((C))) + #define VISIT(C, TYPE, V) {\ if (!compiler_visit_ ## TYPE((C), (V))) \ return 0; \ @@ -1819,6 +1821,24 @@ compiler_call_exit_with_nones(struct compiler *c) { return 1; } +static int +compiler_add_yield_from(struct compiler *c) +{ + basicblock *start, *jump, *exit; + start = compiler_new_block(c); + jump = compiler_new_block(c); + exit = compiler_new_block(c); + if (start == NULL || jump == NULL || exit == NULL) { + return 0; + } + compiler_use_next_block(c, start); + ADDOP_JUMP(c, SEND, exit); + compiler_use_next_block(c, jump); + ADDOP_JUMP(c, JUMP_ABSOLUTE, start); + compiler_use_next_block(c, exit); + return 1; +} + /* Unwind a frame block. If preserve_tos is true, the TOS before * popping the blocks will be restored afterwards, unless another * return, break or continue is found. In which case, the TOS will @@ -1893,7 +1913,7 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info, if (info->fb_type == ASYNC_WITH) { ADDOP(c, GET_AWAITABLE); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); } ADDOP(c, POP_TOP); /* The exit block should appear to execute after the @@ -3006,7 +3026,7 @@ compiler_async_for(struct compiler *c, stmt_ty s) ADDOP_JUMP(c, SETUP_FINALLY, except); ADDOP(c, GET_ANEXT); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); ADDOP(c, POP_BLOCK); /* for SETUP_FINALLY */ /* Success block for __anext__ */ @@ -5192,7 +5212,7 @@ compiler_async_comprehension_generator(struct compiler *c, ADDOP_JUMP(c, SETUP_FINALLY, except); ADDOP(c, GET_ANEXT); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); ADDOP(c, POP_BLOCK); VISIT(c, expr, gen->target); @@ -5342,7 +5362,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, if (is_async_generator && type != COMP_GENEXP) { ADDOP(c, GET_AWAITABLE); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); } return 1; @@ -5493,7 +5513,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) ADDOP(c, BEFORE_ASYNC_WITH); ADDOP(c, GET_AWAITABLE); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); ADDOP_JUMP(c, SETUP_WITH, final); @@ -5530,7 +5550,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) return 0; ADDOP(c, GET_AWAITABLE); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); ADDOP(c, POP_TOP); @@ -5544,7 +5564,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) ADDOP(c, WITH_EXCEPT_START); ADDOP(c, GET_AWAITABLE); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); compiler_with_except_finish(c, cleanup); compiler_use_next_block(c, exit); @@ -5701,7 +5721,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.YieldFrom.value); ADDOP(c, GET_YIELD_FROM_ITER); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); break; case Await_kind: if (!IS_TOP_LEVEL_AWAIT(c)){ @@ -5718,7 +5738,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) VISIT(c, expr, e->v.Await.value); ADDOP(c, GET_AWAITABLE); ADDOP_LOAD_CONST(c, Py_None); - ADDOP(c, YIELD_FROM); + ADD_YIELD_FROM(c); break; case Compare_kind: return compiler_compare(c, e); @@ -7544,10 +7564,13 @@ normalize_jumps(struct assembler *a) continue; } struct instr *last = &b->b_instr[b->b_iused-1]; - if (last->i_opcode == JUMP_ABSOLUTE && - last->i_target->b_visited == 0 - ) { - last->i_opcode = JUMP_FORWARD; + if (last->i_opcode == JUMP_ABSOLUTE) { + if (last->i_target->b_visited == 0) { + last->i_opcode = JUMP_FORWARD; + } + else if (b->b_iused >= 2 && b->b_instr[b->b_iused-2].i_opcode == SEND) { + last->i_opcode = JUMP_ABSOLUTE_QUICK; + } } } } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 05bd9392d8e..3b2e99d5a32 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -71,22 +71,22 @@ static void *opcode_targets[256] = { &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_YIELD_FROM, + &&TARGET_LOAD_METHOD_CLASS, &&TARGET_GET_AWAITABLE, &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_LOAD_METHOD_CLASS, &&TARGET_LOAD_METHOD_MODULE, &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, &&TARGET_YIELD_VALUE, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, @@ -122,7 +122,7 @@ static void *opcode_targets[256] = { &&TARGET_COPY, &&TARGET_JUMP_IF_NOT_EXC_MATCH, &&TARGET_BINARY_OP, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_SEND, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST,