GH-111485: Use micro-ops to split specialization code from base action (GH-111561)

Include/internal/pycore_opcode_metadata.h

@@ -27,68 +27,94 @@
 
 #define _EXIT_TRACE 300
 #define _SET_IP 301
-#define _GUARD_BOTH_INT 302
+#define _SPECIALIZE_TO_BOOL 302
-#define _BINARY_OP_MULTIPLY_INT 303
+#define _TO_BOOL 303
-#define _BINARY_OP_ADD_INT 304
+#define _GUARD_BOTH_INT 304
-#define _BINARY_OP_SUBTRACT_INT 305
+#define _BINARY_OP_MULTIPLY_INT 305
-#define _GUARD_BOTH_FLOAT 306
+#define _BINARY_OP_ADD_INT 306
-#define _BINARY_OP_MULTIPLY_FLOAT 307
+#define _BINARY_OP_SUBTRACT_INT 307
-#define _BINARY_OP_ADD_FLOAT 308
+#define _GUARD_BOTH_FLOAT 308
-#define _BINARY_OP_SUBTRACT_FLOAT 309
+#define _BINARY_OP_MULTIPLY_FLOAT 309
-#define _GUARD_BOTH_UNICODE 310
+#define _BINARY_OP_ADD_FLOAT 310
-#define _BINARY_OP_ADD_UNICODE 311
+#define _BINARY_OP_SUBTRACT_FLOAT 311
-#define _BINARY_OP_INPLACE_ADD_UNICODE 312
+#define _GUARD_BOTH_UNICODE 312
-#define _POP_FRAME 313
+#define _BINARY_OP_ADD_UNICODE 313
-#define _GUARD_GLOBALS_VERSION 314
+#define _BINARY_OP_INPLACE_ADD_UNICODE 314
-#define _GUARD_BUILTINS_VERSION 315
+#define _SPECIALIZE_BINARY_SUBSCR 315
-#define _LOAD_GLOBAL_MODULE 316
+#define _BINARY_SUBSCR 316
-#define _LOAD_GLOBAL_BUILTINS 317
+#define _SPECIALIZE_STORE_SUBSCR 317
-#define _GUARD_TYPE_VERSION 318
+#define _STORE_SUBSCR 318
-#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319
+#define _POP_FRAME 319
-#define _LOAD_ATTR_INSTANCE_VALUE 320
+#define _SPECIALIZE_SEND 320
-#define _CHECK_ATTR_MODULE 321
+#define _SEND 321
-#define _LOAD_ATTR_MODULE 322
+#define _SPECIALIZE_UNPACK_SEQUENCE 322
-#define _CHECK_ATTR_WITH_HINT 323
+#define _UNPACK_SEQUENCE 323
-#define _LOAD_ATTR_WITH_HINT 324
+#define _SPECIALIZE_STORE_ATTR 324
-#define _LOAD_ATTR_SLOT 325
+#define _STORE_ATTR 325
-#define _CHECK_ATTR_CLASS 326
+#define _SPECIALIZE_LOAD_GLOBAL 326
-#define _LOAD_ATTR_CLASS 327
+#define _LOAD_GLOBAL 327
-#define _GUARD_DORV_VALUES 328
+#define _GUARD_GLOBALS_VERSION 328
-#define _STORE_ATTR_INSTANCE_VALUE 329
+#define _GUARD_BUILTINS_VERSION 329
-#define _STORE_ATTR_SLOT 330
+#define _LOAD_GLOBAL_MODULE 330
-#define _IS_NONE 331
+#define _LOAD_GLOBAL_BUILTINS 331
-#define _ITER_CHECK_LIST 332
+#define _SPECIALIZE_LOAD_SUPER_ATTR 332
-#define _ITER_JUMP_LIST 333
+#define _LOAD_SUPER_ATTR 333
-#define _IS_ITER_EXHAUSTED_LIST 334
+#define _SPECIALIZE_LOAD_ATTR 334
-#define _ITER_NEXT_LIST 335
+#define _LOAD_ATTR 335
-#define _ITER_CHECK_TUPLE 336
+#define _GUARD_TYPE_VERSION 336
-#define _ITER_JUMP_TUPLE 337
+#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337
-#define _IS_ITER_EXHAUSTED_TUPLE 338
+#define _LOAD_ATTR_INSTANCE_VALUE 338
-#define _ITER_NEXT_TUPLE 339
+#define _CHECK_ATTR_MODULE 339
-#define _ITER_CHECK_RANGE 340
+#define _LOAD_ATTR_MODULE 340
-#define _ITER_JUMP_RANGE 341
+#define _CHECK_ATTR_WITH_HINT 341
-#define _IS_ITER_EXHAUSTED_RANGE 342
+#define _LOAD_ATTR_WITH_HINT 342
-#define _ITER_NEXT_RANGE 343
+#define _LOAD_ATTR_SLOT 343
-#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 344
+#define _CHECK_ATTR_CLASS 344
-#define _GUARD_KEYS_VERSION 345
+#define _LOAD_ATTR_CLASS 345
-#define _LOAD_ATTR_METHOD_WITH_VALUES 346
+#define _GUARD_DORV_VALUES 346
-#define _LOAD_ATTR_METHOD_NO_DICT 347
+#define _STORE_ATTR_INSTANCE_VALUE 347
-#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 348
+#define _STORE_ATTR_SLOT 348
-#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 349
+#define _SPECIALIZE_COMPARE_OP 349
-#define _CHECK_ATTR_METHOD_LAZY_DICT 350
+#define _COMPARE_OP 350
-#define _LOAD_ATTR_METHOD_LAZY_DICT 351
+#define _IS_NONE 351
-#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 352
+#define _SPECIALIZE_FOR_ITER 352
-#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 353
+#define _FOR_ITER 353
-#define _CHECK_PEP_523 354
+#define _ITER_CHECK_LIST 354
-#define _CHECK_FUNCTION_EXACT_ARGS 355
+#define _ITER_JUMP_LIST 355
-#define _CHECK_STACK_SPACE 356
+#define _IS_ITER_EXHAUSTED_LIST 356
-#define _INIT_CALL_PY_EXACT_ARGS 357
+#define _ITER_NEXT_LIST 357
-#define _PUSH_FRAME 358
+#define _ITER_CHECK_TUPLE 358
-#define _POP_JUMP_IF_FALSE 359
+#define _ITER_JUMP_TUPLE 359
-#define _POP_JUMP_IF_TRUE 360
+#define _IS_ITER_EXHAUSTED_TUPLE 360
-#define _JUMP_TO_TOP 361
+#define _ITER_NEXT_TUPLE 361
-#define _SAVE_RETURN_OFFSET 362
+#define _ITER_CHECK_RANGE 362
-#define _INSERT 363
+#define _ITER_JUMP_RANGE 363
+#define _IS_ITER_EXHAUSTED_RANGE 364
+#define _ITER_NEXT_RANGE 365
+#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 366
+#define _GUARD_KEYS_VERSION 367
+#define _LOAD_ATTR_METHOD_WITH_VALUES 368
+#define _LOAD_ATTR_METHOD_NO_DICT 369
+#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 370
+#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 371
+#define _CHECK_ATTR_METHOD_LAZY_DICT 372
+#define _LOAD_ATTR_METHOD_LAZY_DICT 373
+#define _SPECIALIZE_CALL 374
+#define _CALL 375
+#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 376
+#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 377
+#define _CHECK_PEP_523 378
+#define _CHECK_FUNCTION_EXACT_ARGS 379
+#define _CHECK_STACK_SPACE 380
+#define _INIT_CALL_PY_EXACT_ARGS 381
+#define _PUSH_FRAME 382
+#define _SPECIALIZE_BINARY_OP 383
+#define _BINARY_OP 384
+#define _POP_JUMP_IF_FALSE 385
+#define _POP_JUMP_IF_TRUE 386
+#define _JUMP_TO_TOP 387
+#define _SAVE_RETURN_OFFSET 388
+#define _INSERT 389
 
 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump);
 #ifdef NEED_OPCODE_METADATA
@@ -138,6 +164,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case UNARY_NOT:
             return 1;
+        case _SPECIALIZE_TO_BOOL:
+            return 1;
+        case _TO_BOOL:
+            return 1;
         case TO_BOOL:
             return 1;
         case TO_BOOL_BOOL:
@@ -192,6 +222,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 2;
         case BINARY_OP_INPLACE_ADD_UNICODE:
             return 2;
+        case _SPECIALIZE_BINARY_SUBSCR:
+            return 2;
+        case _BINARY_SUBSCR:
+            return 2;
         case BINARY_SUBSCR:
             return 2;
         case BINARY_SLICE:
@@ -212,6 +246,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return (oparg-1) + 2;
         case SET_ADD:
             return (oparg-1) + 2;
+        case _SPECIALIZE_STORE_SUBSCR:
+            return 2;
+        case _STORE_SUBSCR:
+            return 3;
         case STORE_SUBSCR:
             return 3;
         case STORE_SUBSCR_LIST_INT:
@@ -244,6 +282,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case GET_AWAITABLE:
             return 1;
+        case _SPECIALIZE_SEND:
+            return 2;
+        case _SEND:
+            return 2;
         case SEND:
             return 2;
         case SEND_GEN:
@@ -268,6 +310,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case DELETE_NAME:
             return 0;
+        case _SPECIALIZE_UNPACK_SEQUENCE:
+            return 1;
+        case _UNPACK_SEQUENCE:
+            return 1;
         case UNPACK_SEQUENCE:
             return 1;
         case UNPACK_SEQUENCE_TWO_TUPLE:
@@ -278,6 +324,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case UNPACK_EX:
             return 1;
+        case _SPECIALIZE_STORE_ATTR:
+            return 1;
+        case _STORE_ATTR:
+            return 2;
         case STORE_ATTR:
             return 2;
         case DELETE_ATTR:
@@ -292,6 +342,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case LOAD_NAME:
             return 0;
+        case _SPECIALIZE_LOAD_GLOBAL:
+            return 0;
+        case _LOAD_GLOBAL:
+            return 0;
         case LOAD_GLOBAL:
             return 0;
         case _GUARD_GLOBALS_VERSION:
@@ -346,6 +400,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return (oparg - 1) + 3;
         case INSTRUMENTED_LOAD_SUPER_ATTR:
             return 3;
+        case _SPECIALIZE_LOAD_SUPER_ATTR:
+            return 3;
+        case _LOAD_SUPER_ATTR:
+            return 3;
         case LOAD_SUPER_ATTR:
             return 3;
         case LOAD_SUPER_METHOD:
@@ -358,6 +416,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 3;
         case LOAD_SUPER_ATTR_METHOD:
             return 3;
+        case _SPECIALIZE_LOAD_ATTR:
+            return 1;
+        case _LOAD_ATTR:
+            return 1;
         case LOAD_ATTR:
             return 1;
         case LOAD_METHOD:
@@ -408,6 +470,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 2;
         case STORE_ATTR_SLOT:
             return 2;
+        case _SPECIALIZE_COMPARE_OP:
+            return 2;
+        case _COMPARE_OP:
+            return 2;
         case COMPARE_OP:
             return 2;
         case COMPARE_OP_FLOAT:
@@ -464,6 +530,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case GET_YIELD_FROM_ITER:
             return 1;
+        case _SPECIALIZE_FOR_ITER:
+            return 1;
+        case _FOR_ITER:
+            return 1;
         case FOR_ITER:
             return 1;
         case INSTRUMENTED_FOR_ITER:
@@ -544,6 +614,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 1;
         case INSTRUMENTED_CALL:
             return 0;
+        case _SPECIALIZE_CALL:
+            return oparg + 2;
+        case _CALL:
+            return oparg + 2;
         case CALL:
             return oparg + 2;
         case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS:
@@ -622,6 +696,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump)  {
             return 2;
         case COPY:
             return (oparg-1) + 1;
+        case _SPECIALIZE_BINARY_OP:
+            return 2;
+        case _BINARY_OP:
+            return 2;
         case BINARY_OP:
             return 2;
         case SWAP:
@@ -714,6 +792,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 1;
         case UNARY_NOT:
             return 1;
+        case _SPECIALIZE_TO_BOOL:
+            return 1;
+        case _TO_BOOL:
+            return 1;
         case TO_BOOL:
             return 1;
         case TO_BOOL_BOOL:
@@ -768,6 +850,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 0;
         case BINARY_OP_INPLACE_ADD_UNICODE:
             return 0;
+        case _SPECIALIZE_BINARY_SUBSCR:
+            return 2;
+        case _BINARY_SUBSCR:
+            return 1;
         case BINARY_SUBSCR:
             return 1;
         case BINARY_SLICE:
@@ -788,6 +874,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return (oparg-1) + 1;
         case SET_ADD:
             return (oparg-1) + 1;
+        case _SPECIALIZE_STORE_SUBSCR:
+            return 2;
+        case _STORE_SUBSCR:
+            return 0;
         case STORE_SUBSCR:
             return 0;
         case STORE_SUBSCR_LIST_INT:
@@ -820,6 +910,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 2;
         case GET_AWAITABLE:
             return 1;
+        case _SPECIALIZE_SEND:
+            return 2;
+        case _SEND:
+            return 2;
         case SEND:
             return 2;
         case SEND_GEN:
@@ -844,6 +938,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 0;
         case DELETE_NAME:
             return 0;
+        case _SPECIALIZE_UNPACK_SEQUENCE:
+            return 1;
+        case _UNPACK_SEQUENCE:
+            return oparg;
         case UNPACK_SEQUENCE:
             return oparg;
         case UNPACK_SEQUENCE_TWO_TUPLE:
@@ -854,6 +952,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return oparg;
         case UNPACK_EX:
             return (oparg & 0xFF) + (oparg >> 8) + 1;
+        case _SPECIALIZE_STORE_ATTR:
+            return 1;
+        case _STORE_ATTR:
+            return 0;
         case STORE_ATTR:
             return 0;
         case DELETE_ATTR:
@@ -868,8 +970,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 1;
         case LOAD_NAME:
             return 1;
-        case LOAD_GLOBAL:
+        case _SPECIALIZE_LOAD_GLOBAL:
+            return 0;
+        case _LOAD_GLOBAL:
             return ((oparg & 1) ? 1 : 0) + 1;
+        case LOAD_GLOBAL:
+            return (oparg & 1 ? 1 : 0) + 1;
         case _GUARD_GLOBALS_VERSION:
             return 0;
         case _GUARD_BUILTINS_VERSION:
@@ -922,22 +1028,30 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return (oparg - 1) + 1;
         case INSTRUMENTED_LOAD_SUPER_ATTR:
             return ((oparg & 1) ? 1 : 0) + 1;
+        case _SPECIALIZE_LOAD_SUPER_ATTR:
+            return 3;
+        case _LOAD_SUPER_ATTR:
+            return ((oparg & 1) ? 1 : 0) + 1;
         case LOAD_SUPER_ATTR:
-            return ((oparg & 1) ? 1 : 0) + 1;
+            return (oparg & 1 ? 1 : 0) + 1;
         case LOAD_SUPER_METHOD:
-            return ((oparg & 1) ? 1 : 0) + 1;
+            return (oparg & 1 ? 1 : 0) + 1;
         case LOAD_ZERO_SUPER_METHOD:
-            return ((oparg & 1) ? 1 : 0) + 1;
+            return (oparg & 1 ? 1 : 0) + 1;
         case LOAD_ZERO_SUPER_ATTR:
-            return ((oparg & 1) ? 1 : 0) + 1;
+            return (oparg & 1 ? 1 : 0) + 1;
         case LOAD_SUPER_ATTR_ATTR:
             return 1;
         case LOAD_SUPER_ATTR_METHOD:
             return 2;
+        case _SPECIALIZE_LOAD_ATTR:
+            return 1;
+        case _LOAD_ATTR:
+            return ((oparg & 1) ? 1 : 0) + 1;
         case LOAD_ATTR:
-            return ((oparg & 1) ? 1 : 0) + 1;
+            return (oparg & 1 ? 1 : 0) + 1;
         case LOAD_METHOD:
-            return ((oparg & 1) ? 1 : 0) + 1;
+            return (oparg & 1 ? 1 : 0) + 1;
         case _GUARD_TYPE_VERSION:
             return 1;
         case _CHECK_MANAGED_OBJECT_HAS_VALUES:
@@ -984,6 +1098,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 0;
         case STORE_ATTR_SLOT:
             return 0;
+        case _SPECIALIZE_COMPARE_OP:
+            return 2;
+        case _COMPARE_OP:
+            return 1;
         case COMPARE_OP:
             return 1;
         case COMPARE_OP_FLOAT:
@@ -1040,6 +1158,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 1;
         case GET_YIELD_FROM_ITER:
             return 1;
+        case _SPECIALIZE_FOR_ITER:
+            return 1;
+        case _FOR_ITER:
+            return 2;
         case FOR_ITER:
             return 2;
         case INSTRUMENTED_FOR_ITER:
@@ -1120,6 +1242,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 2;
         case INSTRUMENTED_CALL:
             return 0;
+        case _SPECIALIZE_CALL:
+            return oparg + 2;
+        case _CALL:
+            return 1;
         case CALL:
             return 1;
         case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS:
@@ -1198,6 +1324,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump)  {
             return 1;
         case COPY:
             return (oparg-1) + 2;
+        case _SPECIALIZE_BINARY_OP:
+            return 2;
+        case _BINARY_OP:
+            return 1;
         case BINARY_OP:
             return 1;
         case SWAP:
@@ -1248,6 +1378,7 @@ enum InstructionFormat {
     INSTR_FMT_IBC0,
     INSTR_FMT_IBC00,
     INSTR_FMT_IBC000,
+    INSTR_FMT_IBC0000000,
     INSTR_FMT_IBC00000000,
     INSTR_FMT_IX,
     INSTR_FMT_IXC,
@@ -1331,6 +1462,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [UNARY_NOT] = { true, INSTR_FMT_IX, 0 },
+    [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, 0 },
+    [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG },
     [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG },
     [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
     [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG },
@@ -1358,6 +1491,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
     [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
     [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 },
+    [_BINARY_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG },
     [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
@@ -1368,6 +1503,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
     [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, 0 },
+    [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG },
     [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
     [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
@@ -1384,6 +1521,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_SEND] = { true, INSTR_FMT_IXC, 0 },
+    [_SEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG },
     [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG },
     [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
@@ -1396,11 +1535,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG },
+    [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
@@ -1408,6 +1551,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_LOAD_GLOBAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG },
+    [_LOAD_GLOBAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
     [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG },
@@ -1435,12 +1580,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_SPECIALIZE_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_ZERO_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_ZERO_SUPER_ATTR] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
     [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_LOAD_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG },
+    [_LOAD_ATTR] = { true, INSTR_FMT_IBC0000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [LOAD_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG },
     [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG },
@@ -1466,6 +1615,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG },
     [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, 0 },
     [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG },
+    [_SPECIALIZE_COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_COMPARE_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
@@ -1494,6 +1645,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
+    [_SPECIALIZE_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG },
     [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG },
     [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG },
@@ -1534,6 +1687,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
     [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG },
+    [_SPECIALIZE_CALL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_CALL] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG },
     [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG },
     [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
     [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
@@ -1573,6 +1728,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
     [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
     [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
+    [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG },
+    [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG },
     [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
     [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG },
@@ -1614,7 +1771,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } },
     [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } },
     [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } },
-    [TO_BOOL] = { .nuops = 1, .uops = { { TO_BOOL, 0, 0 } } },
     [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } },
     [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } },
     [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } },
@@ -1629,7 +1785,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } },
     [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } },
     [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } },
-    [BINARY_SUBSCR] = { .nuops = 1, .uops = { { BINARY_SUBSCR, 0, 0 } } },
     [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } },
     [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } },
     [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } },
@@ -1638,7 +1793,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } },
     [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } },
     [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } },
-    [STORE_SUBSCR] = { .nuops = 1, .uops = { { STORE_SUBSCR, 0, 0 } } },
     [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } },
     [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } },
     [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } },
@@ -1654,19 +1808,17 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } },
     [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } },
     [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } },
-    [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE, 0, 0 } } },
+    [UNPACK_SEQUENCE] = { .nuops = 2, .uops = { { _SPECIALIZE_UNPACK_SEQUENCE, 1, 0 }, { _UNPACK_SEQUENCE, 0, 0 } } },
     [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } },
     [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } },
     [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } },
     [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } },
-    [STORE_ATTR] = { .nuops = 1, .uops = { { STORE_ATTR, 0, 0 } } },
     [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } },
     [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } },
     [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } },
     [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } },
     [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } },
     [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } },
-    [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } },
     [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } },
     [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } },
     [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } },
@@ -1690,7 +1842,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } },
     [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } },
     [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } },
-    [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } },
     [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } },
     [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } },
     [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } },
@@ -1698,7 +1849,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } },
     [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
     [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } },
-    [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } },
     [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } },
     [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } },
     [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } },
@@ -1744,7 +1894,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN
     [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } },
     [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } },
     [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } },
-    [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } },
     [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } },
 };
 #endif // NEED_OPCODE_METADATA
@@ -1754,6 +1903,8 @@ extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];
 const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = {
     [_EXIT_TRACE] = "_EXIT_TRACE",
     [_SET_IP] = "_SET_IP",
+    [_SPECIALIZE_TO_BOOL] = "_SPECIALIZE_TO_BOOL",
+    [_TO_BOOL] = "_TO_BOOL",
     [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT",
     [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT",
     [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT",
@@ -1765,11 +1916,27 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = {
     [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE",
     [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE",
     [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE",
+    [_SPECIALIZE_BINARY_SUBSCR] = "_SPECIALIZE_BINARY_SUBSCR",
+    [_BINARY_SUBSCR] = "_BINARY_SUBSCR",
+    [_SPECIALIZE_STORE_SUBSCR] = "_SPECIALIZE_STORE_SUBSCR",
+    [_STORE_SUBSCR] = "_STORE_SUBSCR",
     [_POP_FRAME] = "_POP_FRAME",
+    [_SPECIALIZE_SEND] = "_SPECIALIZE_SEND",
+    [_SEND] = "_SEND",
+    [_SPECIALIZE_UNPACK_SEQUENCE] = "_SPECIALIZE_UNPACK_SEQUENCE",
+    [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE",
+    [_SPECIALIZE_STORE_ATTR] = "_SPECIALIZE_STORE_ATTR",
+    [_STORE_ATTR] = "_STORE_ATTR",
+    [_SPECIALIZE_LOAD_GLOBAL] = "_SPECIALIZE_LOAD_GLOBAL",
+    [_LOAD_GLOBAL] = "_LOAD_GLOBAL",
     [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION",
     [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION",
     [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE",
     [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS",
+    [_SPECIALIZE_LOAD_SUPER_ATTR] = "_SPECIALIZE_LOAD_SUPER_ATTR",
+    [_LOAD_SUPER_ATTR] = "_LOAD_SUPER_ATTR",
+    [_SPECIALIZE_LOAD_ATTR] = "_SPECIALIZE_LOAD_ATTR",
+    [_LOAD_ATTR] = "_LOAD_ATTR",
     [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION",
     [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES",
     [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE",
@@ -1783,7 +1950,11 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = {
     [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES",
     [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE",
     [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT",
+    [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP",
+    [_COMPARE_OP] = "_COMPARE_OP",
     [_IS_NONE] = "_IS_NONE",
+    [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER",
+    [_FOR_ITER] = "_FOR_ITER",
     [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST",
     [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST",
     [_IS_ITER_EXHAUSTED_LIST] = "_IS_ITER_EXHAUSTED_LIST",
@@ -1804,6 +1975,8 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = {
     [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT",
     [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT",
     [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT",
+    [_SPECIALIZE_CALL] = "_SPECIALIZE_CALL",
+    [_CALL] = "_CALL",
     [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS",
     [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS",
     [_CHECK_PEP_523] = "_CHECK_PEP_523",
@@ -1811,6 +1984,8 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = {
     [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE",
     [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS",
     [_PUSH_FRAME] = "_PUSH_FRAME",
+    [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP",
+    [_BINARY_OP] = "_BINARY_OP",
     [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE",
     [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE",
     [_JUMP_TO_TOP] = "_JUMP_TO_TOP",

Lib/test/test_capi/test_misc.py

@@ -2595,7 +2595,7 @@ class TestUops(unittest.TestCase):
         ex = get_first_executor(testfunc)
         self.assertIsNotNone(ex)
         uops = {opname for opname, _, _ in ex}
-        self.assertIn("UNPACK_SEQUENCE", uops)
+        self.assertIn("_UNPACK_SEQUENCE", uops)
 
     def test_pop_jump_if_false(self):
         def testfunc(n):

Python/abstract_interp_cases.c.h

@@ -38,7 +38,7 @@
             break;
         }
 
-        case TO_BOOL: {
+        case _TO_BOOL: {
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
             break;
         }
@@ -113,7 +113,7 @@
             break;
         }
 
-        case BINARY_SUBSCR: {
+        case _BINARY_SUBSCR: {
             STACK_SHRINK(1);
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
             break;
@@ -164,7 +164,7 @@
             break;
         }
 
-        case STORE_SUBSCR: {
+        case _STORE_SUBSCR: {
             STACK_SHRINK(3);
             break;
         }
@@ -242,7 +242,11 @@
             break;
         }
 
-        case UNPACK_SEQUENCE: {
+        case _SPECIALIZE_UNPACK_SEQUENCE: {
+            break;
+        }
+
+        case _UNPACK_SEQUENCE: {
             STACK_SHRINK(1);
             STACK_GROW(oparg);
             break;
@@ -272,7 +276,7 @@
             break;
         }
 
-        case STORE_ATTR: {
+        case _STORE_ATTR: {
             STACK_SHRINK(2);
             break;
         }
@@ -308,7 +312,7 @@
             break;
         }
 
-        case LOAD_GLOBAL: {
+        case _LOAD_GLOBAL: {
             STACK_GROW(1);
             STACK_GROW(((oparg & 1) ? 1 : 0));
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true);
@@ -456,7 +460,7 @@
             break;
         }
 
-        case LOAD_ATTR: {
+        case _LOAD_ATTR: {
             STACK_GROW(((oparg & 1) ? 1 : 0));
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true);
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true);
@@ -532,7 +536,7 @@
             break;
         }
 
-        case COMPARE_OP: {
+        case _COMPARE_OP: {
             STACK_SHRINK(1);
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
             break;
@@ -901,7 +905,7 @@
             break;
         }
 
-        case BINARY_OP: {
+        case _BINARY_OP: {
             STACK_SHRINK(1);
             PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
             break;

Python/bytecodes.c

@@ -311,9 +311,10 @@ dummy_func(
             TO_BOOL_STR,
         };
 
-        inst(TO_BOOL, (unused/1, unused/2, value -- res)) {
+        op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_ToBool(value, next_instr);
                 DISPATCH_SAME_OPARG();
@@ -321,12 +322,17 @@ dummy_func(
             STAT_INC(TO_BOOL, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_TO_BOOL, (unused/2, value -- res)) {
             int err = PyObject_IsTrue(value);
             DECREF_INPUTS();
             ERROR_IF(err < 0, error);
             res = err ? Py_True : Py_False;
         }
 
+        macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + _TO_BOOL;
+
         inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {
             DEOPT_IF(!PyBool_Check(value));
             STAT_INC(TO_BOOL, hit);
@@ -530,9 +536,10 @@ dummy_func(
             BINARY_SUBSCR_TUPLE_INT,
         };
 
-        inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) {
+        op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_BinarySubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
@@ -540,11 +547,16 @@ dummy_func(
             STAT_INC(BINARY_SUBSCR, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_BINARY_SUBSCR, (container, sub -- res)) {
             res = PyObject_GetItem(container, sub);
             DECREF_INPUTS();
             ERROR_IF(res == NULL, error);
         }
 
+        macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR;
+
         inst(BINARY_SLICE, (container, start, stop -- res)) {
             PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
             // Can't use ERROR_IF() here, because we haven't
@@ -677,9 +689,10 @@ dummy_func(
             STORE_SUBSCR_LIST_INT,
         };
 
-        inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) {
+        op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_StoreSubscr(container, sub, next_instr);
                 DISPATCH_SAME_OPARG();
@@ -687,12 +700,17 @@ dummy_func(
             STAT_INC(STORE_SUBSCR, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_STORE_SUBSCR, (v, container, sub -- )) {
             /* container[sub] = v */
             int err = PyObject_SetItem(container, sub, v);
             DECREF_INPUTS();
             ERROR_IF(err, error);
         }
 
+        macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR;
+
         inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {
             DEOPT_IF(!PyLong_CheckExact(sub));
             DEOPT_IF(!PyList_CheckExact(list));
@@ -956,9 +974,10 @@ dummy_func(
             SEND_GEN,
         };
 
-        inst(SEND, (unused/1, receiver, v -- receiver, retval)) {
+        op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_Send(receiver, next_instr);
                 DISPATCH_SAME_OPARG();
@@ -966,6 +985,9 @@ dummy_func(
             STAT_INC(SEND, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_SEND, (receiver, v -- receiver, retval)) {
             assert(frame != &entry_frame);
             if ((tstate->interp->eval_frame == NULL) &&
                 (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) &&
@@ -1004,6 +1026,8 @@ dummy_func(
             Py_DECREF(v);
         }
 
+        macro(SEND) = _SPECIALIZE_SEND + _SEND;
+
         inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) {
             DEOPT_IF(tstate->interp->eval_frame);
             PyGenObject *gen = (PyGenObject *)receiver;
@@ -1182,9 +1206,9 @@ dummy_func(
             UNPACK_SEQUENCE_LIST,
         };
 
-        inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) {
+        op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) {
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
@@ -1192,12 +1216,17 @@ dummy_func(
             STAT_INC(UNPACK_SEQUENCE, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) {
             PyObject **top = stack_pointer + oparg - 1;
             int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
             DECREF_INPUTS();
             ERROR_IF(res == 0, error);
         }
 
+        macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE;
+
         inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) {
             DEOPT_IF(!PyTuple_CheckExact(seq));
             DEOPT_IF(PyTuple_GET_SIZE(seq) != 2);
@@ -1244,9 +1273,10 @@ dummy_func(
             STORE_ATTR_WITH_HINT,
         };
 
-        inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) {
+        op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
                 next_instr = this_instr;
                 _Py_Specialize_StoreAttr(owner, next_instr, name);
@@ -1255,12 +1285,17 @@ dummy_func(
             STAT_INC(STORE_ATTR, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_STORE_ATTR, (unused/3, v, owner --)) {
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(owner, name, v);
             DECREF_INPUTS();
             ERROR_IF(err, error);
         }
 
+        macro(STORE_ATTR) = _SPECIALIZE_STORE_ATTR + _STORE_ATTR;
+
         inst(DELETE_ATTR, (owner --)) {
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_DelAttr(owner, name);
@@ -1365,9 +1400,10 @@ dummy_func(
             LOAD_GLOBAL_BUILTIN,
         };
 
-        inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) {
+        op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
                 next_instr = this_instr;
                 _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
@@ -1376,6 +1412,9 @@ dummy_func(
             STAT_INC(LOAD_GLOBAL, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_LOAD_GLOBAL, (unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) {
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (PyDict_CheckExact(GLOBALS())
                 && PyDict_CheckExact(BUILTINS()))
@@ -1396,7 +1435,6 @@ dummy_func(
             }
             else {
                 /* Slow-path if globals or builtins is not a dict */
-
                 /* namespace 1: globals */
                 ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error);
                 if (res == NULL) {
@@ -1413,6 +1451,8 @@ dummy_func(
             null = NULL;
         }
 
+        macro(LOAD_GLOBAL) = _SPECIALIZE_LOAD_GLOBAL + _LOAD_GLOBAL;
+
         op(_GUARD_GLOBALS_VERSION, (version/1 --)) {
             PyDictObject *dict = (PyDictObject *)GLOBALS();
             DEOPT_IF(!PyDict_CheckExact(dict));
@@ -1701,12 +1741,11 @@ dummy_func(
             LOAD_SUPER_ATTR_METHOD,
         };
 
-        inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) {
+        op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) {
             TIER_ONE_ONLY
-            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
             #if ENABLE_SPECIALIZATION
             int load_method = oparg & 1;
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
                 DISPATCH_SAME_OPARG();
@@ -1714,7 +1753,10 @@ dummy_func(
             STAT_INC(LOAD_SUPER_ATTR, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
 
+        op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) {
+            TIER_ONE_ONLY
             if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
                 PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
                 int err = _Py_call_instrumentation_2args(
@@ -1722,7 +1764,6 @@ dummy_func(
                         frame, this_instr, global_super, arg);
                 ERROR_IF(err, error);
             }
-
             // we make no attempt to optimize here; specializations should
             // handle any case whose performance we care about
             PyObject *stack[] = {class, self};
@@ -1745,12 +1786,15 @@ dummy_func(
             }
             DECREF_INPUTS();
             ERROR_IF(super == NULL, error);
+            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
             attr = PyObject_GetAttr(super, name);
             Py_DECREF(super);
             ERROR_IF(attr == NULL, error);
             null = NULL;
         }
 
+        macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR;
+
         pseudo(LOAD_SUPER_METHOD) = {
             LOAD_SUPER_ATTR,
         };
@@ -1813,9 +1857,10 @@ dummy_func(
             LOAD_ATTR_NONDESCRIPTOR_NO_DICT,
         };
 
-        inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) {
+        op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
                 next_instr = this_instr;
                 _Py_Specialize_LoadAttr(owner, next_instr, name);
@@ -1824,6 +1869,9 @@ dummy_func(
             STAT_INC(LOAD_ATTR, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_LOAD_ATTR, (unused/8, owner -- attr, self_or_null if (oparg & 1))) {
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
             if (oparg & 1) {
                 /* Designed to work in tandem with CALL, pushes two values. */
@@ -1831,7 +1879,6 @@ dummy_func(
                 if (_PyObject_GetMethod(owner, name, &attr)) {
                     /* We can bypass temporary bound method object.
                        meth is unbound method and obj is self.
-
                        meth | self | arg1 | ... | argN
                      */
                     assert(attr != NULL);  // No errors on this branch
@@ -1842,7 +1889,6 @@ dummy_func(
                        something was returned by a descriptor protocol).  Set
                        the second element of the stack to NULL, to signal
                        CALL that it's not a method call.
-
                        NULL | meth | arg1 | ... | argN
                     */
                     DECREF_INPUTS();
@@ -1858,6 +1904,8 @@ dummy_func(
             }
         }
 
+        macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + _LOAD_ATTR;
+
         pseudo(LOAD_METHOD) = {
             LOAD_ATTR,
         };
@@ -2133,9 +2181,10 @@ dummy_func(
             COMPARE_OP_STR,
         };
 
-        inst(COMPARE_OP, (unused/1, left, right -- res)) {
+        op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_CompareOp(left, right, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
@@ -2143,6 +2192,9 @@ dummy_func(
             STAT_INC(COMPARE_OP, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_COMPARE_OP, (left, right -- res)) {
             assert((oparg >> 5) <= Py_GE);
             res = PyObject_RichCompare(left, right, oparg >> 5);
             DECREF_INPUTS();
@@ -2155,6 +2207,8 @@ dummy_func(
             }
         }
 
+        macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP;
+
         inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) {
             DEOPT_IF(!PyFloat_CheckExact(left));
             DEOPT_IF(!PyFloat_CheckExact(right));
@@ -2448,9 +2502,10 @@ dummy_func(
             FOR_ITER_GEN,
         };
 
-        inst(FOR_ITER, (unused/1, iter -- iter, next)) {
+        op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_ForIter(iter, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
@@ -2458,6 +2513,9 @@ dummy_func(
             STAT_INC(FOR_ITER, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+        }
+
+        op(_FOR_ITER, (iter -- iter, next)) {
             /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
             next = (*Py_TYPE(iter)->tp_iternext)(iter);
             if (next == NULL) {
@@ -2480,6 +2538,8 @@ dummy_func(
             // Common case: no jump, leave it to the code generator
         }
 
+        macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER;
+
         inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) {
             _Py_CODEUNIT *target;
             PyObject *iter = TOP();
@@ -2937,24 +2997,28 @@ dummy_func(
             CALL_ALLOC_AND_ENTER_INIT,
         };
 
+        op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
+            TIER_ONE_ONLY
+            #if ENABLE_SPECIALIZATION
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+                next_instr = this_instr;
+                _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
+                DISPATCH_SAME_OPARG();
+            }
+            STAT_INC(CALL, deferred);
+            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+            #endif  /* ENABLE_SPECIALIZATION */
+        }
+
         // When calling Python, inline the call using DISPATCH_INLINED().
-        inst(CALL, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) {
+        op(_CALL, (unused/2, callable, self_or_null, args[oparg] -- res)) {
             // oparg counts all of the args, but *not* self:
             int total_args = oparg;
             if (self_or_null != NULL) {
                 args--;
                 total_args++;
             }
-            #if ENABLE_SPECIALIZATION
+            else if (Py_TYPE(callable) == &PyMethod_Type) {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_Call(callable, next_instr, total_args);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(CALL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
                 args--;
                 total_args++;
                 PyObject *self = ((PyMethodObject *)callable)->im_self;
@@ -3017,6 +3081,8 @@ dummy_func(
             CHECK_EVAL_BREAKER();
         }
 
+        macro(CALL) = _SPECIALIZE_CALL + _CALL;
+
         op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) {
             DEOPT_IF(null != NULL);
             DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type);
@@ -3799,9 +3865,10 @@ dummy_func(
             top = Py_NewRef(bottom);
         }
 
-        inst(BINARY_OP, (unused/1, lhs, rhs -- res)) {
+        op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) {
+            TIER_ONE_ONLY
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
                 DISPATCH_SAME_OPARG();
@@ -3811,12 +3878,17 @@ dummy_func(
             #endif  /* ENABLE_SPECIALIZATION */
             assert(NB_ADD <= oparg);
             assert(oparg <= NB_INPLACE_XOR);
+        }
+
+        op(_BINARY_OP, (lhs, rhs -- res)) {
             assert(_PyEval_BinaryOps[oparg]);
             res = _PyEval_BinaryOps[oparg](lhs, rhs);
             DECREF_INPUTS();
             ERROR_IF(res == NULL, error);
         }
 
+        macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP;
+
         inst(SWAP, (bottom, unused[oparg-2], top --
                     top, unused[oparg-2], bottom)) {
             assert(oparg >= 2);

Python/executor_cases.c.h

@@ -114,19 +114,10 @@
             break;
         }
 
-        case TO_BOOL: {
+        case _TO_BOOL: {
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_ToBool(value, next_instr);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(TO_BOOL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             int err = PyObject_IsTrue(value);
             Py_DECREF(value);
             if (err < 0) goto pop_1_error;
@@ -372,21 +363,12 @@
             break;
         }
 
-        case BINARY_SUBSCR: {
+        case _BINARY_SUBSCR: {
             PyObject *sub;
             PyObject *container;
             PyObject *res;
             sub = stack_pointer[-1];
             container = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_BinarySubscr(container, sub, next_instr);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(BINARY_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             res = PyObject_GetItem(container, sub);
             Py_DECREF(container);
             Py_DECREF(sub);
@@ -564,22 +546,13 @@
             break;
         }
 
-        case STORE_SUBSCR: {
+        case _STORE_SUBSCR: {
             PyObject *sub;
             PyObject *container;
             PyObject *v;
             sub = stack_pointer[-1];
             container = stack_pointer[-2];
             v = stack_pointer[-3];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_StoreSubscr(container, sub, next_instr);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(STORE_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             /* container[sub] = v */
             int err = PyObject_SetItem(container, sub, v);
             Py_DECREF(v);
@@ -893,11 +866,12 @@
             break;
         }
 
-        case UNPACK_SEQUENCE: {
+        case _SPECIALIZE_UNPACK_SEQUENCE: {
             PyObject *seq;
             seq = stack_pointer[-1];
+            uint16_t counter = (uint16_t)operand;
             #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+            if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
                 next_instr = this_instr;
                 _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
                 DISPATCH_SAME_OPARG();
@@ -905,6 +879,12 @@
             STAT_INC(UNPACK_SEQUENCE, deferred);
             DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
             #endif  /* ENABLE_SPECIALIZATION */
+            break;
+        }
+
+        case _UNPACK_SEQUENCE: {
+            PyObject *seq;
+            seq = stack_pointer[-1];
             PyObject **top = stack_pointer + oparg - 1;
             int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
             Py_DECREF(seq);
@@ -979,21 +959,11 @@
             break;
         }
 
-        case STORE_ATTR: {
+        case _STORE_ATTR: {
             PyObject *owner;
             PyObject *v;
             owner = stack_pointer[-1];
             v = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
-                next_instr = this_instr;
-                _Py_Specialize_StoreAttr(owner, next_instr, name);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(STORE_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
             int err = PyObject_SetAttr(owner, name, v);
             Py_DECREF(v);
@@ -1124,19 +1094,9 @@
             break;
         }
 
-        case LOAD_GLOBAL: {
+        case _LOAD_GLOBAL: {
             PyObject *res;
             PyObject *null = NULL;
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr = this_instr;
-                _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(LOAD_GLOBAL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (PyDict_CheckExact(GLOBALS())
                 && PyDict_CheckExact(BUILTINS()))
@@ -1157,7 +1117,6 @@
             }
             else {
                 /* Slow-path if globals or builtins is not a dict */
-
                 /* namespace 1: globals */
                 if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error;
                 if (res == NULL) {
@@ -1624,21 +1583,11 @@
             break;
         }
 
-        case LOAD_ATTR: {
+        case _LOAD_ATTR: {
             PyObject *owner;
             PyObject *attr;
             PyObject *self_or_null = NULL;
             owner = stack_pointer[-1];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr = this_instr;
-                _Py_Specialize_LoadAttr(owner, next_instr, name);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(LOAD_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
             if (oparg & 1) {
                 /* Designed to work in tandem with CALL, pushes two values. */
@@ -1646,7 +1595,6 @@
                 if (_PyObject_GetMethod(owner, name, &attr)) {
                     /* We can bypass temporary bound method object.
                        meth is unbound method and obj is self.
-
                        meth | self | arg1 | ... | argN
                      */
                     assert(attr != NULL);  // No errors on this branch
@@ -1657,7 +1605,6 @@
                        something was returned by a descriptor protocol).  Set
                        the second element of the stack to NULL, to signal
                        CALL that it's not a method call.
-
                        NULL | meth | arg1 | ... | argN
                     */
                     Py_DECREF(owner);
@@ -1885,21 +1832,12 @@
             break;
         }
 
-        case COMPARE_OP: {
+        case _COMPARE_OP: {
             PyObject *right;
             PyObject *left;
             PyObject *res;
             right = stack_pointer[-1];
             left = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_CompareOp(left, right, next_instr, oparg);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(COMPARE_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
             assert((oparg >> 5) <= Py_GE);
             res = PyObject_RichCompare(left, right, oparg >> 5);
             Py_DECREF(left);
@@ -3246,23 +3184,12 @@
             break;
         }
 
-        case BINARY_OP: {
+        case _BINARY_OP: {
             PyObject *rhs;
             PyObject *lhs;
             PyObject *res;
             rhs = stack_pointer[-1];
             lhs = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(BINARY_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            assert(NB_ADD <= oparg);
-            assert(oparg <= NB_INPLACE_XOR);
             assert(_PyEval_BinaryOps[oparg]);
             res = _PyEval_BinaryOps[oparg](lhs, rhs);
             Py_DECREF(lhs);

Python/generated_cases.c.h

@@ -332,20 +332,28 @@
             static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
             PyObject *value;
             PyObject *res;
+            // _SPECIALIZE_TO_BOOL
             value = stack_pointer[-1];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                next_instr = this_instr;
+                TIER_ONE_ONLY
-                _Py_Specialize_ToBool(value, next_instr);
+                #if ENABLE_SPECIALIZATION
-                DISPATCH_SAME_OPARG();
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+                    next_instr = this_instr;
+                    _Py_Specialize_ToBool(value, next_instr);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(TO_BOOL, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _TO_BOOL
+            {
+                int err = PyObject_IsTrue(value);
+                Py_DECREF(value);
+                if (err < 0) goto pop_1_error;
+                res = err ? Py_True : Py_False;
             }
-            STAT_INC(TO_BOOL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            int err = PyObject_IsTrue(value);
-            Py_DECREF(value);
-            if (err < 0) goto pop_1_error;
-            res = err ? Py_True : Py_False;
             stack_pointer[-1] = res;
             DISPATCH();
         }
@@ -708,21 +716,29 @@
             PyObject *sub;
             PyObject *container;
             PyObject *res;
+            // _SPECIALIZE_BINARY_SUBSCR
             sub = stack_pointer[-1];
             container = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                next_instr = this_instr;
+                TIER_ONE_ONLY
-                _Py_Specialize_BinarySubscr(container, sub, next_instr);
+                #if ENABLE_SPECIALIZATION
-                DISPATCH_SAME_OPARG();
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+                    next_instr = this_instr;
+                    _Py_Specialize_BinarySubscr(container, sub, next_instr);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(BINARY_SUBSCR, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _BINARY_SUBSCR
+            {
+                res = PyObject_GetItem(container, sub);
+                Py_DECREF(container);
+                Py_DECREF(sub);
+                if (res == NULL) goto pop_2_error;
             }
-            STAT_INC(BINARY_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            res = PyObject_GetItem(container, sub);
-            Py_DECREF(container);
-            Py_DECREF(sub);
-            if (res == NULL) goto pop_2_error;
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             DISPATCH();
@@ -962,24 +978,32 @@
             PyObject *sub;
             PyObject *container;
             PyObject *v;
+            // _SPECIALIZE_STORE_SUBSCR
             sub = stack_pointer[-1];
             container = stack_pointer[-2];
-            v = stack_pointer[-3];
+            {
-            #if ENABLE_SPECIALIZATION
+                uint16_t counter = read_u16(&this_instr[1].cache);
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                TIER_ONE_ONLY
-                next_instr = this_instr;
+                #if ENABLE_SPECIALIZATION
-                _Py_Specialize_StoreSubscr(container, sub, next_instr);
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                DISPATCH_SAME_OPARG();
+                    next_instr = this_instr;
+                    _Py_Specialize_StoreSubscr(container, sub, next_instr);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(STORE_SUBSCR, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _STORE_SUBSCR
+            v = stack_pointer[-3];
+            {
+                /* container[sub] = v */
+                int err = PyObject_SetItem(container, sub, v);
+                Py_DECREF(v);
+                Py_DECREF(container);
+                Py_DECREF(sub);
+                if (err) goto pop_3_error;
             }
-            STAT_INC(STORE_SUBSCR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            /* container[sub] = v */
-            int err = PyObject_SetItem(container, sub, v);
-            Py_DECREF(v);
-            Py_DECREF(container);
-            Py_DECREF(sub);
-            if (err) goto pop_3_error;
             STACK_SHRINK(3);
             DISPATCH();
         }
@@ -1378,56 +1402,64 @@
             PREDICTED(SEND);
             _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size");
-            PyObject *v;
             PyObject *receiver;
+            PyObject *v;
             PyObject *retval;
-            v = stack_pointer[-1];
+            // _SPECIALIZE_SEND
             receiver = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_Send(receiver, next_instr);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(SEND, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            assert(frame != &entry_frame);
-            if ((tstate->interp->eval_frame == NULL) &&
-                (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) &&
-                ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING)
             {
-                PyGenObject *gen = (PyGenObject *)receiver;
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
+                TIER_ONE_ONLY
-                STACK_SHRINK(1);
+                #if ENABLE_SPECIALIZATION
-                _PyFrame_StackPush(gen_frame, v);
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                gen->gi_frame_state = FRAME_EXECUTING;
+                    next_instr = this_instr;
-                gen->gi_exc_state.previous_item = tstate->exc_info;
+                    _Py_Specialize_Send(receiver, next_instr);
-                tstate->exc_info = &gen->gi_exc_state;
+                    DISPATCH_SAME_OPARG();
-                assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr);
-                frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
-                DISPATCH_INLINED(gen_frame);
-            }
-            if (Py_IsNone(v) && PyIter_Check(receiver)) {
-                retval = Py_TYPE(receiver)->tp_iternext(receiver);
-            }
-            else {
-                retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
-            }
-            if (retval == NULL) {
-                if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
-                ) {
-                    monitor_raise(tstate, frame, this_instr);
                 }
-                if (_PyGen_FetchStopIterationValue(&retval) == 0) {
+                STAT_INC(SEND, deferred);
-                    assert(retval != NULL);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-                    JUMPBY(oparg);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _SEND
+            v = stack_pointer[-1];
+            {
+                assert(frame != &entry_frame);
+                if ((tstate->interp->eval_frame == NULL) &&
+                    (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) &&
+                    ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING)
+                {
+                    PyGenObject *gen = (PyGenObject *)receiver;
+                    _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
+                    STACK_SHRINK(1);
+                    _PyFrame_StackPush(gen_frame, v);
+                    gen->gi_frame_state = FRAME_EXECUTING;
+                    gen->gi_exc_state.previous_item = tstate->exc_info;
+                    tstate->exc_info = &gen->gi_exc_state;
+                    assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr);
+                    frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
+                    DISPATCH_INLINED(gen_frame);
+                }
+                if (Py_IsNone(v) && PyIter_Check(receiver)) {
+                    retval = Py_TYPE(receiver)->tp_iternext(receiver);
                 }
                 else {
-                    goto error;
+                    retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
                 }
+                if (retval == NULL) {
+                    if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
+                    ) {
+                        monitor_raise(tstate, frame, this_instr);
+                    }
+                    if (_PyGen_FetchStopIterationValue(&retval) == 0) {
+                        assert(retval != NULL);
+                        JUMPBY(oparg);
+                    }
+                    else {
+                        goto error;
+                    }
+                }
+                Py_DECREF(v);
             }
-            Py_DECREF(v);
             stack_pointer[-1] = retval;
             DISPATCH();
         }
@@ -1694,20 +1726,27 @@
             _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
             PyObject *seq;
+            // _SPECIALIZE_UNPACK_SEQUENCE
             seq = stack_pointer[-1];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                next_instr = this_instr;
+                #if ENABLE_SPECIALIZATION
-                _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                DISPATCH_SAME_OPARG();
+                    next_instr = this_instr;
+                    _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(UNPACK_SEQUENCE, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _UNPACK_SEQUENCE
+            {
+                PyObject **top = stack_pointer + oparg - 1;
+                int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
+                Py_DECREF(seq);
+                if (res == 0) goto pop_1_error;
             }
-            STAT_INC(UNPACK_SEQUENCE, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            PyObject **top = stack_pointer + oparg - 1;
-            int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
-            Py_DECREF(seq);
-            if (res == 0) goto pop_1_error;
             STACK_SHRINK(1);
             STACK_GROW(oparg);
             DISPATCH();
@@ -1799,23 +1838,31 @@
             static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size");
             PyObject *owner;
             PyObject *v;
+            // _SPECIALIZE_STORE_ATTR
             owner = stack_pointer[-1];
-            v = stack_pointer[-2];
+            {
-            #if ENABLE_SPECIALIZATION
+                uint16_t counter = read_u16(&this_instr[1].cache);
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                TIER_ONE_ONLY
-                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+                #if ENABLE_SPECIALIZATION
-                next_instr = this_instr;
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                _Py_Specialize_StoreAttr(owner, next_instr, name);
+                    PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
-                DISPATCH_SAME_OPARG();
+                    next_instr = this_instr;
+                    _Py_Specialize_StoreAttr(owner, next_instr, name);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(STORE_ATTR, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _STORE_ATTR
+            v = stack_pointer[-2];
+            {
+                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
+                int err = PyObject_SetAttr(owner, name, v);
+                Py_DECREF(v);
+                Py_DECREF(owner);
+                if (err) goto pop_2_error;
             }
-            STAT_INC(STORE_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
-            int err = PyObject_SetAttr(owner, name, v);
-            Py_DECREF(v);
-            Py_DECREF(owner);
-            if (err) goto pop_2_error;
             STACK_SHRINK(2);
             DISPATCH();
         }
@@ -1968,51 +2015,58 @@
             static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
             PyObject *res;
             PyObject *null = NULL;
-            #if ENABLE_SPECIALIZATION
+            // _SPECIALIZE_LOAD_GLOBAL
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                next_instr = this_instr;
-                _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(LOAD_GLOBAL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-            if (PyDict_CheckExact(GLOBALS())
-                && PyDict_CheckExact(BUILTINS()))
             {
-                res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                                         (PyDictObject *)BUILTINS(),
+                TIER_ONE_ONLY
-                                         name);
+                #if ENABLE_SPECIALIZATION
-                if (res == NULL) {
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                    if (!_PyErr_Occurred(tstate)) {
+                    PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                        /* _PyDict_LoadGlobal() returns NULL without raising
+                    next_instr = this_instr;
-                         * an exception if the key doesn't exist */
+                    _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
-                        _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+                    DISPATCH_SAME_OPARG();
-                                                  NAME_ERROR_MSG, name);
-                    }
-                    if (true) goto error;
                 }
-                Py_INCREF(res);
+                STAT_INC(LOAD_GLOBAL, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
             }
-            else {
+            // _LOAD_GLOBAL
-                /* Slow-path if globals or builtins is not a dict */
+            {
-
+                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
-                /* namespace 1: globals */
+                if (PyDict_CheckExact(GLOBALS())
-                if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error;
+                    && PyDict_CheckExact(BUILTINS()))
-                if (res == NULL) {
+                {
-                    /* namespace 2: builtins */
+                    res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
-                    if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error;
+                                             (PyDictObject *)BUILTINS(),
+                                             name);
                     if (res == NULL) {
-                        _PyEval_FormatExcCheckArg(
+                        if (!_PyErr_Occurred(tstate)) {
-                                    tstate, PyExc_NameError,
+                            /* _PyDict_LoadGlobal() returns NULL without raising
-                                    NAME_ERROR_MSG, name);
+                             * an exception if the key doesn't exist */
+                            _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+                                                      NAME_ERROR_MSG, name);
+                        }
                         if (true) goto error;
                     }
+                    Py_INCREF(res);
                 }
+                else {
+                    /* Slow-path if globals or builtins is not a dict */
+                    /* namespace 1: globals */
+                    if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error;
+                    if (res == NULL) {
+                        /* namespace 2: builtins */
+                        if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error;
+                        if (res == NULL) {
+                            _PyEval_FormatExcCheckArg(
+                                        tstate, PyExc_NameError,
+                                        NAME_ERROR_MSG, name);
+                            if (true) goto error;
+                        }
+                    }
+                }
+                null = NULL;
             }
-            null = NULL;
             STACK_GROW(1);
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res;
@@ -2498,63 +2552,69 @@
             PREDICTED(LOAD_SUPER_ATTR);
             _Py_CODEUNIT *this_instr = next_instr - 2;
             static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size");
-            PyObject *self;
             PyObject *class;
             PyObject *global_super;
+            PyObject *self;
             PyObject *attr;
             PyObject *null = NULL;
-            self = stack_pointer[-1];
+            // _SPECIALIZE_LOAD_SUPER_ATTR
             class = stack_pointer[-2];
             global_super = stack_pointer[-3];
-            TIER_ONE_ONLY
+            {
-            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
+                uint16_t counter = read_u16(&this_instr[1].cache);
-            #if ENABLE_SPECIALIZATION
+                TIER_ONE_ONLY
-            int load_method = oparg & 1;
+                #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                int load_method = oparg & 1;
-                next_instr = this_instr;
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
+                    next_instr = this_instr;
-                DISPATCH_SAME_OPARG();
+                    _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
-            }
+                    DISPATCH_SAME_OPARG();
-            STAT_INC(LOAD_SUPER_ATTR, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-
-            if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
-                PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
-                int err = _Py_call_instrumentation_2args(
-                        tstate, PY_MONITORING_EVENT_CALL,
-                        frame, this_instr, global_super, arg);
-                if (err) goto pop_3_error;
-            }
-
-            // we make no attempt to optimize here; specializations should
-            // handle any case whose performance we care about
-            PyObject *stack[] = {class, self};
-            PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
-            if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
-                PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
-                if (super == NULL) {
-                    _Py_call_instrumentation_exc2(
-                        tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, this_instr, global_super, arg);
                 }
-                else {
+                STAT_INC(LOAD_SUPER_ATTR, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+            }
+            // _LOAD_SUPER_ATTR
+            self = stack_pointer[-1];
+            {
+                TIER_ONE_ONLY
+                if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
+                    PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
                     int err = _Py_call_instrumentation_2args(
-                        tstate, PY_MONITORING_EVENT_C_RETURN,
+                            tstate, PY_MONITORING_EVENT_CALL,
-                        frame, this_instr, global_super, arg);
+                            frame, this_instr, global_super, arg);
-                    if (err < 0) {
+                    if (err) goto pop_3_error;
-                        Py_CLEAR(super);
+                }
+                // we make no attempt to optimize here; specializations should
+                // handle any case whose performance we care about
+                PyObject *stack[] = {class, self};
+                PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL);
+                if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) {
+                    PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING;
+                    if (super == NULL) {
+                        _Py_call_instrumentation_exc2(
+                            tstate, PY_MONITORING_EVENT_C_RAISE,
+                            frame, this_instr, global_super, arg);
+                    }
+                    else {
+                        int err = _Py_call_instrumentation_2args(
+                            tstate, PY_MONITORING_EVENT_C_RETURN,
+                            frame, this_instr, global_super, arg);
+                        if (err < 0) {
+                            Py_CLEAR(super);
+                        }
                     }
                 }
+                Py_DECREF(global_super);
+                Py_DECREF(class);
+                Py_DECREF(self);
+                if (super == NULL) goto pop_3_error;
+                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
+                attr = PyObject_GetAttr(super, name);
+                Py_DECREF(super);
+                if (attr == NULL) goto pop_3_error;
+                null = NULL;
             }
-            Py_DECREF(global_super);
-            Py_DECREF(class);
-            Py_DECREF(self);
-            if (super == NULL) goto pop_3_error;
-            attr = PyObject_GetAttr(super, name);
-            Py_DECREF(super);
-            if (attr == NULL) goto pop_3_error;
-            null = NULL;
             STACK_SHRINK(2);
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
@@ -2637,49 +2697,55 @@
             PyObject *owner;
             PyObject *attr;
             PyObject *self_or_null = NULL;
+            // _SPECIALIZE_LOAD_ATTR
             owner = stack_pointer[-1];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+                TIER_ONE_ONLY
-                next_instr = this_instr;
+                #if ENABLE_SPECIALIZATION
-                _Py_Specialize_LoadAttr(owner, next_instr, name);
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                DISPATCH_SAME_OPARG();
+                    PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+                    next_instr = this_instr;
+                    _Py_Specialize_LoadAttr(owner, next_instr, name);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(LOAD_ATTR, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
             }
-            STAT_INC(LOAD_ATTR, deferred);
+            // _LOAD_ATTR
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+            {
-            #endif  /* ENABLE_SPECIALIZATION */
+                PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
-            PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
+                if (oparg & 1) {
-            if (oparg & 1) {
+                    /* Designed to work in tandem with CALL, pushes two values. */
-                /* Designed to work in tandem with CALL, pushes two values. */
+                    attr = NULL;
-                attr = NULL;
+                    if (_PyObject_GetMethod(owner, name, &attr)) {
-                if (_PyObject_GetMethod(owner, name, &attr)) {
+                        /* We can bypass temporary bound method object.
-                    /* We can bypass temporary bound method object.
+                           meth is unbound method and obj is self.
-                       meth is unbound method and obj is self.
+                           meth | self | arg1 | ... | argN
-
+                         */
-                       meth | self | arg1 | ... | argN
+                        assert(attr != NULL);  // No errors on this branch
-                     */
+                        self_or_null = owner;  // Transfer ownership
-                    assert(attr != NULL);  // No errors on this branch
+                    }
-                    self_or_null = owner;  // Transfer ownership
+                    else {
+                        /* meth is not an unbound method (but a regular attr, or
+                           something was returned by a descriptor protocol).  Set
+                           the second element of the stack to NULL, to signal
+                           CALL that it's not a method call.
+                           NULL | meth | arg1 | ... | argN
+                        */
+                        Py_DECREF(owner);
+                        if (attr == NULL) goto pop_1_error;
+                        self_or_null = NULL;
+                    }
                 }
                 else {
-                    /* meth is not an unbound method (but a regular attr, or
+                    /* Classic, pushes one value. */
-                       something was returned by a descriptor protocol).  Set
+                    attr = PyObject_GetAttr(owner, name);
-                       the second element of the stack to NULL, to signal
-                       CALL that it's not a method call.
-
-                       NULL | meth | arg1 | ... | argN
-                    */
                     Py_DECREF(owner);
                     if (attr == NULL) goto pop_1_error;
-                    self_or_null = NULL;
                 }
             }
-            else {
-                /* Classic, pushes one value. */
-                attr = PyObject_GetAttr(owner, name);
-                Py_DECREF(owner);
-                if (attr == NULL) goto pop_1_error;
-            }
             STACK_GROW(((oparg & 1) ? 1 : 0));
             stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr;
             if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; }
@@ -3076,27 +3142,35 @@
             PyObject *right;
             PyObject *left;
             PyObject *res;
+            // _SPECIALIZE_COMPARE_OP
             right = stack_pointer[-1];
             left = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                next_instr = this_instr;
+                TIER_ONE_ONLY
-                _Py_Specialize_CompareOp(left, right, next_instr, oparg);
+                #if ENABLE_SPECIALIZATION
-                DISPATCH_SAME_OPARG();
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+                    next_instr = this_instr;
+                    _Py_Specialize_CompareOp(left, right, next_instr, oparg);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(COMPARE_OP, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
             }
-            STAT_INC(COMPARE_OP, deferred);
+            // _COMPARE_OP
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+            {
-            #endif  /* ENABLE_SPECIALIZATION */
+                assert((oparg >> 5) <= Py_GE);
-            assert((oparg >> 5) <= Py_GE);
+                res = PyObject_RichCompare(left, right, oparg >> 5);
-            res = PyObject_RichCompare(left, right, oparg >> 5);
+                Py_DECREF(left);
-            Py_DECREF(left);
+                Py_DECREF(right);
-            Py_DECREF(right);
+                if (res == NULL) goto pop_2_error;
-            if (res == NULL) goto pop_2_error;
+                if (oparg & 16) {
-            if (oparg & 16) {
+                    int res_bool = PyObject_IsTrue(res);
-                int res_bool = PyObject_IsTrue(res);
+                    Py_DECREF(res);
-                Py_DECREF(res);
+                    if (res_bool < 0) goto pop_2_error;
-                if (res_bool < 0) goto pop_2_error;
+                    res = res_bool ? Py_True : Py_False;
-                res = res_bool ? Py_True : Py_False;
+                }
             }
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
@@ -3630,36 +3704,44 @@
             static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size");
             PyObject *iter;
             PyObject *next;
+            // _SPECIALIZE_FOR_ITER
             iter = stack_pointer[-1];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                next_instr = this_instr;
+                TIER_ONE_ONLY
-                _Py_Specialize_ForIter(iter, next_instr, oparg);
+                #if ENABLE_SPECIALIZATION
-                DISPATCH_SAME_OPARG();
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-            }
+                    next_instr = this_instr;
-            STAT_INC(FOR_ITER, deferred);
+                    _Py_Specialize_ForIter(iter, next_instr, oparg);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                    DISPATCH_SAME_OPARG();
-            #endif  /* ENABLE_SPECIALIZATION */
-            /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
-            next = (*Py_TYPE(iter)->tp_iternext)(iter);
-            if (next == NULL) {
-                if (_PyErr_Occurred(tstate)) {
-                    if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
-                        goto error;
-                    }
-                    monitor_raise(tstate, frame, this_instr);
-                    _PyErr_Clear(tstate);
                 }
-                /* iterator ended normally */
+                STAT_INC(FOR_ITER, deferred);
-                assert(next_instr[oparg].op.code == END_FOR ||
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-                       next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
+                #endif  /* ENABLE_SPECIALIZATION */
-                Py_DECREF(iter);
+            }
-                STACK_SHRINK(1);
+            // _FOR_ITER
-                /* Jump forward oparg, then skip following END_FOR instruction */
+            {
-                JUMPBY(oparg + 1);
+                /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
-                DISPATCH();
+                next = (*Py_TYPE(iter)->tp_iternext)(iter);
+                if (next == NULL) {
+                    if (_PyErr_Occurred(tstate)) {
+                        if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
+                            goto error;
+                        }
+                        monitor_raise(tstate, frame, this_instr);
+                        _PyErr_Clear(tstate);
+                    }
+                    /* iterator ended normally */
+                    assert(next_instr[oparg].op.code == END_FOR ||
+                           next_instr[oparg].op.code == INSTRUMENTED_END_FOR);
+                    Py_DECREF(iter);
+                    STACK_SHRINK(1);
+                    /* Jump forward oparg, then skip following END_FOR instruction */
+                    JUMPBY(oparg + 1);
+                    DISPATCH();
+                }
+                // Common case: no jump, leave it to the code generator
             }
-            // Common case: no jump, leave it to the code generator
             STACK_GROW(1);
             stack_pointer[-1] = next;
             DISPATCH();
@@ -4211,84 +4293,92 @@
             PyObject *self_or_null;
             PyObject *callable;
             PyObject *res;
+            // _SPECIALIZE_CALL
             args = stack_pointer - oparg;
             self_or_null = stack_pointer[-1 - oparg];
             callable = stack_pointer[-2 - oparg];
-            // oparg counts all of the args, but *not* self:
-            int total_args = oparg;
-            if (self_or_null != NULL) {
-                args--;
-                total_args++;
-            }
-            #if ENABLE_SPECIALIZATION
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
-                next_instr = this_instr;
-                _Py_Specialize_Call(callable, next_instr, total_args);
-                DISPATCH_SAME_OPARG();
-            }
-            STAT_INC(CALL, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) {
-                args--;
-                total_args++;
-                PyObject *self = ((PyMethodObject *)callable)->im_self;
-                args[0] = Py_NewRef(self);
-                PyObject *method = ((PyMethodObject *)callable)->im_func;
-                args[-1] = Py_NewRef(method);
-                Py_DECREF(callable);
-                callable = method;
-            }
-            // Check if the call can be inlined or not
-            if (Py_TYPE(callable) == &PyFunction_Type &&
-                tstate->interp->eval_frame == NULL &&
-                ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
             {
-                int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+                TIER_ONE_ONLY
-                _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
+                #if ENABLE_SPECIALIZATION
-                    tstate, (PyFunctionObject *)callable, locals,
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
-                    args, total_args, NULL
+                    next_instr = this_instr;
-                );
+                    _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
-                // Manipulate stack directly since we leave using DISPATCH_INLINED().
+                    DISPATCH_SAME_OPARG();
-                STACK_SHRINK(oparg + 2);
-                // The frame has stolen all the arguments from the stack,
-                // so there is no need to clean them up.
-                if (new_frame == NULL) {
-                    goto error;
                 }
-                assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
+                STAT_INC(CALL, deferred);
-                frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-                DISPATCH_INLINED(new_frame);
+                #endif  /* ENABLE_SPECIALIZATION */
             }
-            /* Callable is not a normal Python function */
+            // _CALL
-            res = PyObject_Vectorcall(
+            {
-                callable, args,
+                // oparg counts all of the args, but *not* self:
-                total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+                int total_args = oparg;
-                NULL);
+                if (self_or_null != NULL) {
-            if (opcode == INSTRUMENTED_CALL) {
+                    args--;
-                PyObject *arg = total_args == 0 ?
+                    total_args++;
-                    &_PyInstrumentation_MISSING : args[0];
-                if (res == NULL) {
-                    _Py_call_instrumentation_exc2(
-                        tstate, PY_MONITORING_EVENT_C_RAISE,
-                        frame, this_instr, callable, arg);
                 }
-                else {
+                else if (Py_TYPE(callable) == &PyMethod_Type) {
-                    int err = _Py_call_instrumentation_2args(
+                    args--;
-                        tstate, PY_MONITORING_EVENT_C_RETURN,
+                    total_args++;
-                        frame, this_instr, callable, arg);
+                    PyObject *self = ((PyMethodObject *)callable)->im_self;
-                    if (err < 0) {
+                    args[0] = Py_NewRef(self);
-                        Py_CLEAR(res);
+                    PyObject *method = ((PyMethodObject *)callable)->im_func;
+                    args[-1] = Py_NewRef(method);
+                    Py_DECREF(callable);
+                    callable = method;
+                }
+                // Check if the call can be inlined or not
+                if (Py_TYPE(callable) == &PyFunction_Type &&
+                    tstate->interp->eval_frame == NULL &&
+                    ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall)
+                {
+                    int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags;
+                    PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable));
+                    _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
+                        tstate, (PyFunctionObject *)callable, locals,
+                        args, total_args, NULL
+                    );
+                    // Manipulate stack directly since we leave using DISPATCH_INLINED().
+                    STACK_SHRINK(oparg + 2);
+                    // The frame has stolen all the arguments from the stack,
+                    // so there is no need to clean them up.
+                    if (new_frame == NULL) {
+                        goto error;
+                    }
+                    assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr);
+                    frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
+                    DISPATCH_INLINED(new_frame);
+                }
+                /* Callable is not a normal Python function */
+                res = PyObject_Vectorcall(
+                    callable, args,
+                    total_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+                    NULL);
+                if (opcode == INSTRUMENTED_CALL) {
+                    PyObject *arg = total_args == 0 ?
+                        &_PyInstrumentation_MISSING : args[0];
+                    if (res == NULL) {
+                        _Py_call_instrumentation_exc2(
+                            tstate, PY_MONITORING_EVENT_C_RAISE,
+                            frame, this_instr, callable, arg);
+                    }
+                    else {
+                        int err = _Py_call_instrumentation_2args(
+                            tstate, PY_MONITORING_EVENT_C_RETURN,
+                            frame, this_instr, callable, arg);
+                        if (err < 0) {
+                            Py_CLEAR(res);
+                        }
                     }
                 }
+                assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+                Py_DECREF(callable);
+                for (int i = 0; i < total_args; i++) {
+                    Py_DECREF(args[i]);
+                }
+                if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
             }
-            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
-            Py_DECREF(callable);
-            for (int i = 0; i < total_args; i++) {
-                Py_DECREF(args[i]);
-            }
-            if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; }
             STACK_SHRINK(oparg);
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
@@ -5505,24 +5595,32 @@
             PyObject *rhs;
             PyObject *lhs;
             PyObject *res;
+            // _SPECIALIZE_BINARY_OP
             rhs = stack_pointer[-1];
             lhs = stack_pointer[-2];
-            #if ENABLE_SPECIALIZATION
+            {
-            if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) {
+                uint16_t counter = read_u16(&this_instr[1].cache);
-                next_instr = this_instr;
+                TIER_ONE_ONLY
-                _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
+                #if ENABLE_SPECIALIZATION
-                DISPATCH_SAME_OPARG();
+                if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+                    next_instr = this_instr;
+                    _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
+                    DISPATCH_SAME_OPARG();
+                }
+                STAT_INC(BINARY_OP, deferred);
+                DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
+                #endif  /* ENABLE_SPECIALIZATION */
+                assert(NB_ADD <= oparg);
+                assert(oparg <= NB_INPLACE_XOR);
+            }
+            // _BINARY_OP
+            {
+                assert(_PyEval_BinaryOps[oparg]);
+                res = _PyEval_BinaryOps[oparg](lhs, rhs);
+                Py_DECREF(lhs);
+                Py_DECREF(rhs);
+                if (res == NULL) goto pop_2_error;
             }
-            STAT_INC(BINARY_OP, deferred);
-            DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache);
-            #endif  /* ENABLE_SPECIALIZATION */
-            assert(NB_ADD <= oparg);
-            assert(oparg <= NB_INPLACE_XOR);
-            assert(_PyEval_BinaryOps[oparg]);
-            res = _PyEval_BinaryOps[oparg](lhs, rhs);
-            Py_DECREF(lhs);
-            Py_DECREF(rhs);
-            if (res == NULL) goto pop_2_error;
             STACK_SHRINK(1);
             stack_pointer[-1] = res;
             DISPATCH();

Tools/cases_generator/analysis.py

@@ -383,7 +383,12 @@ class Analyzer:
         return MacroInstruction(macro.name, format, flags, macro, parts, offset)
 
     def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction:
-        targets = [self.instrs[target] for target in pseudo.targets]
+        targets: list[Instruction | MacroInstruction] = []
+        for target_name in pseudo.targets:
+            if target_name in self.instrs:
+                targets.append(self.instrs[target_name])
+            else:
+                targets.append(self.macro_instrs[target_name])
         assert targets
         ignored_flags = {"HAS_EVAL_BREAK_FLAG", "HAS_DEOPT_FLAG", "HAS_ERROR_FLAG"}
         assert len({t.instr_flags.bitmap(ignore=ignored_flags) for t in targets}) == 1

Tools/cases_generator/generate_cases.py

@@ -178,17 +178,17 @@ class Generator(Analyzer):
                 # for all targets.
                 for target in self.pseudos[thing.name].targets:
                     target_instr = self.instrs.get(target)
-                    # Currently target is always an instr. This could change
+                    if target_instr is None:
-                    # in the future, e.g., if we have a pseudo targetting a
+                        macro_instr = self.macro_instrs[target]
-                    # macro instruction.
+                        popped, pushed = stacking.get_stack_effect_info_for_macro(macro_instr)
-                    assert target_instr
-                    target_popped = effect_str(target_instr.input_effects)
-                    target_pushed = effect_str(target_instr.output_effects)
-                    if popped is None:
-                        popped, pushed = target_popped, target_pushed
                     else:
-                        assert popped == target_popped
+                        target_popped = effect_str(target_instr.input_effects)
-                        assert pushed == target_pushed
+                        target_pushed = effect_str(target_instr.output_effects)
+                        if popped is None:
+                            popped, pushed = target_popped, target_pushed
+                        else:
+                            assert popped == target_popped
+                            assert pushed == target_pushed
             case _:
                 assert_never(thing)
         assert popped is not None and pushed is not None

Tools/cases_generator/instructions.py

@@ -285,7 +285,7 @@ class PseudoInstruction:
     """A pseudo instruction."""
 
     name: str
-    targets: list[Instruction]
+    targets: list[Instruction | MacroInstruction]
     instr_flags: InstructionFlags