GH-115685: Split _TO_BOOL_ALWAYS_TRUE into micro-ops (GH-116352)

Include/internal/pycore_opcode_metadata.h

@@ -1320,7 +1320,7 @@ _PyOpcode_macro_expansion[256] = {
     [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } },
     [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } },
     [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } },
-    [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { _TO_BOOL_ALWAYS_TRUE, 2, 1 } } },
+    [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, 0, 0 } } },
     [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } },
     [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } },
     [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } },

Include/internal/pycore_uop_ids.h

@@ -220,40 +220,40 @@ extern "C" {
 #define _PUSH_EXC_INFO PUSH_EXC_INFO
 #define _PUSH_FRAME 405
 #define _PUSH_NULL PUSH_NULL
+#define _REPLACE_WITH_TRUE 406
 #define _RESUME_CHECK RESUME_CHECK
-#define _SAVE_RETURN_OFFSET 406
+#define _SAVE_RETURN_OFFSET 407
-#define _SEND 407
+#define _SEND 408
 #define _SEND_GEN SEND_GEN
 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
 #define _SET_ADD SET_ADD
 #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
 #define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 408
+#define _START_EXECUTOR 409
-#define _STORE_ATTR 409
+#define _STORE_ATTR 410
-#define _STORE_ATTR_INSTANCE_VALUE 410
+#define _STORE_ATTR_INSTANCE_VALUE 411
-#define _STORE_ATTR_SLOT 411
+#define _STORE_ATTR_SLOT 412
 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT
 #define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 412
+#define _STORE_FAST 413
-#define _STORE_FAST_0 413
+#define _STORE_FAST_0 414
-#define _STORE_FAST_1 414
+#define _STORE_FAST_1 415
-#define _STORE_FAST_2 415
+#define _STORE_FAST_2 416
-#define _STORE_FAST_3 416
+#define _STORE_FAST_3 417
-#define _STORE_FAST_4 417
+#define _STORE_FAST_4 418
-#define _STORE_FAST_5 418
+#define _STORE_FAST_5 419
-#define _STORE_FAST_6 419
+#define _STORE_FAST_6 420
-#define _STORE_FAST_7 420
+#define _STORE_FAST_7 421
 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
 #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
 #define _STORE_GLOBAL STORE_GLOBAL
 #define _STORE_NAME STORE_NAME
 #define _STORE_SLICE STORE_SLICE
-#define _STORE_SUBSCR 421
+#define _STORE_SUBSCR 422
 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
 #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
 #define _SWAP SWAP
-#define _TO_BOOL 422
+#define _TO_BOOL 423
-#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE
 #define _TO_BOOL_BOOL TO_BOOL_BOOL
 #define _TO_BOOL_INT TO_BOOL_INT
 #define _TO_BOOL_LIST TO_BOOL_LIST
@@ -263,12 +263,12 @@ extern "C" {
 #define _UNARY_NEGATIVE UNARY_NEGATIVE
 #define _UNARY_NOT UNARY_NOT
 #define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 423
+#define _UNPACK_SEQUENCE 424
 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
 #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
 #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
 #define _WITH_EXCEPT_START WITH_EXCEPT_START
-#define MAX_UOP_ID 423
+#define MAX_UOP_ID 424
 
 #ifdef __cplusplus
 }

Include/internal/pycore_uop_metadata.h

@@ -54,7 +54,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
     [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
     [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
-    [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG,
+    [_REPLACE_WITH_TRUE] = 0,
     [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
     [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG,
     [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG,
@@ -430,6 +430,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO",
     [_PUSH_FRAME] = "_PUSH_FRAME",
     [_PUSH_NULL] = "_PUSH_NULL",
+    [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE",
     [_RESUME_CHECK] = "_RESUME_CHECK",
     [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET",
     [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS",
@@ -461,7 +462,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT",
     [_SWAP] = "_SWAP",
     [_TO_BOOL] = "_TO_BOOL",
-    [_TO_BOOL_ALWAYS_TRUE] = "_TO_BOOL_ALWAYS_TRUE",
     [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL",
     [_TO_BOOL_INT] = "_TO_BOOL_INT",
     [_TO_BOOL_LIST] = "_TO_BOOL_LIST",

Python/bytecodes.c

@@ -382,15 +382,16 @@ dummy_func(
             }
         }
 
-        inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) {
+        op(_REPLACE_WITH_TRUE, (value -- res)) {
-            // This one is a bit weird, because we expect *some* failures:
+            Py_DECREF(value);
-            assert(version);
-            EXIT_IF(Py_TYPE(value)->tp_version_tag != version);
-            STAT_INC(TO_BOOL, hit);
-            DECREF_INPUTS();
             res = Py_True;
         }
 
+        macro(TO_BOOL_ALWAYS_TRUE) =
+            unused/1 +
+            _GUARD_TYPE_VERSION +
+            _REPLACE_WITH_TRUE;
+
         inst(UNARY_INVERT, (value -- res)) {
             res = PyNumber_Invert(value);
             DECREF_INPUTS();

Python/executor_cases.c.h

@@ -383,15 +383,10 @@
             break;
         }
 
-        case _TO_BOOL_ALWAYS_TRUE: {
+        case _REPLACE_WITH_TRUE: {
             PyObject *value;
             PyObject *res;
             value = stack_pointer[-1];
-            uint32_t version = (uint32_t)CURRENT_OPERAND();
-            // This one is a bit weird, because we expect *some* failures:
-            assert(version);
-            if (Py_TYPE(value)->tp_version_tag != version) goto side_exit;
-            STAT_INC(TO_BOOL, hit);
             Py_DECREF(value);
             res = Py_True;
             stack_pointer[-1] = res;

Python/generated_cases.c.h

@@ -5580,17 +5580,24 @@
             next_instr += 4;
             INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE);
             static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
+            PyObject *owner;
             PyObject *value;
             PyObject *res;
             /* Skip 1 cache entry */
-            value = stack_pointer[-1];
+            // _GUARD_TYPE_VERSION
-            uint32_t version = read_u32(&this_instr[2].cache);
+            owner = stack_pointer[-1];
-            // This one is a bit weird, because we expect *some* failures:
+            {
-            assert(version);
+                uint32_t type_version = read_u32(&this_instr[2].cache);
-            DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
+                PyTypeObject *tp = Py_TYPE(owner);
-            STAT_INC(TO_BOOL, hit);
+                assert(type_version != 0);
-            Py_DECREF(value);
+                DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL);
-            res = Py_True;
+            }
+            // _REPLACE_WITH_TRUE
+            value = owner;
+            {
+                Py_DECREF(value);
+                res = Py_True;
+            }
             stack_pointer[-1] = res;
             DISPATCH();
         }

Python/optimizer_analysis.c

@@ -456,6 +456,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size)
                     last->opcode = _NOP;
                     buffer[pc].opcode = NOP;
                 }
+                if (last->opcode == _REPLACE_WITH_TRUE) {
+                    last->opcode = _NOP;
+                }
                 break;
             }
             case _JUMP_TO_TOP:

Python/optimizer_cases.c.h

@@ -203,7 +203,7 @@
             break;
         }
 
-        case _TO_BOOL_ALWAYS_TRUE: {
+        case _REPLACE_WITH_TRUE: {
             _Py_UopsSymbol *res;
             res = sym_new_unknown(ctx);
             if (res == NULL) goto out_of_space;