GH-128685: Specialize (rather than quicken) LOAD_CONST into LOAD_CONST_[IM]MORTAL (GH-128708)

Include/internal/pycore_magic_number.h

@@ -265,6 +265,7 @@ Known values:
     Python 3.14a4 3610 (Add VALUE_WITH_FAKE_GLOBALS format to annotationlib)
     Python 3.14a4 3611 (Add NOT_TAKEN instruction)
     Python 3.14a4 3612 (Add POP_ITER and INSTRUMENTED_POP_ITER)
+    Python 3.14a4 3613 (Add LOAD_CONST_MORTAL instruction)
 
     Python 3.15 will start with 3650
 
@@ -277,7 +278,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3612
+#define PYC_MAGIC_NUMBER 3613
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \

Include/internal/pycore_opcode_metadata.h

@@ -321,6 +321,8 @@ int _PyOpcode_num_popped(int opcode, int oparg)  {
             return 0;
         case LOAD_CONST_IMMORTAL:
             return 0;
+        case LOAD_CONST_MORTAL:
+            return 0;
         case LOAD_DEREF:
             return 0;
         case LOAD_FAST:
@@ -788,6 +790,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg)  {
             return 1;
         case LOAD_CONST_IMMORTAL:
             return 1;
+        case LOAD_CONST_MORTAL:
+            return 1;
         case LOAD_DEREF:
             return 1;
         case LOAD_FAST:
@@ -1559,6 +1563,10 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect)  {
             *effect = 1;
             return 0;
         }
+        case LOAD_CONST_MORTAL: {
+            *effect = 1;
+            return 0;
+        }
         case LOAD_DEREF: {
             *effect = 1;
             return 0;
@@ -2109,8 +2117,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = {
     [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
     [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
     [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
-    [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG },
+    [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },
     [LOAD_CONST_IMMORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },
+    [LOAD_CONST_MORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG },
     [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
     [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG },
     [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
@@ -2317,8 +2326,8 @@ _PyOpcode_macro_expansion[256] = {
     [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } },
     [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } },
     [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, 0, 0 } } },
-    [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, 0, 0 } } },
     [LOAD_CONST_IMMORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_IMMORTAL, 0, 0 } } },
+    [LOAD_CONST_MORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_MORTAL, 0, 0 } } },
     [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, 0, 0 } } },
     [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, 0, 0 } } },
     [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, 0, 0 } } },
@@ -2541,6 +2550,7 @@ const char *_PyOpcode_OpName[266] = {
     [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT",
     [LOAD_CONST] = "LOAD_CONST",
     [LOAD_CONST_IMMORTAL] = "LOAD_CONST_IMMORTAL",
+    [LOAD_CONST_MORTAL] = "LOAD_CONST_MORTAL",
     [LOAD_DEREF] = "LOAD_DEREF",
     [LOAD_FAST] = "LOAD_FAST",
     [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR",
@@ -2796,6 +2806,7 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT,
     [LOAD_CONST] = LOAD_CONST,
     [LOAD_CONST_IMMORTAL] = LOAD_CONST,
+    [LOAD_CONST_MORTAL] = LOAD_CONST,
     [LOAD_DEREF] = LOAD_DEREF,
     [LOAD_FAST] = LOAD_FAST,
     [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR,
@@ -2912,7 +2923,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
     case 146: \
     case 147: \
     case 148: \
-    case 228: \
     case 229: \
     case 230: \
     case 231: \

Include/internal/pycore_uop_ids.h

@@ -206,6 +206,7 @@ extern "C" {
 #define _LOAD_CONST_INLINE_BORROW 430
 #define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431
 #define _LOAD_CONST_INLINE_WITH_NULL 432
+#define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL
 #define _LOAD_DEREF LOAD_DEREF
 #define _LOAD_FAST 433
 #define _LOAD_FAST_0 434

Include/internal/pycore_uop_metadata.h

@@ -35,7 +35,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG,
     [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG,
     [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG,
-    [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG,
+    [_LOAD_CONST_MORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG,
     [_LOAD_CONST_IMMORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG,
     [_LOAD_SMALL_INT_0] = 0,
     [_LOAD_SMALL_INT_1] = 0,
@@ -474,12 +474,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT",
     [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS",
     [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT",
-    [_LOAD_CONST] = "_LOAD_CONST",
     [_LOAD_CONST_IMMORTAL] = "_LOAD_CONST_IMMORTAL",
     [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE",
     [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW",
     [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = "_LOAD_CONST_INLINE_BORROW_WITH_NULL",
     [_LOAD_CONST_INLINE_WITH_NULL] = "_LOAD_CONST_INLINE_WITH_NULL",
+    [_LOAD_CONST_MORTAL] = "_LOAD_CONST_MORTAL",
     [_LOAD_DEREF] = "_LOAD_DEREF",
     [_LOAD_FAST] = "_LOAD_FAST",
     [_LOAD_FAST_0] = "_LOAD_FAST_0",
@@ -617,7 +617,7 @@ int _PyUop_num_popped(int opcode, int oparg)
             return 0;
         case _LOAD_FAST_LOAD_FAST:
             return 0;
-        case _LOAD_CONST:
+        case _LOAD_CONST_MORTAL:
             return 0;
         case _LOAD_CONST_IMMORTAL:
             return 0;

Include/opcode_ids.h

@@ -187,26 +187,27 @@ extern "C" {
 #define LOAD_ATTR_SLOT                         205
 #define LOAD_ATTR_WITH_HINT                    206
 #define LOAD_CONST_IMMORTAL                    207
-#define LOAD_GLOBAL_BUILTIN                    208
+#define LOAD_CONST_MORTAL                      208
-#define LOAD_GLOBAL_MODULE                     209
+#define LOAD_GLOBAL_BUILTIN                    209
-#define LOAD_SUPER_ATTR_ATTR                   210
+#define LOAD_GLOBAL_MODULE                     210
-#define LOAD_SUPER_ATTR_METHOD                 211
+#define LOAD_SUPER_ATTR_ATTR                   211
-#define RESUME_CHECK                           212
+#define LOAD_SUPER_ATTR_METHOD                 212
-#define SEND_GEN                               213
+#define RESUME_CHECK                           213
-#define STORE_ATTR_INSTANCE_VALUE              214
+#define SEND_GEN                               214
-#define STORE_ATTR_SLOT                        215
+#define STORE_ATTR_INSTANCE_VALUE              215
-#define STORE_ATTR_WITH_HINT                   216
+#define STORE_ATTR_SLOT                        216
-#define STORE_SUBSCR_DICT                      217
+#define STORE_ATTR_WITH_HINT                   217
-#define STORE_SUBSCR_LIST_INT                  218
+#define STORE_SUBSCR_DICT                      218
-#define TO_BOOL_ALWAYS_TRUE                    219
+#define STORE_SUBSCR_LIST_INT                  219
-#define TO_BOOL_BOOL                           220
+#define TO_BOOL_ALWAYS_TRUE                    220
-#define TO_BOOL_INT                            221
+#define TO_BOOL_BOOL                           221
-#define TO_BOOL_LIST                           222
+#define TO_BOOL_INT                            222
-#define TO_BOOL_NONE                           223
+#define TO_BOOL_LIST                           223
-#define TO_BOOL_STR                            224
+#define TO_BOOL_NONE                           224
-#define UNPACK_SEQUENCE_LIST                   225
+#define TO_BOOL_STR                            225
-#define UNPACK_SEQUENCE_TUPLE                  226
+#define UNPACK_SEQUENCE_LIST                   226
-#define UNPACK_SEQUENCE_TWO_TUPLE              227
+#define UNPACK_SEQUENCE_TUPLE                  227
+#define UNPACK_SEQUENCE_TWO_TUPLE              228
 #define INSTRUMENTED_END_FOR                   235
 #define INSTRUMENTED_POP_ITER                  236
 #define INSTRUMENTED_END_SEND                  237

Lib/_opcode_metadata.py

@@ -7,6 +7,7 @@ _specializations = {
         "RESUME_CHECK",
     ],
     "LOAD_CONST": [
+        "LOAD_CONST_MORTAL",
         "LOAD_CONST_IMMORTAL",
     ],
     "TO_BOOL": [
@@ -178,26 +179,27 @@ _specialized_opmap = {
     'LOAD_ATTR_SLOT': 205,
     'LOAD_ATTR_WITH_HINT': 206,
     'LOAD_CONST_IMMORTAL': 207,
-    'LOAD_GLOBAL_BUILTIN': 208,
+    'LOAD_CONST_MORTAL': 208,
-    'LOAD_GLOBAL_MODULE': 209,
+    'LOAD_GLOBAL_BUILTIN': 209,
-    'LOAD_SUPER_ATTR_ATTR': 210,
+    'LOAD_GLOBAL_MODULE': 210,
-    'LOAD_SUPER_ATTR_METHOD': 211,
+    'LOAD_SUPER_ATTR_ATTR': 211,
-    'RESUME_CHECK': 212,
+    'LOAD_SUPER_ATTR_METHOD': 212,
-    'SEND_GEN': 213,
+    'RESUME_CHECK': 213,
-    'STORE_ATTR_INSTANCE_VALUE': 214,
+    'SEND_GEN': 214,
-    'STORE_ATTR_SLOT': 215,
+    'STORE_ATTR_INSTANCE_VALUE': 215,
-    'STORE_ATTR_WITH_HINT': 216,
+    'STORE_ATTR_SLOT': 216,
-    'STORE_SUBSCR_DICT': 217,
+    'STORE_ATTR_WITH_HINT': 217,
-    'STORE_SUBSCR_LIST_INT': 218,
+    'STORE_SUBSCR_DICT': 218,
-    'TO_BOOL_ALWAYS_TRUE': 219,
+    'STORE_SUBSCR_LIST_INT': 219,
-    'TO_BOOL_BOOL': 220,
+    'TO_BOOL_ALWAYS_TRUE': 220,
-    'TO_BOOL_INT': 221,
+    'TO_BOOL_BOOL': 221,
-    'TO_BOOL_LIST': 222,
+    'TO_BOOL_INT': 222,
-    'TO_BOOL_NONE': 223,
+    'TO_BOOL_LIST': 223,
-    'TO_BOOL_STR': 224,
+    'TO_BOOL_NONE': 224,
-    'UNPACK_SEQUENCE_LIST': 225,
+    'TO_BOOL_STR': 225,
-    'UNPACK_SEQUENCE_TUPLE': 226,
+    'UNPACK_SEQUENCE_LIST': 226,
-    'UNPACK_SEQUENCE_TWO_TUPLE': 227,
+    'UNPACK_SEQUENCE_TUPLE': 227,
+    'UNPACK_SEQUENCE_TWO_TUPLE': 228,
 }
 
 opmap = {

Lib/test/test_dis.py

@@ -892,7 +892,7 @@ dis_loop_test_quickened_code = """\
 %3d           RESUME_CHECK             0
 
 %3d           BUILD_LIST               0
-              LOAD_CONST               0 ((1, 2, 3))
+              LOAD_CONST_MORTAL        0 ((1, 2, 3))
               LIST_EXTEND              1
               LOAD_SMALL_INT           3
               BINARY_OP                5 (*)
@@ -2548,7 +2548,7 @@ class TestDisCLI(unittest.TestCase):
         expect = '''
             0           RESUME                   0
 
-            1           LOAD_CONST_IMMORTAL      0 (None)
+            1           LOAD_CONST               0 (None)
                         RETURN_VALUE
         '''
         for flag in ['-S', '--specialized']:

Python/bytecodes.c

@@ -285,11 +285,25 @@ dummy_func(
         }
 
         family(LOAD_CONST, 0) = {
+            LOAD_CONST_MORTAL,
             LOAD_CONST_IMMORTAL,
         };
 
-        pure inst(LOAD_CONST, (-- value)) {
+        inst(LOAD_CONST, (-- value)) {
-            value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
+            /* We can't do this in the bytecode compiler as
+             * marshalling can intern strings and make them immortal. */
+            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
+            value = PyStackRef_FromPyObjectNew(obj);
+#if ENABLE_SPECIALIZATION
+            if (this_instr->op.code == LOAD_CONST) {
+                this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL;
+            }
+#endif
+        }
+
+        inst(LOAD_CONST_MORTAL, (-- value)) {
+            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
+            value = PyStackRef_FromPyObjectNew(obj);
         }
 
         inst(LOAD_CONST_IMMORTAL, (-- value)) {

Python/executor_cases.c.h

@@ -209,10 +209,13 @@
             break;
         }
 
-        case _LOAD_CONST: {
+        /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
+
+        case _LOAD_CONST_MORTAL: {
             _PyStackRef value;
             oparg = CURRENT_OPARG();
-            value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
+            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
+            value = PyStackRef_FromPyObjectNew(obj);
             stack_pointer[0] = value;
             stack_pointer += 1;
             assert(WITHIN_STACK_BOUNDS());

Python/generated_cases.c.h

@@ -5913,8 +5913,18 @@
             next_instr += 1;
             INSTRUCTION_STATS(LOAD_CONST);
             PREDICTED(LOAD_CONST);
+            _Py_CODEUNIT* const this_instr = next_instr - 1;
+            (void)this_instr;
             _PyStackRef value;
-            value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg));
+            /* We can't do this in the bytecode compiler as
+             * marshalling can intern strings and make them immortal. */
+            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
+            value = PyStackRef_FromPyObjectNew(obj);
+            #if ENABLE_SPECIALIZATION
+            if (this_instr->op.code == LOAD_CONST) {
+                this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL;
+            }
+            #endif
             stack_pointer[0] = value;
             stack_pointer += 1;
             assert(WITHIN_STACK_BOUNDS());
@@ -5936,6 +5946,20 @@
             DISPATCH();
         }
 
+        TARGET(LOAD_CONST_MORTAL) {
+            frame->instr_ptr = next_instr;
+            next_instr += 1;
+            INSTRUCTION_STATS(LOAD_CONST_MORTAL);
+            static_assert(0 == 0, "incorrect cache size");
+            _PyStackRef value;
+            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
+            value = PyStackRef_FromPyObjectNew(obj);
+            stack_pointer[0] = value;
+            stack_pointer += 1;
+            assert(WITHIN_STACK_BOUNDS());
+            DISPATCH();
+        }
+
         TARGET(LOAD_DEREF) {
             frame->instr_ptr = next_instr;
             next_instr += 1;

Python/opcode_targets.h

@@ -207,6 +207,7 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_ATTR_SLOT,
     &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LOAD_CONST_IMMORTAL,
+    &&TARGET_LOAD_CONST_MORTAL,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_GLOBAL_MODULE,
     &&TARGET_LOAD_SUPER_ATTR_ATTR,
@@ -233,7 +234,6 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
-    &&_unknown_opcode,
     &&TARGET_INSTRUMENTED_END_FOR,
     &&TARGET_INSTRUMENTED_POP_ITER,
     &&TARGET_INSTRUMENTED_END_SEND,

Python/optimizer_bytecodes.c

@@ -479,6 +479,13 @@ dummy_func(void) {
         value = sym_new_const(ctx, val);
     }
 
+    op(_LOAD_CONST_MORTAL, (-- value)) {
+        PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg);
+        int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE;
+        REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val);
+        value = sym_new_const(ctx, val);
+    }
+
     op(_LOAD_CONST_IMMORTAL, (-- value)) {
         PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg);
         REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);

Python/optimizer_cases.c.h

@@ -58,7 +58,9 @@
             break;
         }
 
-        case _LOAD_CONST: {
+        /* _LOAD_CONST is not a viable micro-op for tier 2 */
+
+        case _LOAD_CONST_MORTAL: {
             _Py_UopsSymbol *value;
             PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg);
             int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE;

Python/specialize.c

@@ -478,15 +478,6 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts,
             }
             i += caches;
         }
-        else if (opcode == LOAD_CONST) {
-            /* We can't do this in the bytecode compiler as
-             * marshalling can intern strings and make them immortal. */
-
-            PyObject *obj = PyTuple_GET_ITEM(consts, oparg);
-            if (_Py_IsImmortal(obj)) {
-                instructions[i].op.code = LOAD_CONST_IMMORTAL;
-            }
-        }
         if (opcode != EXTENDED_ARG) {
             oparg = 0;
         }