GH-98686: Quicken everything (GH-98687)

Include/cpython/code.h

@@ -70,7 +70,6 @@ typedef struct {
     PyObject *co_exceptiontable;   /* Byte string encoding exception handling  \
                                       table */                                 \
     int co_flags;                  /* CO_..., see below */                     \
-    short co_warmup;                 /* Warmup counter for quickening */       \
     short _co_linearray_entry_size;  /* Size of each entry in _co_linearray */ \
                                                                                \
     /* The rest are not so impactful on performance. */                        \

Include/internal/pycore_code.h

@@ -91,28 +91,8 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
 
-#define QUICKENING_WARMUP_DELAY 8
-
-/* We want to compare to zero for efficiency, so we offset values accordingly */
-#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
-
-void _PyCode_Quicken(PyCodeObject *code);
-
-static inline void
-_PyCode_Warmup(PyCodeObject *code)
-{
-    if (code->co_warmup != 0) {
-        code->co_warmup++;
-        if (code->co_warmup == 0) {
-            _PyCode_Quicken(code);
-        }
-    }
-}
-
 extern uint8_t _PyOpcode_Adaptive[256];
 
-extern Py_ssize_t _Py_QuickenedCount;
-
 // Borrowed references to common callables:
 struct callable_cache {
     PyObject *isinstance;
@@ -252,10 +232,10 @@ extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
                                           int oparg);
 extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr);
 
-/* Deallocator function for static codeobjects used in deepfreeze.py */
+/* Finalizer function for static codeobjects used in deepfreeze.py */
-extern void _PyStaticCode_Dealloc(PyCodeObject *co);
+extern void _PyStaticCode_Fini(PyCodeObject *co);
-/* Function to intern strings of codeobjects */
+/* Function to intern strings of codeobjects and quicken the bytecode */
-extern int _PyStaticCode_InternStrings(PyCodeObject *co);
+extern int _PyStaticCode_Init(PyCodeObject *co);
 
 #ifdef Py_STATS
 
@@ -397,8 +377,8 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 
 /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
 #define ADAPTIVE_BACKOFF_BITS 4
-/* The initial counter value is 31 == 2**ADAPTIVE_BACKOFF_START - 1 */
+/* The initial counter value is 1 == 2**ADAPTIVE_BACKOFF_START - 1 */
-#define ADAPTIVE_BACKOFF_START 5
+#define ADAPTIVE_BACKOFF_START 1
 
 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)
 

Include/internal/pycore_opcode.h

@@ -142,7 +142,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [IS_OP] = IS_OP,
     [JUMP_BACKWARD] = JUMP_BACKWARD,
     [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT,
-    [JUMP_BACKWARD_QUICK] = JUMP_BACKWARD,
     [JUMP_FORWARD] = JUMP_FORWARD,
     [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP,
     [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP,
@@ -200,7 +199,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [RAISE_VARARGS] = RAISE_VARARGS,
     [RERAISE] = RERAISE,
     [RESUME] = RESUME,
-    [RESUME_QUICK] = RESUME,
     [RETURN_GENERATOR] = RETURN_GENERATOR,
     [RETURN_VALUE] = RETURN_VALUE,
     [SEND] = SEND,
@@ -313,21 +311,21 @@ static const char *const _PyOpcode_OpName[263] = {
     [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER",
     [PRINT_EXPR] = "PRINT_EXPR",
     [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS",
-    [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK",
     [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE",
+    [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
     [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR",
     [RETURN_GENERATOR] = "RETURN_GENERATOR",
-    [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
     [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
     [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE",
     [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
     [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY",
     [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT",
+    [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
     [LIST_TO_TUPLE] = "LIST_TO_TUPLE",
     [RETURN_VALUE] = "RETURN_VALUE",
     [IMPORT_STAR] = "IMPORT_STAR",
     [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS",
-    [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
+    [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
     [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP",
     [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR",
     [POP_EXCEPT] = "POP_EXCEPT",
@@ -354,7 +352,7 @@ static const char *const _PyOpcode_OpName[263] = {
     [JUMP_FORWARD] = "JUMP_FORWARD",
     [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP",
     [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP",
-    [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
+    [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
     [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE",
     [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE",
     [LOAD_GLOBAL] = "LOAD_GLOBAL",
@@ -362,7 +360,7 @@ static const char *const _PyOpcode_OpName[263] = {
     [CONTAINS_OP] = "CONTAINS_OP",
     [RERAISE] = "RERAISE",
     [COPY] = "COPY",
-    [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
+    [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
     [BINARY_OP] = "BINARY_OP",
     [SEND] = "SEND",
     [LOAD_FAST] = "LOAD_FAST",
@@ -382,9 +380,9 @@ static const char *const _PyOpcode_OpName[263] = {
     [STORE_DEREF] = "STORE_DEREF",
     [DELETE_DEREF] = "DELETE_DEREF",
     [JUMP_BACKWARD] = "JUMP_BACKWARD",
-    [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
-    [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
     [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES",
+    [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
+    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
     [EXTENDED_ARG] = "EXTENDED_ARG",
     [LIST_APPEND] = "LIST_APPEND",
     [SET_ADD] = "SET_ADD",
@@ -394,28 +392,26 @@ static const char *const _PyOpcode_OpName[263] = {
     [YIELD_VALUE] = "YIELD_VALUE",
     [RESUME] = "RESUME",
     [MATCH_CLASS] = "MATCH_CLASS",
-    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
     [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
+    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [FORMAT_VALUE] = "FORMAT_VALUE",
     [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP",
     [BUILD_STRING] = "BUILD_STRING",
-    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE",
     [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
     [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE",
+    [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
     [LIST_EXTEND] = "LIST_EXTEND",
     [SET_UPDATE] = "SET_UPDATE",
     [DICT_MERGE] = "DICT_MERGE",
     [DICT_UPDATE] = "DICT_UPDATE",
-    [RESUME_QUICK] = "RESUME_QUICK",
-    [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
     [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
     [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT",
     [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT",
-    [CALL] = "CALL",
-    [KW_NAMES] = "KW_NAMES",
     [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
     [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
+    [CALL] = "CALL",
+    [KW_NAMES] = "KW_NAMES",
     [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE",
     [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
     [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
@@ -423,6 +419,8 @@ static const char *const _PyOpcode_OpName[263] = {
     [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
     [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
     [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
+    [180] = "<180>",
+    [181] = "<181>",
     [182] = "<182>",
     [183] = "<183>",
     [184] = "<184>",
@@ -508,6 +506,8 @@ static const char *const _PyOpcode_OpName[263] = {
 #endif
 
 #define EXTRA_CASES \
+    case 180: \
+    case 181: \
     case 182: \
     case 183: \
     case 184: \

Include/opcode.h

@@ -167,39 +167,37 @@ extern "C" {
 #define FOR_ITER_ADAPTIVE                       65
 #define FOR_ITER_LIST                           66
 #define FOR_ITER_RANGE                          67
-#define JUMP_BACKWARD_QUICK                     72
+#define LOAD_ATTR_ADAPTIVE                      72
-#define LOAD_ATTR_ADAPTIVE                      73
+#define LOAD_ATTR_CLASS                         73
-#define LOAD_ATTR_CLASS                         76
+#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN       76
-#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN       77
+#define LOAD_ATTR_INSTANCE_VALUE                77
-#define LOAD_ATTR_INSTANCE_VALUE                78
+#define LOAD_ATTR_MODULE                        78
-#define LOAD_ATTR_MODULE                        79
+#define LOAD_ATTR_PROPERTY                      79
-#define LOAD_ATTR_PROPERTY                      80
+#define LOAD_ATTR_SLOT                          80
-#define LOAD_ATTR_SLOT                          81
+#define LOAD_ATTR_WITH_HINT                     81
-#define LOAD_ATTR_WITH_HINT                     86
+#define LOAD_ATTR_METHOD_LAZY_DICT              86
-#define LOAD_ATTR_METHOD_LAZY_DICT             113
+#define LOAD_ATTR_METHOD_NO_DICT               113
-#define LOAD_ATTR_METHOD_NO_DICT               121
+#define LOAD_ATTR_METHOD_WITH_DICT             121
-#define LOAD_ATTR_METHOD_WITH_DICT             141
+#define LOAD_ATTR_METHOD_WITH_VALUES           141
-#define LOAD_ATTR_METHOD_WITH_VALUES           143
+#define LOAD_CONST__LOAD_FAST                  143
-#define LOAD_CONST__LOAD_FAST                  153
+#define LOAD_FAST__LOAD_CONST                  153
-#define LOAD_FAST__LOAD_CONST                  154
+#define LOAD_FAST__LOAD_FAST                   154
-#define LOAD_FAST__LOAD_FAST                   158
+#define LOAD_GLOBAL_ADAPTIVE                   158
-#define LOAD_GLOBAL_ADAPTIVE                   159
+#define LOAD_GLOBAL_BUILTIN                    159
-#define LOAD_GLOBAL_BUILTIN                    160
+#define LOAD_GLOBAL_MODULE                     160
-#define LOAD_GLOBAL_MODULE                     161
+#define STORE_ATTR_ADAPTIVE                    161
-#define RESUME_QUICK                           166
+#define STORE_ATTR_INSTANCE_VALUE              166
-#define STORE_ATTR_ADAPTIVE                    167
+#define STORE_ATTR_SLOT                        167
-#define STORE_ATTR_INSTANCE_VALUE              168
+#define STORE_ATTR_WITH_HINT                   168
-#define STORE_ATTR_SLOT                        169
+#define STORE_FAST__LOAD_FAST                  169
-#define STORE_ATTR_WITH_HINT                   170
+#define STORE_FAST__STORE_FAST                 170
-#define STORE_FAST__LOAD_FAST                  173
+#define STORE_SUBSCR_ADAPTIVE                  173
-#define STORE_FAST__STORE_FAST                 174
+#define STORE_SUBSCR_DICT                      174
-#define STORE_SUBSCR_ADAPTIVE                  175
+#define STORE_SUBSCR_LIST_INT                  175
-#define STORE_SUBSCR_DICT                      176
+#define UNPACK_SEQUENCE_ADAPTIVE               176
-#define STORE_SUBSCR_LIST_INT                  177
+#define UNPACK_SEQUENCE_LIST                   177
-#define UNPACK_SEQUENCE_ADAPTIVE               178
+#define UNPACK_SEQUENCE_TUPLE                  178
-#define UNPACK_SEQUENCE_LIST                   179
+#define UNPACK_SEQUENCE_TWO_TUPLE              179
-#define UNPACK_SEQUENCE_TUPLE                  180
-#define UNPACK_SEQUENCE_TWO_TUPLE              181
 #define DO_TRACING                             255
 
 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\

Lib/opcode.py

@@ -327,9 +327,6 @@ _specializations = {
         "FOR_ITER_LIST",
         "FOR_ITER_RANGE",
     ],
-    "JUMP_BACKWARD": [
-        "JUMP_BACKWARD_QUICK",
-    ],
     "LOAD_ATTR": [
         "LOAD_ATTR_ADAPTIVE",
         # These potentially push [NULL, bound method] onto the stack.
@@ -358,9 +355,6 @@ _specializations = {
         "LOAD_GLOBAL_BUILTIN",
         "LOAD_GLOBAL_MODULE",
     ],
-    "RESUME": [
-        "RESUME_QUICK",
-    ],
     "STORE_ATTR": [
         "STORE_ATTR_ADAPTIVE",
         "STORE_ATTR_INSTANCE_VALUE",

Lib/test/libregrtest/refleak.py

@@ -73,7 +73,6 @@ def dash_R(ns, test_name, test_func):
     fd_deltas = [0] * repcount
     getallocatedblocks = sys.getallocatedblocks
     gettotalrefcount = sys.gettotalrefcount
-    _getquickenedcount = sys._getquickenedcount
     fd_count = os_helper.fd_count
     # initialize variables to make pyflakes quiet
     rc_before = alloc_before = fd_before = 0
@@ -93,7 +92,7 @@ def dash_R(ns, test_name, test_func):
         support.gc_collect()
 
         # Read memory statistics immediately after the garbage collection
-        alloc_after = getallocatedblocks() - _getquickenedcount()
+        alloc_after = getallocatedblocks()
         rc_after = gettotalrefcount()
         fd_after = fd_count()
 

Lib/test/test_call.py

@@ -580,7 +580,7 @@ def testfunction_kw(self, *, kw):
     return self
 
 
-QUICKENING_WARMUP_DELAY = 8
+ADAPTIVE_WARMUP_DELAY = 2
 
 
 class TestPEP590(unittest.TestCase):
@@ -771,7 +771,7 @@ class TestPEP590(unittest.TestCase):
         assert_equal(11, f(num))
         function_setvectorcall(f)
         # make sure specializer is triggered by running > 50 times
-        for _ in range(10 * QUICKENING_WARMUP_DELAY):
+        for _ in range(10 * ADAPTIVE_WARMUP_DELAY):
             assert_equal("overridden", f(num))
 
     def test_setvectorcall_load_attr_specialization_skip(self):
@@ -787,7 +787,7 @@ class TestPEP590(unittest.TestCase):
         function_setvectorcall(X.__getattribute__)
         # make sure specialization doesn't trigger
         # when vectorcall is overridden
-        for _ in range(QUICKENING_WARMUP_DELAY):
+        for _ in range(ADAPTIVE_WARMUP_DELAY):
             assert_equal("overridden", x.a)
 
     def test_setvectorcall_load_attr_specialization_deopt(self):
@@ -803,12 +803,12 @@ class TestPEP590(unittest.TestCase):
         assert_equal = self.assertEqual
         x = X()
         # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization
-        for _ in range(QUICKENING_WARMUP_DELAY):
+        for _ in range(ADAPTIVE_WARMUP_DELAY):
             assert_equal("a", get_a(x))
         function_setvectorcall(X.__getattribute__)
         # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
         # gets deopted due to overridden vectorcall
-        for _ in range(QUICKENING_WARMUP_DELAY):
+        for _ in range(ADAPTIVE_WARMUP_DELAY):
             assert_equal("overridden", get_a(x))
 
     @requires_limited_api

Lib/test/test_dis.py

@@ -713,7 +713,7 @@ def load_test(x, y=0):
     return a, b
 
 dis_load_test_quickened_code = """\
-%3d           0 RESUME_QUICK             0
+%3d           0 RESUME                   0
 
 %3d           2 LOAD_FAST__LOAD_FAST     0 (x)
               4 LOAD_FAST                1 (y)
@@ -733,7 +733,7 @@ def loop_test():
         load_test(i)
 
 dis_loop_test_quickened_code = """\
-%3d        RESUME_QUICK             0
+%3d        RESUME                   0
 
 %3d        BUILD_LIST               0
            LOAD_CONST               1 ((1, 2, 3))
@@ -748,7 +748,7 @@ dis_loop_test_quickened_code = """\
            LOAD_FAST                0 (i)
            CALL_PY_WITH_DEFAULTS     1
            POP_TOP
-           JUMP_BACKWARD_QUICK     17 (to 16)
+           JUMP_BACKWARD           17 (to 16)
 
 %3d     >> END_FOR
            LOAD_CONST               0 (None)
@@ -774,7 +774,7 @@ dis_extended_arg_quick_code = """\
 """% (extended_arg_quick.__code__.co_firstlineno,
       extended_arg_quick.__code__.co_firstlineno + 1,)
 
-QUICKENING_WARMUP_DELAY = 8
+ADAPTIVE_WARMUP_DELAY = 2
 
 class DisTestBase(unittest.TestCase):
     "Common utilities for DisTests and TestDisTraceback"
@@ -1079,7 +1079,7 @@ class DisTests(DisTestBase):
         check(dis_nested_2)
 
     @staticmethod
-    def code_quicken(f, times=QUICKENING_WARMUP_DELAY):
+    def code_quicken(f, times=ADAPTIVE_WARMUP_DELAY):
         for _ in range(times):
             f()
 
@@ -1092,7 +1092,7 @@ class DisTests(DisTestBase):
     @cpython_only
     def test_binary_specialize(self):
         binary_op_quicken = """\
-  0           0 RESUME_QUICK             0
+  0           0 RESUME                   0
 
   1           2 LOAD_NAME                0 (a)
               4 LOAD_NAME                1 (b)
@@ -1110,7 +1110,7 @@ class DisTests(DisTestBase):
         self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE     0 (+)", True)
 
         binary_subscr_quicken = """\
-  0           0 RESUME_QUICK             0
+  0           0 RESUME                   0
 
   1           2 LOAD_NAME                0 (a)
               4 LOAD_CONST               0 (0)
@@ -1130,7 +1130,7 @@ class DisTests(DisTestBase):
     @cpython_only
     def test_load_attr_specialize(self):
         load_attr_quicken = """\
-  0           0 RESUME_QUICK             0
+  0           0 RESUME                   0
 
   1           2 LOAD_CONST               0 ('a')
               4 LOAD_ATTR_SLOT           0 (__class__)
@@ -1144,7 +1144,7 @@ class DisTests(DisTestBase):
     @cpython_only
     def test_call_specialize(self):
         call_quicken = """\
-  0        RESUME_QUICK             0
+  0        RESUME                   0
 
   1        PUSH_NULL
            LOAD_NAME                0 (str)
@@ -1190,7 +1190,7 @@ class DisTests(DisTestBase):
         for quickened in (False, True):
             for adaptive in (False, True):
                 with self.subTest(f"{quickened=}, {adaptive=}"):
-                    if quickened and adaptive:
+                    if adaptive:
                         pattern = r"^(\w+: \d+)?$"
                     else:
                         pattern = r"^(\w+: 0)?$"
@@ -1198,11 +1198,10 @@ class DisTests(DisTestBase):
                     for cache in caches:
                         self.assertRegex(cache, pattern)
                     total_caches = 23
-                    empty_caches = 8 if adaptive and quickened else total_caches
+                    empty_caches = 8 if adaptive else total_caches
                     self.assertEqual(caches.count(""), empty_caches)
                     self.assertEqual(len(caches), total_caches)
 
-
 class DisWithFileTests(DisTests):
 
     # Run the tests again, using the file arg instead of print

Lib/test/test_embed.py

@@ -346,33 +346,41 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
         out, err = self.run_embedded_interpreter("test_repeated_simple_init")
         self.assertEqual(out, 'Finalized\n' * INIT_LOOPS)
 
-    def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self):
+    def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self):
         # https://github.com/python/cpython/issues/92031
 
-        # Do these imports outside of the code string to avoid using
+        code = textwrap.dedent("""\
-        # importlib too much from within the code string, so that
+            import dis
-        # _handle_fromlist doesn't get quickened until we intend it to.
-        from dis import _all_opmap
-        resume = _all_opmap["RESUME"]
-        resume_quick = _all_opmap["RESUME_QUICK"]
-        from test.test_dis import QUICKENING_WARMUP_DELAY
-
-        code = textwrap.dedent(f"""\
             import importlib._bootstrap
+            import opcode
+            import test.test_dis
+
+            def is_specialized(f):
+                for instruction in dis.get_instructions(f, adaptive=True):
+                    opname = instruction.opname
+                    if (
+                        opname in opcode._specialized_instructions
+                        # Exclude superinstructions:
+                        and "__" not in opname
+                        # Exclude adaptive instructions:
+                        and not opname.endswith("_ADAPTIVE")
+                        # Exclude "quick" instructions:
+                        and not opname.endswith("_QUICK")
+                    ):
+                        return True
+                return False
+
             func = importlib._bootstrap._handle_fromlist
-            code = func.__code__
 
-            # Assert initially unquickened.
+            # "copy" the code to un-specialize it:
-            # Use sets to account for byte order.
+            func.__code__ = func.__code__.replace()
-            if set(code._co_code_adaptive[:2]) != set([{resume}, 0]):
-                raise AssertionError()
 
-            for i in range({QUICKENING_WARMUP_DELAY}):
+            assert not is_specialized(func), "specialized instructions found"
+
+            for i in range(test.test_dis.ADAPTIVE_WARMUP_DELAY):
                 func(importlib._bootstrap, ["x"], lambda *args: None)
 
-            # Assert quickening worked
+            assert is_specialized(func), "no specialized instructions found"
-            if set(code._co_code_adaptive[:2]) != set([{resume_quick}, 0]):
-                raise AssertionError()
 
             print("Tests passed")
         """)

Lib/test/test_lltrace.py

@@ -8,7 +8,7 @@ from test.support.script_helper import assert_python_ok
 
 def example():
     x = []
-    for i in range(1):
+    for i in range(0):
         x.append(i)
     x = "this is"
     y = "an example"
@@ -75,7 +75,7 @@ class TestLLTrace(unittest.TestCase):
         self.assertIn('this is an example', stdout)
 
         # check that offsets match the output of dis.dis()
-        instr_map = {i.offset: i for i in dis.get_instructions(example)}
+        instr_map = {i.offset: i for i in dis.get_instructions(example, adaptive=True)}
         for line in stdout.splitlines():
             offset, colon, opname_oparg = line.partition(":")
             if not colon:

Misc/NEWS.d/next/Core and Builtins/2022-10-16-13-26-46.gh-issue-98686.D9Gu_Q.rst

@@ -0,0 +1,2 @@
+Quicken all code objects, and specialize adaptive bytecode instructions more
+aggressively.

Objects/codeobject.c

@@ -301,6 +301,8 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
     return 0;
 }
 
+extern void _PyCode_Quicken(PyCodeObject *code);
+
 static void
 init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
 {
@@ -353,7 +355,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
     co->co_extra = NULL;
     co->_co_cached = NULL;
 
-    co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
     co->_co_linearray_entry_size = 0;
     co->_co_linearray = NULL;
     memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code),
@@ -364,6 +365,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
         entry_point++;
     }
     co->_co_firsttraceable = entry_point;
+    _PyCode_Quicken(co);
 }
 
 static int
@@ -1664,9 +1666,6 @@ code_dealloc(PyCodeObject *co)
     if (co->_co_linearray) {
         PyMem_Free(co->_co_linearray);
     }
-    if (co->co_warmup == 0) {
-        _Py_QuickenedCount--;
-    }
     PyObject_Free(co);
 }
 
@@ -2224,13 +2223,9 @@ _PyCode_ConstantKey(PyObject *op)
 }
 
 void
-_PyStaticCode_Dealloc(PyCodeObject *co)
+_PyStaticCode_Fini(PyCodeObject *co)
 {
-    if (co->co_warmup == 0) {
-         _Py_QuickenedCount--;
-    }
     deopt_code(_PyCode_CODE(co), Py_SIZE(co));
-    co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
     PyMem_Free(co->co_extra);
     if (co->_co_cached != NULL) {
         Py_CLEAR(co->_co_cached->_co_code);
@@ -2252,7 +2247,7 @@ _PyStaticCode_Dealloc(PyCodeObject *co)
 }
 
 int
-_PyStaticCode_InternStrings(PyCodeObject *co)
+_PyStaticCode_Init(PyCodeObject *co)
 {
     int res = intern_strings(co->co_names);
     if (res < 0) {
@@ -2266,5 +2261,6 @@ _PyStaticCode_InternStrings(PyCodeObject *co)
     if (res < 0) {
         return -1;
     }
+    _PyCode_Quicken(co);
     return 0;
 }

Objects/frameobject.c

@@ -603,7 +603,7 @@ _PyFrame_GetState(PyFrameObject *frame)
             if (_PyInterpreterFrame_LASTI(frame->f_frame) < 0) {
                 return FRAME_CREATED;
             }
-            switch (_PyOpcode_Deopt[_Py_OPCODE(*frame->f_frame->prev_instr)])
+            switch (_Py_OPCODE(*frame->f_frame->prev_instr))
             {
                 case COPY_FREE_VARS:
                 case MAKE_CELL:

Objects/genobject.c

@@ -8,7 +8,6 @@
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
-#include "pycore_opcode.h"        // _PyOpcode_Deopt
 #include "pycore_pyerrors.h"      // _PyErr_ClearExcState()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "structmember.h"         // PyMemberDef
@@ -364,7 +363,7 @@ _PyGen_yf(PyGenObject *gen)
             return NULL;
         }
         _Py_CODEUNIT next = frame->prev_instr[1];
-        if (_PyOpcode_Deopt[_Py_OPCODE(next)] != RESUME || _Py_OPARG(next) < 2)
+        if (_Py_OPCODE(next) != RESUME || _Py_OPARG(next) < 2)
         {
             /* Not in a yield from */
             return NULL;

Python/ceval.c

@@ -143,7 +143,7 @@ lltrace_instruction(_PyInterpreterFrame *frame,
     const char *opname = _PyOpcode_OpName[opcode];
     assert(opname != NULL);
     int offset = (int)(next_instr - _PyCode_CODE(frame->f_code));
-    if (HAS_ARG(opcode)) {
+    if (HAS_ARG(_PyOpcode_Deopt[opcode])) {
         printf("%d: %s %d\n", offset * 2, opname, oparg);
     }
     else {
@@ -1165,14 +1165,7 @@ handle_eval_breaker:
         TARGET(NOP) {
             DISPATCH();
         }
-
         TARGET(RESUME) {
-            _PyCode_Warmup(frame->f_code);
-            GO_TO_INSTRUCTION(RESUME_QUICK);
-        }
-
-        TARGET(RESUME_QUICK) {
-            PREDICTED(RESUME_QUICK);
             assert(tstate->cframe == &cframe);
             assert(frame == cframe.current_frame);
             if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
@@ -1710,7 +1703,7 @@ handle_eval_breaker:
             PyObject *list = PEEK(oparg);
             if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
                 goto error;
-            PREDICT(JUMP_BACKWARD_QUICK);
+            PREDICT(JUMP_BACKWARD);
             DISPATCH();
         }
 
@@ -1722,7 +1715,7 @@ handle_eval_breaker:
             Py_DECREF(v);
             if (err != 0)
                 goto error;
-            PREDICT(JUMP_BACKWARD_QUICK);
+            PREDICT(JUMP_BACKWARD);
             DISPATCH();
         }
 
@@ -2908,7 +2901,7 @@ handle_eval_breaker:
             if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) {
                 goto error;
             }
-            PREDICT(JUMP_BACKWARD_QUICK);
+            PREDICT(JUMP_BACKWARD);
             DISPATCH();
         }
 
@@ -3576,8 +3569,11 @@ handle_eval_breaker:
         }
 
         TARGET(JUMP_BACKWARD) {
-            _PyCode_Warmup(frame->f_code);
+            PREDICTED(JUMP_BACKWARD);
-            GO_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
+            assert(oparg < INSTR_OFFSET());
+            JUMPBY(-oparg);
+            CHECK_EVAL_BREAKER();
+            DISPATCH();
         }
 
         TARGET(POP_JUMP_IF_FALSE) {
@@ -3707,14 +3703,6 @@ handle_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_BACKWARD_QUICK) {
-            PREDICTED(JUMP_BACKWARD_QUICK);
-            assert(oparg < INSTR_OFFSET());
-            JUMPBY(-oparg);
-            CHECK_EVAL_BREAKER();
-            DISPATCH();
-        }
-
         TARGET(GET_LEN) {
             // PUSH(len(TOS))
             Py_ssize_t len_i = PyObject_Length(TOP());

Python/clinic/sysmodule.c.h

@@ -884,33 +884,6 @@ exit:
 
 #endif /* defined(Py_REF_DEBUG) */
 
-PyDoc_STRVAR(sys__getquickenedcount__doc__,
-"_getquickenedcount($module, /)\n"
-"--\n"
-"\n");
-
-#define SYS__GETQUICKENEDCOUNT_METHODDEF    \
-    {"_getquickenedcount", (PyCFunction)sys__getquickenedcount, METH_NOARGS, sys__getquickenedcount__doc__},
-
-static Py_ssize_t
-sys__getquickenedcount_impl(PyObject *module);
-
-static PyObject *
-sys__getquickenedcount(PyObject *module, PyObject *Py_UNUSED(ignored))
-{
-    PyObject *return_value = NULL;
-    Py_ssize_t _return_value;
-
-    _return_value = sys__getquickenedcount_impl(module);
-    if ((_return_value == -1) && PyErr_Occurred()) {
-        goto exit;
-    }
-    return_value = PyLong_FromSsize_t(_return_value);
-
-exit:
-    return return_value;
-}
-
 PyDoc_STRVAR(sys_getallocatedblocks__doc__,
 "getallocatedblocks($module, /)\n"
 "--\n"
@@ -1345,4 +1318,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=2b5e1bc24a3348bd input=a9049054013a1b77]*/
+/*[clinic end generated code: output=79228e569529129c input=a9049054013a1b77]*/

Python/opcode_targets.h

@@ -71,21 +71,21 @@ static void *opcode_targets[256] = {
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
-    &&TARGET_JUMP_BACKWARD_QUICK,
     &&TARGET_LOAD_ATTR_ADAPTIVE,
+    &&TARGET_LOAD_ATTR_CLASS,
     &&TARGET_LOAD_ASSERTION_ERROR,
     &&TARGET_RETURN_GENERATOR,
-    &&TARGET_LOAD_ATTR_CLASS,
     &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
     &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
     &&TARGET_LOAD_ATTR_MODULE,
     &&TARGET_LOAD_ATTR_PROPERTY,
     &&TARGET_LOAD_ATTR_SLOT,
+    &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
-    &&TARGET_LOAD_ATTR_WITH_HINT,
+    &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
     &&TARGET_ASYNC_GEN_WRAP,
     &&TARGET_PREP_RERAISE_STAR,
     &&TARGET_POP_EXCEPT,
@@ -112,7 +112,7 @@ static void *opcode_targets[256] = {
     &&TARGET_JUMP_FORWARD,
     &&TARGET_JUMP_IF_FALSE_OR_POP,
     &&TARGET_JUMP_IF_TRUE_OR_POP,
-    &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
+    &&TARGET_LOAD_ATTR_METHOD_NO_DICT,
     &&TARGET_POP_JUMP_IF_FALSE,
     &&TARGET_POP_JUMP_IF_TRUE,
     &&TARGET_LOAD_GLOBAL,
@@ -120,7 +120,7 @@ static void *opcode_targets[256] = {
     &&TARGET_CONTAINS_OP,
     &&TARGET_RERAISE,
     &&TARGET_COPY,
-    &&TARGET_LOAD_ATTR_METHOD_NO_DICT,
+    &&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
     &&TARGET_BINARY_OP,
     &&TARGET_SEND,
     &&TARGET_LOAD_FAST,
@@ -140,9 +140,9 @@ static void *opcode_targets[256] = {
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
     &&TARGET_JUMP_BACKWARD,
-    &&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
-    &&TARGET_CALL_FUNCTION_EX,
     &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES,
+    &&TARGET_CALL_FUNCTION_EX,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
@@ -152,28 +152,26 @@ static void *opcode_targets[256] = {
     &&TARGET_YIELD_VALUE,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_LOAD_FAST__LOAD_CONST,
+    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
     &&TARGET_BUILD_STRING,
-    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_GLOBAL_MODULE,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_LIST_EXTEND,
     &&TARGET_SET_UPDATE,
     &&TARGET_DICT_MERGE,
     &&TARGET_DICT_UPDATE,
-    &&TARGET_RESUME_QUICK,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
     &&TARGET_STORE_ATTR_WITH_HINT,
-    &&TARGET_CALL,
-    &&TARGET_KW_NAMES,
     &&TARGET_STORE_FAST__LOAD_FAST,
     &&TARGET_STORE_FAST__STORE_FAST,
+    &&TARGET_CALL,
+    &&TARGET_KW_NAMES,
     &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_STORE_SUBSCR_LIST_INT,
@@ -254,5 +252,7 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
+    &&_unknown_opcode,
+    &&_unknown_opcode,
     &&TARGET_DO_TRACING
 };

Python/specialize.c

@@ -31,7 +31,6 @@ uint8_t _PyOpcode_Adaptive[256] = {
     [FOR_ITER] = FOR_ITER_ADAPTIVE,
 };
 
-Py_ssize_t _Py_QuickenedCount = 0;
 #ifdef Py_STATS
 PyStats _py_stats_struct = { 0 };
 PyStats *_py_stats = &_py_stats_struct;
@@ -280,16 +279,14 @@ do { \
 void
 _PyCode_Quicken(PyCodeObject *code)
 {
-    _Py_QuickenedCount++;
     int previous_opcode = -1;
     _Py_CODEUNIT *instructions = _PyCode_CODE(code);
     for (int i = 0; i < Py_SIZE(code); i++) {
-        int opcode = _Py_OPCODE(instructions[i]);
+        int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])];
         uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode];
         if (adaptive_opcode) {
             _Py_SET_OPCODE(instructions[i], adaptive_opcode);
-            // Make sure the adaptive counter is zero:
+            instructions[i + 1] = adaptive_counter_start();
-            assert(instructions[i + 1] == 0);
             previous_opcode = -1;
             i += _PyOpcode_Caches[opcode];
         }
@@ -299,12 +296,6 @@ _PyCode_Quicken(PyCodeObject *code)
                 case EXTENDED_ARG:
                     _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK);
                     break;
-                case JUMP_BACKWARD:
-                    _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
-                    break;
-                case RESUME:
-                    _Py_SET_OPCODE(instructions[i], RESUME_QUICK);
-                    break;
                 case LOAD_FAST:
                     switch(previous_opcode) {
                         case LOAD_FAST:

Python/sysmodule.c

@@ -17,7 +17,6 @@ Data members:
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_SetAsyncGenFinalizer()
-#include "pycore_code.h"          // _Py_QuickenedCount
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
 #include "pycore_long.h"          // _PY_LONG_MAX_STR_DIGITS_THRESHOLD
@@ -1855,17 +1854,6 @@ sys_gettotalrefcount_impl(PyObject *module)
 
 #endif /* Py_REF_DEBUG */
 
-/*[clinic input]
-sys._getquickenedcount -> Py_ssize_t
-[clinic start generated code]*/
-
-static Py_ssize_t
-sys__getquickenedcount_impl(PyObject *module)
-/*[clinic end generated code: output=1ab259e7f91248a2 input=249d448159eca912]*/
-{
-    return _Py_QuickenedCount;
-}
-
 /*[clinic input]
 sys.getallocatedblocks -> Py_ssize_t
 
@@ -2216,7 +2204,6 @@ static PyMethodDef sys_methods[] = {
     SYS_GETALLOCATEDBLOCKS_METHODDEF
     SYS_GETFILESYSTEMENCODING_METHODDEF
     SYS_GETFILESYSTEMENCODEERRORS_METHODDEF
-    SYS__GETQUICKENEDCOUNT_METHODDEF
 #ifdef Py_TRACE_REFS
     {"getobjects", _Py_GetObjects, METH_VARARGS},
 #endif

Tools/build/deepfreeze.py

@@ -114,9 +114,8 @@ class Printer:
         self.file = file
         self.cache: Dict[tuple[type, object, str], str] = {}
         self.hits, self.misses = 0, 0
-        self.patchups: list[str] = []
+        self.finis: list[str] = []
-        self.deallocs: list[str] = []
+        self.inits: list[str] = []
-        self.interns: list[str] = []
         self.write('#include "Python.h"')
         self.write('#include "internal/pycore_gc.h"')
         self.write('#include "internal/pycore_code.h"')
@@ -257,7 +256,6 @@ class Printer:
             self.write(f".co_names = {co_names},")
             self.write(f".co_exceptiontable = {co_exceptiontable},")
             self.field(code, "co_flags")
-            self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,")
             self.write("._co_linearray_entry_size = 0,")
             self.field(code, "co_argcount")
             self.field(code, "co_posonlyargcount")
@@ -284,8 +282,8 @@ class Printer:
                     self.write(f"._co_firsttraceable = {i},")
                     break
         name_as_code = f"(PyCodeObject *)&{name}"
-        self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});")
+        self.finis.append(f"_PyStaticCode_Fini({name_as_code});")
-        self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})")
+        self.inits.append(f"_PyStaticCode_Init({name_as_code})")
         return f"& {name}.ob_base.ob_base"
 
     def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str:
@@ -373,11 +371,7 @@ class Printer:
     def generate_file(self, module: str, code: object)-> None:
         module = module.replace(".", "_")
         self.generate(f"{module}_toplevel", code)
-        with self.block(f"static void {module}_do_patchups(void)"):
+        self.write(EPILOGUE.format(name=module))
-            for p in self.patchups:
-                self.write(p)
-        self.patchups.clear()
-        self.write(EPILOGUE.replace("%%NAME%%", module))
 
     def generate(self, name: str, obj: object) -> str:
         # Use repr() in the key to distinguish -0.0 from +0.0
@@ -421,11 +415,10 @@ class Printer:
 
 EPILOGUE = """
 PyObject *
-_Py_get_%%NAME%%_toplevel(void)
+_Py_get_{name}_toplevel(void)
-{
+{{
-    %%NAME%%_do_patchups();
+    return Py_NewRef((PyObject *) &{name}_toplevel);
-    return Py_NewRef((PyObject *) &%%NAME%%_toplevel);
+}}
-}
 """
 
 FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */"
@@ -461,10 +454,10 @@ def generate(args: list[str], output: TextIO) -> None:
                 code = compile(fd.read(), f"<frozen {modname}>", "exec")
             printer.generate_file(modname, code)
     with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"):
-        for p in printer.deallocs:
+        for p in printer.finis:
             printer.write(p)
     with printer.block(f"int\n_Py_Deepfreeze_Init(void)"):
-        for p in printer.interns:
+        for p in printer.inits:
             with printer.block(f"if ({p} < 0)"):
                 printer.write("return -1;")
         printer.write("return 0;")

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -477,7 +477,6 @@ Python/pyfpe.c	-	PyFPE_counter	-
 Python/pylifecycle.c	_Py_FatalErrorFormat	reentrant	-
 Python/pylifecycle.c	-	_Py_UnhandledKeyboardInterrupt	-
 Python/pylifecycle.c	fatal_error	reentrant	-
-Python/specialize.c	-	_Py_QuickenedCount	-
 
 
 ##################################