mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 17:08:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

GH-98686: Quicken everything (GH-98687)

Browse source
This commit is contained in:
Brandt Bucher 2022-11-02 10:42:57 -07:00 • committed by GitHub
parent 18fc232e07
commit 276d77724f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 144 additions and 239 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Include/cpython/code.h
View file

@ -70,7 +70,6 @@ typedef struct {
PyObject *co_exceptiontable; /* Byte string encoding exception handling \
table */ \
int co_flags; /* CO_..., see below */ \
short co_warmup; /* Warmup counter for quickening */ \
short _co_linearray_entry_size; /* Size of each entry in _co_linearray */ \
\
/* The rest are not so impactful on performance. */ \

32
Include/internal/pycore_code.h
View file

@ -91,28 +91,8 @@ typedef struct {
#define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
#define QUICKENING_WARMUP_DELAY 8
/* We want to compare to zero for efficiency, so we offset values accordingly */
#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
void _PyCode_Quicken(PyCodeObject *code);
static inline void
_PyCode_Warmup(PyCodeObject *code)
{
if (code->co_warmup != 0) {
code->co_warmup++;
if (code->co_warmup == 0) {
_PyCode_Quicken(code);
}
}
}
extern uint8_t _PyOpcode_Adaptive[256];
extern Py_ssize_t _Py_QuickenedCount;
// Borrowed references to common callables:
struct callable_cache {
PyObject *isinstance;
@ -252,10 +232,10 @@ extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
int oparg);
extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr);
/* Deallocator function for static codeobjects used in deepfreeze.py */
extern void _PyStaticCode_Dealloc(PyCodeObject *co);
/* Function to intern strings of codeobjects */
extern int _PyStaticCode_InternStrings(PyCodeObject *co);
/* Finalizer function for static codeobjects used in deepfreeze.py */
extern void _PyStaticCode_Fini(PyCodeObject *co);
/* Function to intern strings of codeobjects and quicken the bytecode */
extern int _PyStaticCode_Init(PyCodeObject *co);
#ifdef Py_STATS
@ -397,8 +377,8 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
#define ADAPTIVE_BACKOFF_BITS 4
/* The initial counter value is 31 == 2**ADAPTIVE_BACKOFF_START - 1 */
#define ADAPTIVE_BACKOFF_START 5
/* The initial counter value is 1 == 2**ADAPTIVE_BACKOFF_START - 1 */
#define ADAPTIVE_BACKOFF_START 1
#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)

30
Include/internal/pycore_opcode.h generated
View file

@ -142,7 +142,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
[IS_OP] = IS_OP,
[JUMP_BACKWARD] = JUMP_BACKWARD,
[JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT,
[JUMP_BACKWARD_QUICK] = JUMP_BACKWARD,
[JUMP_FORWARD] = JUMP_FORWARD,
[JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP,
[JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP,
@ -200,7 +199,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
[RAISE_VARARGS] = RAISE_VARARGS,
[RERAISE] = RERAISE,
[RESUME] = RESUME,
[RESUME_QUICK] = RESUME,
[RETURN_GENERATOR] = RETURN_GENERATOR,
[RETURN_VALUE] = RETURN_VALUE,
[SEND] = SEND,
@ -313,21 +311,21 @@ static const char *const _PyOpcode_OpName[263] = {
[GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER",
[PRINT_EXPR] = "PRINT_EXPR",
[LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS",
[JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK",
[LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE",
[LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
[LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR",
[RETURN_GENERATOR] = "RETURN_GENERATOR",
[LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
[LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
[LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE",
[LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
[LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY",
[LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT",
[LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
[LIST_TO_TUPLE] = "LIST_TO_TUPLE",
[RETURN_VALUE] = "RETURN_VALUE",
[IMPORT_STAR] = "IMPORT_STAR",
[SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS",
[LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
[LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
[ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP",
[PREP_RERAISE_STAR] = "PREP_RERAISE_STAR",
[POP_EXCEPT] = "POP_EXCEPT",
@ -354,7 +352,7 @@ static const char *const _PyOpcode_OpName[263] = {
[JUMP_FORWARD] = "JUMP_FORWARD",
[JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP",
[JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP",
[LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
[LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
[POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE",
[POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE",
[LOAD_GLOBAL] = "LOAD_GLOBAL",
@ -362,7 +360,7 @@ static const char *const _PyOpcode_OpName[263] = {
[CONTAINS_OP] = "CONTAINS_OP",
[RERAISE] = "RERAISE",
[COPY] = "COPY",
[LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
[LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
[BINARY_OP] = "BINARY_OP",
[SEND] = "SEND",
[LOAD_FAST] = "LOAD_FAST",
@ -382,9 +380,9 @@ static const char *const _PyOpcode_OpName[263] = {
[STORE_DEREF] = "STORE_DEREF",
[DELETE_DEREF] = "DELETE_DEREF",
[JUMP_BACKWARD] = "JUMP_BACKWARD",
[LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
[CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
[LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES",
[CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
[LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
[EXTENDED_ARG] = "EXTENDED_ARG",
[LIST_APPEND] = "LIST_APPEND",
[SET_ADD] = "SET_ADD",
@ -394,28 +392,26 @@ static const char *const _PyOpcode_OpName[263] = {
[YIELD_VALUE] = "YIELD_VALUE",
[RESUME] = "RESUME",
[MATCH_CLASS] = "MATCH_CLASS",
[LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
[LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
[LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
[FORMAT_VALUE] = "FORMAT_VALUE",
[BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP",
[BUILD_STRING] = "BUILD_STRING",
[LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
[LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE",
[LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
[LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE",
[STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
[LIST_EXTEND] = "LIST_EXTEND",
[SET_UPDATE] = "SET_UPDATE",
[DICT_MERGE] = "DICT_MERGE",
[DICT_UPDATE] = "DICT_UPDATE",
[RESUME_QUICK] = "RESUME_QUICK",
[STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
[STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
[STORE_ATTR_SLOT] = "STORE_ATTR_SLOT",
[STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT",
[CALL] = "CALL",
[KW_NAMES] = "KW_NAMES",
[STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
[STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
[CALL] = "CALL",
[KW_NAMES] = "KW_NAMES",
[STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE",
[STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
[STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
@ -423,6 +419,8 @@ static const char *const _PyOpcode_OpName[263] = {
[UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
[UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
[UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
[180] = "<180>",
[181] = "<181>",
[182] = "<182>",
[183] = "<183>",
[184] = "<184>",
@ -508,6 +506,8 @@ static const char *const _PyOpcode_OpName[263] = {
#endif
#define EXTRA_CASES \
case 180: \
case 181: \
case 182: \
case 183: \
case 184: \

64
Include/opcode.h generated
View file

@ -167,39 +167,37 @@ extern "C" {
#define FOR_ITER_ADAPTIVE 65
#define FOR_ITER_LIST 66
#define FOR_ITER_RANGE 67
#define JUMP_BACKWARD_QUICK 72
#define LOAD_ATTR_ADAPTIVE 73
#define LOAD_ATTR_CLASS 76
#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 77
#define LOAD_ATTR_INSTANCE_VALUE 78
#define LOAD_ATTR_MODULE 79
#define LOAD_ATTR_PROPERTY 80
#define LOAD_ATTR_SLOT 81
#define LOAD_ATTR_WITH_HINT 86
#define LOAD_ATTR_METHOD_LAZY_DICT 113
#define LOAD_ATTR_METHOD_NO_DICT 121
#define LOAD_ATTR_METHOD_WITH_DICT 141
#define LOAD_ATTR_METHOD_WITH_VALUES 143
#define LOAD_CONST__LOAD_FAST 153
#define LOAD_FAST__LOAD_CONST 154
#define LOAD_FAST__LOAD_FAST 158
#define LOAD_GLOBAL_ADAPTIVE 159
#define LOAD_GLOBAL_BUILTIN 160
#define LOAD_GLOBAL_MODULE 161
#define RESUME_QUICK 166
#define STORE_ATTR_ADAPTIVE 167
#define STORE_ATTR_INSTANCE_VALUE 168
#define STORE_ATTR_SLOT 169
#define STORE_ATTR_WITH_HINT 170
#define STORE_FAST__LOAD_FAST 173
#define STORE_FAST__STORE_FAST 174
#define STORE_SUBSCR_ADAPTIVE 175
#define STORE_SUBSCR_DICT 176
#define STORE_SUBSCR_LIST_INT 177
#define UNPACK_SEQUENCE_ADAPTIVE 178
#define UNPACK_SEQUENCE_LIST 179
#define UNPACK_SEQUENCE_TUPLE 180
#define UNPACK_SEQUENCE_TWO_TUPLE 181
#define LOAD_ATTR_ADAPTIVE 72
#define LOAD_ATTR_CLASS 73
#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76
#define LOAD_ATTR_INSTANCE_VALUE 77
#define LOAD_ATTR_MODULE 78
#define LOAD_ATTR_PROPERTY 79
#define LOAD_ATTR_SLOT 80
#define LOAD_ATTR_WITH_HINT 81
#define LOAD_ATTR_METHOD_LAZY_DICT 86
#define LOAD_ATTR_METHOD_NO_DICT 113
#define LOAD_ATTR_METHOD_WITH_DICT 121
#define LOAD_ATTR_METHOD_WITH_VALUES 141
#define LOAD_CONST__LOAD_FAST 143
#define LOAD_FAST__LOAD_CONST 153
#define LOAD_FAST__LOAD_FAST 154
#define LOAD_GLOBAL_ADAPTIVE 158
#define LOAD_GLOBAL_BUILTIN 159
#define LOAD_GLOBAL_MODULE 160
#define STORE_ATTR_ADAPTIVE 161
#define STORE_ATTR_INSTANCE_VALUE 166
#define STORE_ATTR_SLOT 167
#define STORE_ATTR_WITH_HINT 168
#define STORE_FAST__LOAD_FAST 169
#define STORE_FAST__STORE_FAST 170
#define STORE_SUBSCR_ADAPTIVE 173
#define STORE_SUBSCR_DICT 174
#define STORE_SUBSCR_LIST_INT 175
#define UNPACK_SEQUENCE_ADAPTIVE 176
#define UNPACK_SEQUENCE_LIST 177
#define UNPACK_SEQUENCE_TUPLE 178
#define UNPACK_SEQUENCE_TWO_TUPLE 179
#define DO_TRACING 255
#define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\

6
Lib/opcode.py
View file

@ -327,9 +327,6 @@ _specializations = {
"FOR_ITER_LIST",
"FOR_ITER_RANGE",
],
"JUMP_BACKWARD": [
"JUMP_BACKWARD_QUICK",
],
"LOAD_ATTR": [
"LOAD_ATTR_ADAPTIVE",
# These potentially push [NULL, bound method] onto the stack.
@ -358,9 +355,6 @@ _specializations = {
"LOAD_GLOBAL_BUILTIN",
"LOAD_GLOBAL_MODULE",
],
"RESUME": [
"RESUME_QUICK",
],
"STORE_ATTR": [
"STORE_ATTR_ADAPTIVE",
"STORE_ATTR_INSTANCE_VALUE",

3
Lib/test/libregrtest/refleak.py
View file

@ -73,7 +73,6 @@ def dash_R(ns, test_name, test_func):
fd_deltas = [0] * repcount
getallocatedblocks = sys.getallocatedblocks
gettotalrefcount = sys.gettotalrefcount
_getquickenedcount = sys._getquickenedcount
fd_count = os_helper.fd_count
# initialize variables to make pyflakes quiet
rc_before = alloc_before = fd_before = 0
@ -93,7 +92,7 @@ def dash_R(ns, test_name, test_func):
support.gc_collect()
# Read memory statistics immediately after the garbage collection
alloc_after = getallocatedblocks() - _getquickenedcount()
alloc_after = getallocatedblocks()
rc_after = gettotalrefcount()
fd_after = fd_count()

10
Lib/test/test_call.py
View file

@ -580,7 +580,7 @@ def testfunction_kw(self, *, kw):
return self
QUICKENING_WARMUP_DELAY = 8
ADAPTIVE_WARMUP_DELAY = 2
class TestPEP590(unittest.TestCase):
@ -771,7 +771,7 @@ class TestPEP590(unittest.TestCase):
assert_equal(11, f(num))
function_setvectorcall(f)
# make sure specializer is triggered by running > 50 times
for _ in range(10 * QUICKENING_WARMUP_DELAY):
for _ in range(10 * ADAPTIVE_WARMUP_DELAY):
assert_equal("overridden", f(num))
def test_setvectorcall_load_attr_specialization_skip(self):
@ -787,7 +787,7 @@ class TestPEP590(unittest.TestCase):
function_setvectorcall(X.__getattribute__)
# make sure specialization doesn't trigger
# when vectorcall is overridden
for _ in range(QUICKENING_WARMUP_DELAY):
for _ in range(ADAPTIVE_WARMUP_DELAY):
assert_equal("overridden", x.a)
def test_setvectorcall_load_attr_specialization_deopt(self):
@ -803,12 +803,12 @@ class TestPEP590(unittest.TestCase):
assert_equal = self.assertEqual
x = X()
# trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization
for _ in range(QUICKENING_WARMUP_DELAY):
for _ in range(ADAPTIVE_WARMUP_DELAY):
assert_equal("a", get_a(x))
function_setvectorcall(X.__getattribute__)
# make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
# gets deopted due to overridden vectorcall
for _ in range(QUICKENING_WARMUP_DELAY):
for _ in range(ADAPTIVE_WARMUP_DELAY):
assert_equal("overridden", get_a(x))
@requires_limited_api

23
Lib/test/test_dis.py
View file

@ -713,7 +713,7 @@ def load_test(x, y=0):
return a, b
dis_load_test_quickened_code = """\
%3d 0 RESUME_QUICK 0
%3d 0 RESUME 0
%3d 2 LOAD_FAST__LOAD_FAST 0 (x)
4 LOAD_FAST 1 (y)
@ -733,7 +733,7 @@ def loop_test():
load_test(i)
dis_loop_test_quickened_code = """\
%3d RESUME_QUICK 0
%3d RESUME 0
%3d BUILD_LIST 0
LOAD_CONST 1 ((1, 2, 3))
@ -748,7 +748,7 @@ dis_loop_test_quickened_code = """\
LOAD_FAST 0 (i)
CALL_PY_WITH_DEFAULTS 1
POP_TOP
JUMP_BACKWARD_QUICK 17 (to 16)
JUMP_BACKWARD 17 (to 16)
%3d >> END_FOR
LOAD_CONST 0 (None)
@ -774,7 +774,7 @@ dis_extended_arg_quick_code = """\
"""% (extended_arg_quick.__code__.co_firstlineno,
extended_arg_quick.__code__.co_firstlineno + 1,)
QUICKENING_WARMUP_DELAY = 8
ADAPTIVE_WARMUP_DELAY = 2
class DisTestBase(unittest.TestCase):
"Common utilities for DisTests and TestDisTraceback"
@ -1079,7 +1079,7 @@ class DisTests(DisTestBase):
check(dis_nested_2)
@staticmethod
def code_quicken(f, times=QUICKENING_WARMUP_DELAY):
def code_quicken(f, times=ADAPTIVE_WARMUP_DELAY):
for _ in range(times):
f()
@ -1092,7 +1092,7 @@ class DisTests(DisTestBase):
@cpython_only
def test_binary_specialize(self):
binary_op_quicken = """\
0 0 RESUME_QUICK 0
0 0 RESUME 0
1 2 LOAD_NAME 0 (a)
4 LOAD_NAME 1 (b)
@ -1110,7 +1110,7 @@ class DisTests(DisTestBase):
self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)", True)
binary_subscr_quicken = """\
0 0 RESUME_QUICK 0
0 0 RESUME 0
1 2 LOAD_NAME 0 (a)
4 LOAD_CONST 0 (0)
@ -1130,7 +1130,7 @@ class DisTests(DisTestBase):
@cpython_only
def test_load_attr_specialize(self):
load_attr_quicken = """\
0 0 RESUME_QUICK 0
0 0 RESUME 0
1 2 LOAD_CONST 0 ('a')
4 LOAD_ATTR_SLOT 0 (__class__)
@ -1144,7 +1144,7 @@ class DisTests(DisTestBase):
@cpython_only
def test_call_specialize(self):
call_quicken = """\
0 RESUME_QUICK 0
0 RESUME 0
1 PUSH_NULL
LOAD_NAME 0 (str)
@ -1190,7 +1190,7 @@ class DisTests(DisTestBase):
for quickened in (False, True):
for adaptive in (False, True):
with self.subTest(f"{quickened=}, {adaptive=}"):
if quickened and adaptive:
if adaptive:
pattern = r"^(\w+: \d+)?$"
else:
pattern = r"^(\w+: 0)?$"
@ -1198,11 +1198,10 @@ class DisTests(DisTestBase):
for cache in caches:
self.assertRegex(cache, pattern)
total_caches = 23
empty_caches = 8 if adaptive and quickened else total_caches
empty_caches = 8 if adaptive else total_caches
self.assertEqual(caches.count(""), empty_caches)
self.assertEqual(len(caches), total_caches)
class DisWithFileTests(DisTests):
# Run the tests again, using the file arg instead of print

46
Lib/test/test_embed.py
View file

@ -346,33 +346,41 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
out, err = self.run_embedded_interpreter("test_repeated_simple_init")
self.assertEqual(out, 'Finalized\n' * INIT_LOOPS)
def test_quickened_static_code_gets_unquickened_at_Py_FINALIZE(self):
def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self):
# https://github.com/python/cpython/issues/92031
# Do these imports outside of the code string to avoid using
# importlib too much from within the code string, so that
# _handle_fromlist doesn't get quickened until we intend it to.
from dis import _all_opmap
resume = _all_opmap["RESUME"]
resume_quick = _all_opmap["RESUME_QUICK"]
from test.test_dis import QUICKENING_WARMUP_DELAY
code = textwrap.dedent(f"""\
code = textwrap.dedent("""\
import dis
import importlib._bootstrap
import opcode
import test.test_dis
def is_specialized(f):
for instruction in dis.get_instructions(f, adaptive=True):
opname = instruction.opname
if (
opname in opcode._specialized_instructions
# Exclude superinstructions:
and "__" not in opname
# Exclude adaptive instructions:
and not opname.endswith("_ADAPTIVE")
# Exclude "quick" instructions:
and not opname.endswith("_QUICK")
):
return True
return False
func = importlib._bootstrap._handle_fromlist
code = func.__code__
# Assert initially unquickened.
# Use sets to account for byte order.
if set(code._co_code_adaptive[:2]) != set([{resume}, 0]):
raise AssertionError()
# "copy" the code to un-specialize it:
func.__code__ = func.__code__.replace()
for i in range({QUICKENING_WARMUP_DELAY}):
assert not is_specialized(func), "specialized instructions found"
for i in range(test.test_dis.ADAPTIVE_WARMUP_DELAY):
func(importlib._bootstrap, ["x"], lambda *args: None)
# Assert quickening worked
if set(code._co_code_adaptive[:2]) != set([{resume_quick}, 0]):
raise AssertionError()
assert is_specialized(func), "no specialized instructions found"
print("Tests passed")
""")

4
Lib/test/test_lltrace.py
View file

@ -8,7 +8,7 @@ from test.support.script_helper import assert_python_ok
def example():
x = []
for i in range(1):
for i in range(0):
x.append(i)
x = "this is"
y = "an example"
@ -75,7 +75,7 @@ class TestLLTrace(unittest.TestCase):
self.assertIn('this is an example', stdout)
# check that offsets match the output of dis.dis()
instr_map = {i.offset: i for i in dis.get_instructions(example)}
instr_map = {i.offset: i for i in dis.get_instructions(example, adaptive=True)}
for line in stdout.splitlines():
offset, colon, opname_oparg = line.partition(":")
if not colon:

2
Misc/NEWS.d/next/Core and Builtins/2022-10-16-13-26-46.gh-issue-98686.D9Gu_Q.rst Normal file
View file

@ -0,0 +1,2 @@
Quicken all code objects, and specialize adaptive bytecode instructions more
aggressively.

16
Objects/codeobject.c
View file

@ -301,6 +301,8 @@ _PyCode_Validate(struct _PyCodeConstructor *con)
return 0;
}
extern void _PyCode_Quicken(PyCodeObject *code);
static void
init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
{
@ -353,7 +355,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
co->co_extra = NULL;
co->_co_cached = NULL;
co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
co->_co_linearray_entry_size = 0;
co->_co_linearray = NULL;
memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code),
@ -364,6 +365,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
entry_point++;
}
co->_co_firsttraceable = entry_point;
_PyCode_Quicken(co);
}
static int
@ -1664,9 +1666,6 @@ code_dealloc(PyCodeObject *co)
if (co->_co_linearray) {
PyMem_Free(co->_co_linearray);
}
if (co->co_warmup == 0) {
_Py_QuickenedCount--;
}
PyObject_Free(co);
}
@ -2224,13 +2223,9 @@ _PyCode_ConstantKey(PyObject *op)
}
void
_PyStaticCode_Dealloc(PyCodeObject *co)
_PyStaticCode_Fini(PyCodeObject *co)
{
if (co->co_warmup == 0) {
_Py_QuickenedCount--;
}
deopt_code(_PyCode_CODE(co), Py_SIZE(co));
co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
PyMem_Free(co->co_extra);
if (co->_co_cached != NULL) {
Py_CLEAR(co->_co_cached->_co_code);
@ -2252,7 +2247,7 @@ _PyStaticCode_Dealloc(PyCodeObject *co)
}
int
_PyStaticCode_InternStrings(PyCodeObject *co)
_PyStaticCode_Init(PyCodeObject *co)
{
int res = intern_strings(co->co_names);
if (res < 0) {
@ -2266,5 +2261,6 @@ _PyStaticCode_InternStrings(PyCodeObject *co)
if (res < 0) {
return -1;
}
_PyCode_Quicken(co);
return 0;
}

2
Objects/frameobject.c
View file

@ -603,7 +603,7 @@ _PyFrame_GetState(PyFrameObject *frame)
if (_PyInterpreterFrame_LASTI(frame->f_frame) < 0) {
return FRAME_CREATED;
}
switch (_PyOpcode_Deopt[_Py_OPCODE(*frame->f_frame->prev_instr)])
switch (_Py_OPCODE(*frame->f_frame->prev_instr))
{
case COPY_FREE_VARS:
case MAKE_CELL:

3
Objects/genobject.c
View file

@ -8,7 +8,6 @@
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_genobject.h" // struct _Py_async_gen_state
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode.h" // _PyOpcode_Deopt
#include "pycore_pyerrors.h" // _PyErr_ClearExcState()
#include "pycore_pystate.h" // _PyThreadState_GET()
#include "structmember.h" // PyMemberDef
@ -364,7 +363,7 @@ _PyGen_yf(PyGenObject *gen)
return NULL;
}
_Py_CODEUNIT next = frame->prev_instr[1];
if (_PyOpcode_Deopt[_Py_OPCODE(next)] != RESUME || _Py_OPARG(next) < 2)
if (_Py_OPCODE(next) != RESUME || _Py_OPARG(next) < 2)
{
/* Not in a yield from */
return NULL;

30
Python/ceval.c
View file

@ -143,7 +143,7 @@ lltrace_instruction(_PyInterpreterFrame *frame,
const char *opname = _PyOpcode_OpName[opcode];
assert(opname != NULL);
int offset = (int)(next_instr - _PyCode_CODE(frame->f_code));
if (HAS_ARG(opcode)) {
if (HAS_ARG(_PyOpcode_Deopt[opcode])) {
printf("%d: %s %d\n", offset * 2, opname, oparg);
}
else {
@ -1165,14 +1165,7 @@ handle_eval_breaker:
TARGET(NOP) {
DISPATCH();
}
TARGET(RESUME) {
_PyCode_Warmup(frame->f_code);
GO_TO_INSTRUCTION(RESUME_QUICK);
}
TARGET(RESUME_QUICK) {
PREDICTED(RESUME_QUICK);
assert(tstate->cframe == &cframe);
assert(frame == cframe.current_frame);
if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
@ -1710,7 +1703,7 @@ handle_eval_breaker:
PyObject *list = PEEK(oparg);
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
goto error;
PREDICT(JUMP_BACKWARD_QUICK);
PREDICT(JUMP_BACKWARD);
DISPATCH();
}
@ -1722,7 +1715,7 @@ handle_eval_breaker:
Py_DECREF(v);
if (err != 0)
goto error;
PREDICT(JUMP_BACKWARD_QUICK);
PREDICT(JUMP_BACKWARD);
DISPATCH();
}
@ -2908,7 +2901,7 @@ handle_eval_breaker:
if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) {
goto error;
}
PREDICT(JUMP_BACKWARD_QUICK);
PREDICT(JUMP_BACKWARD);
DISPATCH();
}
@ -3576,8 +3569,11 @@ handle_eval_breaker:
}
TARGET(JUMP_BACKWARD) {
_PyCode_Warmup(frame->f_code);
GO_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
PREDICTED(JUMP_BACKWARD);
assert(oparg < INSTR_OFFSET());
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(POP_JUMP_IF_FALSE) {
@ -3707,14 +3703,6 @@ handle_eval_breaker:
DISPATCH();
}
TARGET(JUMP_BACKWARD_QUICK) {
PREDICTED(JUMP_BACKWARD_QUICK);
assert(oparg < INSTR_OFFSET());
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
TARGET(GET_LEN) {
// PUSH(len(TOS))
Py_ssize_t len_i = PyObject_Length(TOP());

29
Python/clinic/sysmodule.c.h generated
View file

@ -884,33 +884,6 @@ exit:
#endif /* defined(Py_REF_DEBUG) */
PyDoc_STRVAR(sys__getquickenedcount__doc__,
"_getquickenedcount($module, /)\n"
"--\n"
"\n");
#define SYS__GETQUICKENEDCOUNT_METHODDEF \
{"_getquickenedcount", (PyCFunction)sys__getquickenedcount, METH_NOARGS, sys__getquickenedcount__doc__},
static Py_ssize_t
sys__getquickenedcount_impl(PyObject *module);
static PyObject *
sys__getquickenedcount(PyObject *module, PyObject *Py_UNUSED(ignored))
{
PyObject *return_value = NULL;
Py_ssize_t _return_value;
_return_value = sys__getquickenedcount_impl(module);
if ((_return_value == -1) && PyErr_Occurred()) {
goto exit;
}
return_value = PyLong_FromSsize_t(_return_value);
exit:
return return_value;
}
PyDoc_STRVAR(sys_getallocatedblocks__doc__,
"getallocatedblocks($module, /)\n"
"--\n"
@ -1345,4 +1318,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored))
#ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
#define SYS_GETANDROIDAPILEVEL_METHODDEF
#endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
/*[clinic end generated code: output=2b5e1bc24a3348bd input=a9049054013a1b77]*/
/*[clinic end generated code: output=79228e569529129c input=a9049054013a1b77]*/

26
Python/opcode_targets.h generated
View file

@ -71,21 +71,21 @@ static void *opcode_targets[256] = {
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_PRINT_EXPR,
&&TARGET_LOAD_BUILD_CLASS,
&&TARGET_JUMP_BACKWARD_QUICK,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_CLASS,
&&TARGET_LOAD_ASSERTION_ERROR,
&&TARGET_RETURN_GENERATOR,
&&TARGET_LOAD_ATTR_CLASS,
&&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_ATTR_PROPERTY,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&TARGET_SETUP_ANNOTATIONS,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
&&TARGET_ASYNC_GEN_WRAP,
&&TARGET_PREP_RERAISE_STAR,
&&TARGET_POP_EXCEPT,
@ -112,7 +112,7 @@ static void *opcode_targets[256] = {
&&TARGET_JUMP_FORWARD,
&&TARGET_JUMP_IF_FALSE_OR_POP,
&&TARGET_JUMP_IF_TRUE_OR_POP,
&&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
&&TARGET_LOAD_ATTR_METHOD_NO_DICT,
&&TARGET_POP_JUMP_IF_FALSE,
&&TARGET_POP_JUMP_IF_TRUE,
&&TARGET_LOAD_GLOBAL,
@ -120,7 +120,7 @@ static void *opcode_targets[256] = {
&&TARGET_CONTAINS_OP,
&&TARGET_RERAISE,
&&TARGET_COPY,
&&TARGET_LOAD_ATTR_METHOD_NO_DICT,
&&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
&&TARGET_BINARY_OP,
&&TARGET_SEND,
&&TARGET_LOAD_FAST,
@ -140,9 +140,9 @@ static void *opcode_targets[256] = {
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
&&TARGET_JUMP_BACKWARD,
&&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
&&TARGET_CALL_FUNCTION_EX,
&&TARGET_LOAD_ATTR_METHOD_WITH_VALUES,
&&TARGET_CALL_FUNCTION_EX,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_EXTENDED_ARG,
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
@ -152,28 +152,26 @@ static void *opcode_targets[256] = {
&&TARGET_YIELD_VALUE,
&&TARGET_RESUME,
&&TARGET_MATCH_CLASS,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
&&TARGET_BUILD_STRING,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&TARGET_RESUME_QUICK,
&&TARGET_STORE_ATTR_ADAPTIVE,
&&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_STORE_ATTR_SLOT,
&&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&TARGET_STORE_SUBSCR_ADAPTIVE,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_STORE_SUBSCR_LIST_INT,
@ -254,5 +252,7 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_DO_TRACING
};

13
Python/specialize.c
View file

@ -31,7 +31,6 @@ uint8_t _PyOpcode_Adaptive[256] = {
[FOR_ITER] = FOR_ITER_ADAPTIVE,
};
Py_ssize_t _Py_QuickenedCount = 0;
#ifdef Py_STATS
PyStats _py_stats_struct = { 0 };
PyStats *_py_stats = &_py_stats_struct;
@ -280,16 +279,14 @@ do { \
void
_PyCode_Quicken(PyCodeObject *code)
{
_Py_QuickenedCount++;
int previous_opcode = -1;
_Py_CODEUNIT *instructions = _PyCode_CODE(code);
for (int i = 0; i < Py_SIZE(code); i++) {
int opcode = _Py_OPCODE(instructions[i]);
int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])];
uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode];
if (adaptive_opcode) {
_Py_SET_OPCODE(instructions[i], adaptive_opcode);
// Make sure the adaptive counter is zero:
assert(instructions[i + 1] == 0);
instructions[i + 1] = adaptive_counter_start();
previous_opcode = -1;
i += _PyOpcode_Caches[opcode];
}
@ -299,12 +296,6 @@ _PyCode_Quicken(PyCodeObject *code)
case EXTENDED_ARG:
_Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK);
break;
case JUMP_BACKWARD:
_Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
break;
case RESUME:
_Py_SET_OPCODE(instructions[i], RESUME_QUICK);
break;
case LOAD_FAST:
switch(previous_opcode) {
case LOAD_FAST:

13
Python/sysmodule.c
View file

@ -17,7 +17,6 @@ Data members:
#include "Python.h"
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _PyEval_SetAsyncGenFinalizer()
#include "pycore_code.h" // _Py_QuickenedCount
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_initconfig.h" // _PyStatus_EXCEPTION()
#include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD
@ -1855,17 +1854,6 @@ sys_gettotalrefcount_impl(PyObject *module)
#endif /* Py_REF_DEBUG */
/*[clinic input]
sys._getquickenedcount -> Py_ssize_t
[clinic start generated code]*/
static Py_ssize_t
sys__getquickenedcount_impl(PyObject *module)
/*[clinic end generated code: output=1ab259e7f91248a2 input=249d448159eca912]*/
{
return _Py_QuickenedCount;
}
/*[clinic input]
sys.getallocatedblocks -> Py_ssize_t
@ -2216,7 +2204,6 @@ static PyMethodDef sys_methods[] = {
SYS_GETALLOCATEDBLOCKS_METHODDEF
SYS_GETFILESYSTEMENCODING_METHODDEF
SYS_GETFILESYSTEMENCODEERRORS_METHODDEF
SYS__GETQUICKENEDCOUNT_METHODDEF
#ifdef Py_TRACE_REFS
{"getobjects", _Py_GetObjects, METH_VARARGS},
#endif

29
Tools/build/deepfreeze.py
View file

@ -114,9 +114,8 @@ class Printer:
self.file = file
self.cache: Dict[tuple[type, object, str], str] = {}
self.hits, self.misses = 0, 0
self.patchups: list[str] = []
self.deallocs: list[str] = []
self.interns: list[str] = []
self.finis: list[str] = []
self.inits: list[str] = []
self.write('#include "Python.h"')
self.write('#include "internal/pycore_gc.h"')
self.write('#include "internal/pycore_code.h"')
@ -257,7 +256,6 @@ class Printer:
self.write(f".co_names = {co_names},")
self.write(f".co_exceptiontable = {co_exceptiontable},")
self.field(code, "co_flags")
self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,")
self.write("._co_linearray_entry_size = 0,")
self.field(code, "co_argcount")
self.field(code, "co_posonlyargcount")
@ -284,8 +282,8 @@ class Printer:
self.write(f"._co_firsttraceable = {i},")
break
name_as_code = f"(PyCodeObject *)&{name}"
self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});")
self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})")
self.finis.append(f"_PyStaticCode_Fini({name_as_code});")
self.inits.append(f"_PyStaticCode_Init({name_as_code})")
return f"& {name}.ob_base.ob_base"
def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str:
@ -373,11 +371,7 @@ class Printer:
def generate_file(self, module: str, code: object)-> None:
module = module.replace(".", "_")
self.generate(f"{module}_toplevel", code)
with self.block(f"static void {module}_do_patchups(void)"):
for p in self.patchups:
self.write(p)
self.patchups.clear()
self.write(EPILOGUE.replace("%%NAME%%", module))
self.write(EPILOGUE.format(name=module))
def generate(self, name: str, obj: object) -> str:
# Use repr() in the key to distinguish -0.0 from +0.0
@ -421,11 +415,10 @@ class Printer:
EPILOGUE = """
PyObject *
_Py_get_%%NAME%%_toplevel(void)
{
%%NAME%%_do_patchups();
return Py_NewRef((PyObject *) &%%NAME%%_toplevel);
}
_Py_get_{name}_toplevel(void)
{{
return Py_NewRef((PyObject *) &{name}_toplevel);
}}
"""
FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */"
@ -461,10 +454,10 @@ def generate(args: list[str], output: TextIO) -> None:
code = compile(fd.read(), f"<frozen {modname}>", "exec")
printer.generate_file(modname, code)
with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"):
for p in printer.deallocs:
for p in printer.finis:
printer.write(p)
with printer.block(f"int\n_Py_Deepfreeze_Init(void)"):
for p in printer.interns:
for p in printer.inits:
with printer.block(f"if ({p} < 0)"):
printer.write("return -1;")
printer.write("return 0;")

1
Tools/c-analyzer/cpython/globals-to-fix.tsv
View file

@ -477,7 +477,6 @@ Python/pyfpe.c - PyFPE_counter -
Python/pylifecycle.c _Py_FatalErrorFormat reentrant -
Python/pylifecycle.c - _Py_UnhandledKeyboardInterrupt -
Python/pylifecycle.c fatal_error reentrant -
Python/specialize.c - _Py_QuickenedCount -
##################################

Can't render this file because it has a wrong number of fields in line 4.