gh-126835: Move constant tuple folding from ast_opt to CFG (#130769)

This commit is contained in:
Yan Yanchii 2025-03-19 21:59:55 +01:00 committed by GitHub
parent 54efe296bc
commit 75103d975c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 264 additions and 172 deletions

View file

@ -153,22 +153,6 @@ class AST_Tests(unittest.TestCase):
self.assertIsInstance(res.body[0].value, ast.Name)
self.assertEqual(res.body[0].value.id, expected)
def test_optimization_levels_const_folding(self):
folded = ('Expr', (1, 0, 1, 6), ('Constant', (1, 0, 1, 6), (1, 2), None))
not_folded = ('Expr', (1, 0, 1, 6),
('Tuple', (1, 0, 1, 6),
[('Constant', (1, 1, 1, 2), 1, None),
('Constant', (1, 4, 1, 5), 2, None)], ('Load',)))
cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)]
for (optval, expected) in cases:
with self.subTest(optval=optval):
tree1 = ast.parse("(1, 2)", optimize=optval)
tree2 = ast.parse(ast.parse("(1, 2)"), optimize=optval)
for tree in [tree1, tree2]:
res = to_tuple(tree.body[0])
self.assertEqual(res, expected)
def test_invalid_position_information(self):
invalid_linenos = [
(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)
@ -3193,101 +3177,6 @@ class ASTOptimiziationTests(unittest.TestCase):
self.assert_ast(code, non_optimized_target, optimized_target)
def test_folding_tuple(self):
code = "(1,)"
non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)]))
optimized_target = self.wrap_expr(ast.Constant(value=(1,)))
self.assert_ast(code, non_optimized_target, optimized_target)
def test_folding_type_param_in_function_def(self):
code = "def foo[%s = (1, 2)](): pass"
unoptimized_tuple = ast.Tuple(elts=[ast.Constant(1), ast.Constant(2)])
unoptimized_type_params = [
("T", "T", ast.TypeVar),
("**P", "P", ast.ParamSpec),
("*Ts", "Ts", ast.TypeVarTuple),
]
for type, name, type_param in unoptimized_type_params:
result_code = code % type
optimized_target = self.wrap_statement(
ast.FunctionDef(
name='foo',
args=ast.arguments(),
body=[ast.Pass()],
type_params=[type_param(name=name, default_value=ast.Constant((1, 2)))]
)
)
non_optimized_target = self.wrap_statement(
ast.FunctionDef(
name='foo',
args=ast.arguments(),
body=[ast.Pass()],
type_params=[type_param(name=name, default_value=unoptimized_tuple)]
)
)
self.assert_ast(result_code, non_optimized_target, optimized_target)
def test_folding_type_param_in_class_def(self):
code = "class foo[%s = (1, 2)]: pass"
unoptimized_tuple = ast.Tuple(elts=[ast.Constant(1), ast.Constant(2)])
unoptimized_type_params = [
("T", "T", ast.TypeVar),
("**P", "P", ast.ParamSpec),
("*Ts", "Ts", ast.TypeVarTuple),
]
for type, name, type_param in unoptimized_type_params:
result_code = code % type
optimized_target = self.wrap_statement(
ast.ClassDef(
name='foo',
body=[ast.Pass()],
type_params=[type_param(name=name, default_value=ast.Constant((1, 2)))]
)
)
non_optimized_target = self.wrap_statement(
ast.ClassDef(
name='foo',
body=[ast.Pass()],
type_params=[type_param(name=name, default_value=unoptimized_tuple)]
)
)
self.assert_ast(result_code, non_optimized_target, optimized_target)
def test_folding_type_param_in_type_alias(self):
code = "type foo[%s = (1, 2)] = 1"
unoptimized_tuple = ast.Tuple(elts=[ast.Constant(1), ast.Constant(2)])
unoptimized_type_params = [
("T", "T", ast.TypeVar),
("**P", "P", ast.ParamSpec),
("*Ts", "Ts", ast.TypeVarTuple),
]
for type, name, type_param in unoptimized_type_params:
result_code = code % type
optimized_target = self.wrap_statement(
ast.TypeAlias(
name=ast.Name(id='foo', ctx=ast.Store()),
type_params=[type_param(name=name, default_value=ast.Constant((1, 2)))],
value=ast.Constant(value=1),
)
)
non_optimized_target = self.wrap_statement(
ast.TypeAlias(
name=ast.Name(id='foo', ctx=ast.Store()),
type_params=[type_param(name=name, default_value=unoptimized_tuple)],
value=ast.Constant(value=1),
)
)
self.assert_ast(result_code, non_optimized_target, optimized_target)
def test_folding_match_case_allowed_expressions(self):
def get_match_case_values(node):
result = []

View file

@ -554,7 +554,7 @@ class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase):
self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)
def test_compile_ast(self):
args = ("a*(1,2)", "f.py", "exec")
args = ("a*__debug__", "f.py", "exec")
raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0]
opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0]
opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0]
@ -565,14 +565,14 @@ class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase):
self.assertIsInstance(tree.value.left, ast.Name)
self.assertEqual(tree.value.left.id, 'a')
raw_right = raw.value.right # expect Tuple((1, 2))
self.assertIsInstance(raw_right, ast.Tuple)
self.assertListEqual([elt.value for elt in raw_right.elts], [1, 2])
raw_right = raw.value.right
self.assertIsInstance(raw_right, ast.Name)
self.assertEqual(raw_right.id, "__debug__")
for opt in [opt1, opt2]:
opt_right = opt.value.right # expect Constant((1,2))
opt_right = opt.value.right
self.assertIsInstance(opt_right, ast.Constant)
self.assertEqual(opt_right.value, (1, 2))
self.assertEqual(opt_right.value, True)
def test_delattr(self):
sys.spam = 1

View file

@ -793,9 +793,9 @@ class TestSpecifics(unittest.TestCase):
f1, f2 = lambda: "not a name", lambda: ("not a name",)
f3 = lambda x: x in {("not a name",)}
self.assertIs(f1.__code__.co_consts[0],
f2.__code__.co_consts[0][0])
f2.__code__.co_consts[1][0])
self.assertIs(next(iter(f3.__code__.co_consts[1])),
f2.__code__.co_consts[0])
f2.__code__.co_consts[1])
# {0} is converted to a constant frozenset({0}) by the peephole
# optimizer
@ -1129,6 +1129,31 @@ class TestSpecifics(unittest.TestCase):
self.assertIn('LOAD_ATTR', instructions)
self.assertIn('CALL', instructions)
def test_folding_type_param(self):
get_code_fn_cls = lambda x: x.co_consts[0].co_consts[2]
get_code_type_alias = lambda x: x.co_consts[0].co_consts[3]
snippets = [
("def foo[T = 40 + 5](): pass", get_code_fn_cls),
("def foo[**P = 40 + 5](): pass", get_code_fn_cls),
("def foo[*Ts = 40 + 5](): pass", get_code_fn_cls),
("class foo[T = 40 + 5]: pass", get_code_fn_cls),
("class foo[**P = 40 + 5]: pass", get_code_fn_cls),
("class foo[*Ts = 40 + 5]: pass", get_code_fn_cls),
("type foo[T = 40 + 5] = 1", get_code_type_alias),
("type foo[**P = 40 + 5] = 1", get_code_type_alias),
("type foo[*Ts = 40 + 5] = 1", get_code_type_alias),
]
for snippet, get_code in snippets:
c = compile(snippet, "<dummy>", "exec")
code = get_code(c)
opcodes = list(dis.get_instructions(code))
instructions = [opcode.opname for opcode in opcodes]
args = [opcode.oparg for opcode in opcodes]
self.assertNotIn(40, args)
self.assertNotIn(5, args)
self.assertIn('LOAD_SMALL_INT', instructions)
self.assertIn(45, args)
def test_lineno_procedure_call(self):
def call():
(

View file

@ -1666,7 +1666,8 @@ class TestSpecializer(TestBase):
def test_unpack_sequence(self):
def unpack_sequence_two_tuple():
for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
a, b = 1, 2
t = 1, 2
a, b = t
self.assertEqual(a, 1)
self.assertEqual(b, 2)
@ -1677,8 +1678,11 @@ class TestSpecializer(TestBase):
def unpack_sequence_tuple():
for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
a, = 1,
a, b, c, d = 1, 2, 3, 4
self.assertEqual(a, 1)
self.assertEqual(b, 2)
self.assertEqual(c, 3)
self.assertEqual(d, 4)
unpack_sequence_tuple()
self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE")

View file

@ -154,7 +154,7 @@ class TestTranforms(BytecodeTestCase):
for line, elem in (
('a = 1,2,3', (1, 2, 3)),
('("a","b","c")', ('a', 'b', 'c')),
('a,b,c = 1,2,3', (1, 2, 3)),
('a,b,c,d = 1,2,3,4', (1, 2, 3, 4)),
('(None, 1, None)', (None, 1, None)),
('((1, 2), 3, 4)', ((1, 2), 3, 4)),
):
@ -1349,6 +1349,111 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase):
]
self.cfg_optimization_test(same, same, consts=[])
def test_fold_constant_intrinsic_list_to_tuple(self):
INTRINSIC_LIST_TO_TUPLE = 6
# long tuple
consts = 1000
before = (
[('BUILD_LIST', 0, 0)] +
[('LOAD_CONST', 0, 0), ('LIST_APPEND', 1, 0)] * consts +
[('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), ('RETURN_VALUE', None, 0)]
)
after = [
('LOAD_CONST', 1, 0),
('RETURN_VALUE', None, 0)
]
result_const = tuple(["test"] * consts)
self.cfg_optimization_test(before, after, consts=["test"], expected_consts=["test", result_const])
# empty list
before = [
('BUILD_LIST', 0, 0),
('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0),
('RETURN_VALUE', None, 0)
]
after = [
('LOAD_CONST', 0, 0),
('RETURN_VALUE', None, 0)
]
self.cfg_optimization_test(before, after, consts=[], expected_consts=[()])
# multiple BUILD_LIST 0: ([], 1, [], 2)
same = [
('BUILD_LIST', 0, 0),
('BUILD_LIST', 0, 0),
('LIST_APPEND', 1, 0),
('LOAD_SMALL_INT', 1, 0),
('LIST_APPEND', 1, 0),
('BUILD_LIST', 0, 0),
('LIST_APPEND', 1, 0),
('LOAD_SMALL_INT', 2, 0),
('LIST_APPEND', 1, 0),
('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0),
('RETURN_VALUE', None, 0)
]
self.cfg_optimization_test(same, same, consts=[])
# nested folding: (1, 1+1, 3)
before = [
('BUILD_LIST', 0, 0),
('LOAD_SMALL_INT', 1, 0),
('LIST_APPEND', 1, 0),
('LOAD_SMALL_INT', 1, 0),
('LOAD_SMALL_INT', 1, 0),
('BINARY_OP', get_binop_argval('NB_ADD'), 0),
('LIST_APPEND', 1, 0),
('LOAD_SMALL_INT', 3, 0),
('LIST_APPEND', 1, 0),
('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0),
('RETURN_VALUE', None, 0)
]
after = [
('LOAD_CONST', 0, 0),
('RETURN_VALUE', None, 0)
]
self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)])
# NOP's in between: (1, 2, 3)
before = [
('BUILD_LIST', 0, 0),
('NOP', None, 0),
('LOAD_SMALL_INT', 1, 0),
('NOP', None, 0),
('NOP', None, 0),
('LIST_APPEND', 1, 0),
('NOP', None, 0),
('LOAD_SMALL_INT', 2, 0),
('NOP', None, 0),
('NOP', None, 0),
('LIST_APPEND', 1, 0),
('NOP', None, 0),
('LOAD_SMALL_INT', 3, 0),
('NOP', None, 0),
('LIST_APPEND', 1, 0),
('NOP', None, 0),
('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0),
('RETURN_VALUE', None, 0)
]
after = [
('LOAD_CONST', 0, 0),
('RETURN_VALUE', None, 0)
]
self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)])
# no sequence start
same = [
('LOAD_SMALL_INT', 1, 0),
('LIST_APPEND', 1, 0),
('LOAD_SMALL_INT', 2, 0),
('LIST_APPEND', 1, 0),
('LOAD_SMALL_INT', 3, 0),
('LIST_APPEND', 1, 0),
('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0),
('RETURN_VALUE', None, 0)
]
self.cfg_optimization_test(same, same, consts=[])
def test_optimize_if_const_list(self):
before = [
('NOP', None, 0),

View file

@ -9,19 +9,19 @@ unsigned char M_test_frozenmain[] = {
30,0,89,1,78,8,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,32,0,50,0,0,0,0,0,
0,0,80,4,43,26,0,0,0,0,0,0,0,0,0,0,
112,5,80,5,15,0,68,24,0,0,112,6,89,2,32,0,
80,6,89,6,11,0,80,7,89,5,89,6,43,26,0,0,
112,5,80,7,15,0,68,24,0,0,112,6,89,2,32,0,
80,5,89,6,11,0,80,6,89,5,89,6,43,26,0,0,
0,0,0,0,0,0,0,0,11,0,48,4,50,1,0,0,
0,0,0,0,30,0,73,26,0,0,8,0,29,0,80,1,
34,0,41,8,233,0,0,0,0,78,122,18,70,114,111,122,
101,110,32,72,101,108,108,111,32,87,111,114,108,100,122,8,
115,121,115,46,97,114,103,118,218,6,99,111,110,102,105,103,
41,5,218,12,112,114,111,103,114,97,109,95,110,97,109,101,
218,10,101,120,101,99,117,116,97,98,108,101,218,15,117,115,
101,95,101,110,118,105,114,111,110,109,101,110,116,218,17,99,
111,110,102,105,103,117,114,101,95,99,95,115,116,100,105,111,
218,14,98,117,102,102,101,114,101,100,95,115,116,100,105,111,
122,7,99,111,110,102,105,103,32,122,2,58,32,41,7,218,
122,7,99,111,110,102,105,103,32,122,2,58,32,41,5,218,
12,112,114,111,103,114,97,109,95,110,97,109,101,218,10,101,
120,101,99,117,116,97,98,108,101,218,15,117,115,101,95,101,
110,118,105,114,111,110,109,101,110,116,218,17,99,111,110,102,
105,103,117,114,101,95,99,95,115,116,100,105,111,218,14,98,
117,102,102,101,114,101,100,95,115,116,100,105,111,41,7,218,
3,115,121,115,218,17,95,116,101,115,116,105,110,116,101,114,
110,97,108,99,97,112,105,218,5,112,114,105,110,116,218,4,
97,114,103,118,218,11,103,101,116,95,99,111,110,102,105,103,

View file

@ -390,44 +390,6 @@ fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
return 1;
}
static PyObject*
make_const_tuple(asdl_expr_seq *elts)
{
for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) {
expr_ty e = (expr_ty)asdl_seq_GET(elts, i);
if (e->kind != Constant_kind) {
return NULL;
}
}
PyObject *newval = PyTuple_New(asdl_seq_LEN(elts));
if (newval == NULL) {
return NULL;
}
for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) {
expr_ty e = (expr_ty)asdl_seq_GET(elts, i);
PyObject *v = e->v.Constant.value;
PyTuple_SET_ITEM(newval, i, Py_NewRef(v));
}
return newval;
}
static int
fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state)
{
if (state->syntax_check_only) {
return 1;
}
PyObject *newval;
if (node->v.Tuple.ctx != Load)
return 1;
newval = make_const_tuple(node->v.Tuple.elts);
return make_const(node, newval, arena);
}
static int astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
static int astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state);
@ -620,7 +582,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state)
break;
case Tuple_kind:
CALL_SEQ(astfold_expr, expr, node_->v.Tuple.elts);
CALL(fold_tuple, expr_ty, node_);
break;
case Name_kind:
if (state->syntax_check_only) {

View file

@ -1688,11 +1688,26 @@ codegen_typealias(compiler *c, stmt_ty s)
return SUCCESS;
}
static bool
is_const_tuple(asdl_expr_seq *elts)
{
for (Py_ssize_t i = 0; i < asdl_seq_LEN(elts); i++) {
expr_ty e = (expr_ty)asdl_seq_GET(elts, i);
if (e->kind != Constant_kind) {
return false;
}
}
return true;
}
/* Return false if the expression is a constant value except named singletons.
Return true otherwise. */
static bool
check_is_arg(expr_ty e)
{
if (e->kind == Tuple_kind) {
return !is_const_tuple(e->v.Tuple.elts);
}
if (e->kind != Constant_kind) {
return true;
}

View file

@ -1482,6 +1482,95 @@ fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const
return instr_make_load_const(instr, const_tuple, consts, const_cache);
}
/* Replace:
BUILD_LIST 0
LOAD_CONST c1
LIST_APPEND 1
LOAD_CONST c2
LIST_APPEND 1
...
LOAD_CONST cN
LIST_APPEND 1
CALL_INTRINSIC_1 INTRINSIC_LIST_TO_TUPLE
with:
LOAD_CONST (c1, c2, ... cN)
*/
static int
fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i,
PyObject *consts, PyObject *const_cache)
{
assert(PyDict_CheckExact(const_cache));
assert(PyList_CheckExact(consts));
assert(i >= 0);
assert(i < bb->b_iused);
cfg_instr *intrinsic = &bb->b_instr[i];
assert(intrinsic->i_opcode == CALL_INTRINSIC_1);
assert(intrinsic->i_oparg == INTRINSIC_LIST_TO_TUPLE);
int consts_found = 0;
bool expect_append = true;
for (int pos = i - 1; pos >= 0; pos--) {
cfg_instr *instr = &bb->b_instr[pos];
int opcode = instr->i_opcode;
int oparg = instr->i_oparg;
if (opcode == NOP) {
continue;
}
if (opcode == BUILD_LIST && oparg == 0) {
if (!expect_append) {
/* Not a sequence start. */
return SUCCESS;
}
/* Sequence start, we are done. */
PyObject *newconst = PyTuple_New((Py_ssize_t)consts_found);
if (newconst == NULL) {
return ERROR;
}
for (int newpos = i - 1; newpos >= pos; newpos--) {
instr = &bb->b_instr[newpos];
if (instr->i_opcode == NOP) {
continue;
}
if (loads_const(instr->i_opcode)) {
PyObject *constant = get_const_value(instr->i_opcode, instr->i_oparg, consts);
if (constant == NULL) {
Py_DECREF(newconst);
return ERROR;
}
assert(consts_found > 0);
PyTuple_SET_ITEM(newconst, --consts_found, constant);
}
nop_out(&instr, 1);
}
assert(consts_found == 0);
return instr_make_load_const(intrinsic, newconst, consts, const_cache);
}
if (expect_append) {
if (opcode != LIST_APPEND || oparg != 1) {
return SUCCESS;
}
}
else {
if (!loads_const(opcode)) {
return SUCCESS;
}
consts_found++;
}
expect_append = !expect_append;
}
/* Did not find sequence start. */
return SUCCESS;
}
#define MIN_CONST_SEQUENCE_SIZE 3
/*
Optimize lists and sets for:
@ -2378,9 +2467,13 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts)
RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache));
break;
case CALL_INTRINSIC_1:
// for _ in (*foo, *bar) -> for _ in [*foo, *bar]
if (oparg == INTRINSIC_LIST_TO_TUPLE && nextop == GET_ITER) {
INSTR_SET_OP0(inst, NOP);
if (oparg == INTRINSIC_LIST_TO_TUPLE) {
if (nextop == GET_ITER) {
INSTR_SET_OP0(inst, NOP);
}
else {
RETURN_IF_ERROR(fold_constant_intrinsic_list_to_tuple(bb, i, consts, const_cache));
}
}
else if (oparg == INTRINSIC_UNARY_POSITIVE) {
RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache));