mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-04 08:59:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

[3.14] GH-135171: Fix generator expressions one last time (hopefully) (GH-135225)

Browse source 
* Add NULL check to FOR_ITER

* Move GET_ITER back to genexpr creation
This commit is contained in:
Mark Shannon 2025-06-09 11:04:23 +01:00 • committed by GitHub
parent 1497866198
commit a5274cd215
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 92 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Lib/test/test_dis.py
View file

@ -204,6 +204,7 @@ dis_bug1333982 = """\
LOAD_CONST 1 (<code object <genexpr> at 0x..., file "%s", line %d>)
MAKE_FUNCTION
LOAD_FAST_BORROW 0 (x)
GET_ITER
CALL 0
%3d LOAD_SMALL_INT 1
@ -832,6 +833,7 @@ Disassembly of <code object foo at 0x..., file "%s", line %d>:
MAKE_FUNCTION
SET_FUNCTION_ATTRIBUTE 8 (closure)
LOAD_DEREF 1 (y)
GET_ITER
CALL 0
CALL 1
RETURN_VALUE
@ -851,8 +853,7 @@ Disassembly of <code object <genexpr> at 0x..., file "%s", line %d>:
%4d RETURN_GENERATOR
POP_TOP
L1: RESUME 0
LOAD_FAST_BORROW 0 (.0)
GET_ITER
LOAD_FAST 0 (.0)
L2: FOR_ITER 14 (to L3)
STORE_FAST 1 (z)
LOAD_DEREF 2 (x)

21
Lib/test/test_generators.py
View file

@ -318,21 +318,26 @@ class ModifyUnderlyingIterableTest(unittest.TestCase):
yield x
return gen(range(10))
def process_tests(self, get_generator):
def process_tests(self, get_generator, is_expr):
err_iterator = "'.*' object is not an iterator"
err_iterable = "'.*' object is not iterable"
for obj in self.iterables:
g_obj = get_generator(obj)
with self.subTest(g_obj=g_obj, obj=obj):
self.assertListEqual(list(g_obj), list(obj))
if is_expr:
self.assertRaisesRegex(TypeError, err_iterator, list, g_obj)
else:
self.assertListEqual(list(g_obj), list(obj))
g_iter = get_generator(iter(obj))
with self.subTest(g_iter=g_iter, obj=obj):
self.assertListEqual(list(g_iter), list(obj))
err_regex = "'.*' object is not iterable"
for obj in self.non_iterables:
g_obj = get_generator(obj)
with self.subTest(g_obj=g_obj):
self.assertRaisesRegex(TypeError, err_regex, list, g_obj)
err = err_iterator if is_expr else err_iterable
self.assertRaisesRegex(TypeError, err, list, g_obj)
def test_modify_f_locals(self):
def modify_f_locals(g, local, obj):
@ -345,8 +350,8 @@ class ModifyUnderlyingIterableTest(unittest.TestCase):
def get_generator_genfunc(obj):
return modify_f_locals(self.genfunc(), 'it', obj)
self.process_tests(get_generator_genexpr)
self.process_tests(get_generator_genfunc)
self.process_tests(get_generator_genexpr, True)
self.process_tests(get_generator_genfunc, False)
def test_new_gen_from_gi_code(self):
def new_gen_from_gi_code(g, obj):
@ -359,8 +364,8 @@ class ModifyUnderlyingIterableTest(unittest.TestCase):
def get_generator_genfunc(obj):
return new_gen_from_gi_code(self.genfunc(), obj)
self.process_tests(get_generator_genexpr)
self.process_tests(get_generator_genfunc)
self.process_tests(get_generator_genexpr, True)
self.process_tests(get_generator_genfunc, False)
class ExceptionTest(unittest.TestCase):

8
Lib/test/test_genexps.py
View file

@ -131,6 +131,14 @@ Verify late binding for the outermost if-expression
>>> list(g)
[1, 9, 25, 49, 81]
Verify that the outermost for-expression makes an immediate check
for iterability
>>> (i for i in 6)
Traceback (most recent call last):
File "<pyshell#4>", line 1, in -toplevel-
(i for i in 6)
TypeError: 'int' object is not iterable
Verify late binding for the innermost for-expression
>>> g = ((i,j) for i in range(3) for j in range(x))

6
Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-18-57-30.gh-issue-135171.0YtLq6.rst Normal file
View file

@ -0,0 +1,6 @@
Reverts the behavior of generator expressions when created with a
non-iterable to the pre-3.13 behavior of raising a TypeError. It is no
longer possible to cause a crash in the debugger by altering the generator
expression's local variables. This is achieved by moving the ``GET_ITER``
instruction back to the creation of the generator expression and adding an
additional check to ``FOR_ITER``.

27
Python/bytecodes.c
View file

@ -3155,7 +3155,14 @@ dummy_func(
replaced op(_FOR_ITER, (iter -- iter, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
if (func == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.100s' object is not an iterator",
Py_TYPE(iter_o)->tp_name);
ERROR_NO_POP();
}
PyObject *next_o = func(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
@ -3179,7 +3186,14 @@ dummy_func(
op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) {
/* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
if (func == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.100s' object is not an iterator",
Py_TYPE(iter_o)->tp_name);
ERROR_NO_POP();
}
PyObject *next_o = func(iter_o);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
@ -3202,7 +3216,14 @@ dummy_func(
inst(INSTRUMENTED_FOR_ITER, (unused/1, iter -- iter, next)) {
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
if (func == NULL) {
_PyErr_Format(tstate, PyExc_TypeError,
"'%.100s' object is not an iterator",
Py_TYPE(iter_o)->tp_name);
ERROR_NO_POP();
}
PyObject *next_o = func(iter_o);
if (next_o != NULL) {
next = PyStackRef_FromPyObjectSteal(next_o);
INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT);

11
Python/codegen.c
View file

@ -4411,7 +4411,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators,
gen_index);
int is_outer_genexpr = gen_index == 0 && type == COMP_GENEXP;
if (!iter_on_stack) {
if (gen_index == 0) {
assert(METADATA(c)->u_argcount == 1);
@ -4442,14 +4442,15 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
}
if (IS_JUMP_TARGET_LABEL(start)) {
VISIT(c, expr, gen->iter);
ADDOP(c, LOC(gen->iter), GET_ITER);
}
}
}
if (IS_JUMP_TARGET_LABEL(start)) {
depth++;
ADDOP(c, LOC(gen->iter), GET_ITER);
if (!is_outer_genexpr) {
ADDOP(c, LOC(gen->iter), GET_ITER);
}
USE_LABEL(c, start);
ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor);
}
@ -4775,6 +4776,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
location loc = LOC(e);
outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
int is_sync_genexpr = type == COMP_GENEXP && !outermost->is_async;
if (is_inlined) {
VISIT(c, expr, outermost->iter);
if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) {
@ -4851,6 +4853,9 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
Py_CLEAR(co);
VISIT(c, expr, outermost->iter);
if (is_sync_genexpr) {
ADDOP(c, loc, GET_ITER);
}
ADDOP_I(c, loc, CALL, 0);
if (is_async_comprehension && type != COMP_GENEXP) {

11
Python/executor_cases.c.h generated
View file

@ -4273,8 +4273,17 @@
_PyStackRef next;
iter = stack_pointer[-1];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
if (func == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_Format(tstate, PyExc_TypeError,
"'%.100s' object is not an iterator",
Py_TYPE(iter_o)->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_ERROR();
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
PyObject *next_o = func(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {

22
Python/generated_cases.c.h generated
View file

@ -5724,8 +5724,17 @@
// _FOR_ITER
{
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
if (func == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_Format(tstate, PyExc_TypeError,
"'%.100s' object is not an iterator",
Py_TYPE(iter_o)->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
PyObject *next_o = func(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o == NULL) {
if (_PyErr_Occurred(tstate)) {
@ -7037,8 +7046,17 @@
/* Skip 1 cache entry */
iter = stack_pointer[-1];
PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter);
iternextfunc func = Py_TYPE(iter_o)->tp_iternext;
if (func == NULL) {
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyErr_Format(tstate, PyExc_TypeError,
"'%.100s' object is not an iterator",
Py_TYPE(iter_o)->tp_name);
stack_pointer = _PyFrame_GetStackPointer(frame);
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o);
PyObject *next_o = func(iter_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (next_o != NULL) {
next = PyStackRef_FromPyObjectSteal(next_o);