Fix issue with frame eval mode and multiple breakpoints in generator. Fixes #348

src/debugpy/_vendored/pydevd/_pydev_runfiles/pydev_runfiles_pytest2.py

@@ -164,10 +164,16 @@ except ImportError:
 
 def _get_error_contents_from_report(report):
     if report.longrepr is not None:
-        tw = TerminalWriter(stringio=True)
+        try:
+            tw = TerminalWriter(stringio=True)
+            stringio = tw.stringio
+        except TypeError:
+            import io
+            stringio = io.StringIO()
+            tw = TerminalWriter(file=stringio)
         tw.hasmarkup = False
         report.toterminal(tw)
-        exc = tw.stringio.getvalue()
+        exc = stringio.getvalue()
         s = exc.strip()
         if s:
             return s

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_eval_main.py

@@ -3,7 +3,7 @@ import os
 from _pydev_bundle import pydev_log
 from _pydevd_bundle.pydevd_trace_dispatch import USING_CYTHON
 from _pydevd_bundle.pydevd_constants import USE_CYTHON_FLAG, ENV_FALSE_LOWER_VALUES, \
-    ENV_TRUE_LOWER_VALUES, IS_PY36_OR_GREATER
+    ENV_TRUE_LOWER_VALUES, IS_PY36_OR_GREATER, SUPPORT_GEVENT
 
 frame_eval_func = None
 stop_frame_eval = None
@@ -18,6 +18,11 @@ use_frame_eval = os.environ.get('PYDEVD_USE_FRAME_EVAL', '').lower()
 if use_frame_eval in ENV_FALSE_LOWER_VALUES or USE_CYTHON_FLAG in ENV_FALSE_LOWER_VALUES or not USING_CYTHON:
     pass
 
+elif SUPPORT_GEVENT:
+    pass
+    # i.e gevent and frame eval mode don't get along very well.
+    # https://github.com/microsoft/debugpy/issues/189
+
 elif use_frame_eval in ENV_TRUE_LOWER_VALUES:
     # Fail if unable to use
     from _pydevd_frame_eval.pydevd_frame_eval_cython_wrapper import frame_eval_func, stop_frame_eval, dummy_trace_dispatch, clear_thread_local_info

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.pyx

@@ -1,17 +1,18 @@
 from __future__ import print_function
-import dis
 from _pydev_imps._pydev_saved_modules import threading, thread
 from _pydevd_bundle.pydevd_constants import GlobalDebuggerHolder
+import dis
 from _pydevd_frame_eval.pydevd_frame_tracing import create_pydev_trace_code_wrapper, update_globals_dict, dummy_tracing_holder
-from _pydevd_frame_eval.pydevd_modify_bytecode import insert_code
+from _pydevd_frame_eval.pydevd_modify_bytecode import insert_code, DebugHelper
 from pydevd_file_utils import get_abs_path_real_path_and_base_from_file, NORM_PATHS_AND_BASE_CONTAINER
 from _pydevd_bundle.pydevd_trace_dispatch import fix_top_level_trace_and_get_trace_func
 
 from _pydevd_bundle.pydevd_additional_thread_info import _set_additional_thread_info_lock
 from _pydevd_bundle.pydevd_cython cimport PyDBAdditionalThreadInfo
 
-
+_get_ident = threading.get_ident  # Note this is py3 only, if py2 needed to be supported, _get_ident would be needed.
 _thread_local_info = threading.local()
+_thread_active = threading._active
 
 def clear_thread_local_info():
     global _thread_local_info
@@ -26,6 +27,12 @@ cdef class ThreadInfo:
     cdef public bint fully_initialized
     cdef public object thread_trace_func
 
+    # Note: whenever get_func_code_info is called, this value is reset (we're using
+    # it as a thread-local value info).
+    # If True the debugger should not go into trace mode even if the new
+    # code for a function is None and there are breakpoints.
+    cdef public bint force_stay_in_untraced_mode
+
     def __init__(self):
         self.additional_info = None
         self.is_pydevd_thread = False
@@ -39,8 +46,8 @@ cdef class ThreadInfo:
         self.inside_frame_eval += 1
 
         try:
-            thread_ident = threading.get_ident()  # Note this is py3 only, if py2 needed to be supported, _get_ident would be needed.
-            t = threading._active.get(thread_ident)
+            thread_ident = _get_ident()
+            t = _thread_active.get(thread_ident)
             if t is None:
                 return  # Cannot initialize until thread becomes active.
 
@@ -140,16 +147,16 @@ def decref_py(obj):
     Py_DECREF(obj)
 
 
-def get_func_code_info_py(frame, code_obj) -> FuncCodeInfo:
+def get_func_code_info_py(thread_info, frame, code_obj) -> FuncCodeInfo:
     '''
     Helper to be called from Python.
     '''
-    return get_func_code_info(<PyFrameObject *> frame, <PyCodeObject *> code_obj)
+    return get_func_code_info(<ThreadInfo> thread_info, <PyFrameObject *> frame, <PyCodeObject *> code_obj)
 
 
 _code_extra_index: Py_SIZE = -1
 
-cdef FuncCodeInfo get_func_code_info(PyFrameObject * frame_obj, PyCodeObject * code_obj):
+cdef FuncCodeInfo get_func_code_info(ThreadInfo thread_info, PyFrameObject * frame_obj, PyCodeObject * code_obj):
     '''
     Provides code-object related info.
 
@@ -158,6 +165,7 @@ cdef FuncCodeInfo get_func_code_info(PyFrameObject * frame_obj, PyCodeObject * c
 
     get_thread_info() *must* be called at least once before get_func_code_info()
     to initialize _code_extra_index.
+
     '''
     # f_code = <object> code_obj
     # DEBUG = f_code.co_filename.endswith('_debugger_case_multiprocessing.py')
@@ -165,6 +173,7 @@ cdef FuncCodeInfo get_func_code_info(PyFrameObject * frame_obj, PyCodeObject * c
     #     print('get_func_code_info', f_code.co_name, f_code.co_filename)
 
     cdef object main_debugger = GlobalDebuggerHolder.global_dbg
+    thread_info.force_stay_in_untraced_mode = False  # This is an output value of the function.
 
     cdef PyObject * extra
     _PyCode_GetExtra(<PyObject *> code_obj, _code_extra_index, & extra)
@@ -179,8 +188,6 @@ cdef FuncCodeInfo get_func_code_info(PyFrameObject * frame_obj, PyCodeObject * c
                 return func_code_info_obj
 
     cdef str co_filename = <str> code_obj.co_filename
-    cdef str co_name = <str> code_obj.co_name
-    cdef set break_at_lines
     cdef dict cache_file_type
     cdef tuple cache_file_type_key
 
@@ -210,38 +217,34 @@ cdef FuncCodeInfo get_func_code_info(PyFrameObject * frame_obj, PyCodeObject * c
             func_code_info.always_skip_code = True
 
     if not func_code_info.always_skip_code:
-        was_break: bool = False
         if main_debugger is not None:
+
             breakpoints: dict = main_debugger.breakpoints.get(func_code_info.real_path)
             # print('\n---')
             # print(main_debugger.breakpoints)
             # print(func_code_info.real_path)
             # print(main_debugger.breakpoints.get(func_code_info.real_path))
             code_obj_py: object = <object> code_obj
-            if breakpoints:
+            cached_code_obj_info: object = _cache.get(code_obj_py)
+            if cached_code_obj_info:
+                # The cache is for new code objects, so, in this case it's already
+                # using the new code and we can't change it as this is a generator!
+                # There's still a catch though: even though we don't replace the code,
+                # we may not want to go into tracing mode (as would usually happen
+                # when the new_code is None).
+                func_code_info.new_code = None
+                breakpoint_found, thread_info.force_stay_in_untraced_mode = \
+                    cached_code_obj_info.compute_force_stay_in_untraced_mode(breakpoints)
+                func_code_info.breakpoint_found = breakpoint_found
+
+            elif breakpoints:
                 # if DEBUG:
                 #    print('found breakpoints', code_obj_py.co_name, breakpoints)
-                break_at_lines = set()
-                new_code = None
-                for offset, line in dis.findlinestarts(code_obj_py):
-                    if line in breakpoints:
-                        # breakpoint = breakpoints[line]
-                        # if DEBUG:
-                        #    print('created breakpoint', code_obj_py.co_name, line)
-                        func_code_info.breakpoint_found = True
-                        break_at_lines.add(line)
-
-                        success, new_code = insert_code(
-                            code_obj_py, create_pydev_trace_code_wrapper(line), line, tuple(break_at_lines))
-                        code_obj_py = new_code
-
-                        if not success:
-                            func_code_info.new_code = None
-                            break
-                else:
-                    # Ok, all succeeded, set to generated code object.
-                    func_code_info.new_code = new_code
 
+                # Note: new_code can be None if unable to generate.
+                # It should automatically put the new code object in the cache.
+                breakpoint_found, func_code_info.new_code = generate_code_with_breakpoints(code_obj_py, breakpoints)
+                func_code_info.breakpoint_found = breakpoint_found
 
     Py_INCREF(func_code_info)
     _PyCode_SetExtra(<PyObject *> code_obj, _code_extra_index, <PyObject *> func_code_info)
@@ -249,6 +252,159 @@ cdef FuncCodeInfo get_func_code_info(PyFrameObject * frame_obj, PyCodeObject * c
     return func_code_info
 
 
+cdef class _CodeLineInfo:
+
+    cdef public dict line_to_offset
+    cdef public int first_line
+    cdef public int last_line
+
+    def __init__(self, dict line_to_offset,  int first_line,  int last_line):
+        self.line_to_offset = line_to_offset
+        self.first_line = first_line
+        self.last_line = last_line
+
+
+# Note: this method has a version in pure-python too.
+def _get_code_line_info(code_obj):
+    line_to_offset: dict = {}
+    first_line: int = None
+    last_line: int = None
+
+    cdef int offset
+    cdef int line
+
+    for offset, line in dis.findlinestarts(code_obj):
+        line_to_offset[line] = offset
+
+    if line_to_offset:
+        first_line = min(line_to_offset)
+        last_line = max(line_to_offset)
+    return _CodeLineInfo(line_to_offset, first_line, last_line)
+
+
+# Note: this is a cache where the key is the code objects we create ourselves so that
+# we always return the same code object for generators.
+# (so, we don't have a cache from the old code to the new info -- that's actually
+# handled by the cython side in `FuncCodeInfo get_func_code_info` by providing the
+# same code info if the debugger mtime is still the same).
+_cache: dict = {}
+
+def get_cached_code_obj_info_py(code_obj_py):
+    '''
+    :return _CacheValue:
+    :note: on cython use _cache.get(code_obj_py) directly.
+    '''
+    return _cache.get(code_obj_py)
+
+
+cdef class _CacheValue(object):
+
+    cdef public object code_obj_py
+    cdef public _CodeLineInfo code_line_info
+    cdef public set breakpoints_hit_at_lines
+    cdef public set code_lines_as_set
+
+    def __init__(self, object code_obj_py, _CodeLineInfo code_line_info, set breakpoints_hit_at_lines):
+        '''
+        :param code_obj_py:
+        :param _CodeLineInfo code_line_info:
+        :param set[int] breakpoints_hit_at_lines:
+        '''
+        self.code_obj_py = code_obj_py
+        self.code_line_info = code_line_info
+        self.breakpoints_hit_at_lines = breakpoints_hit_at_lines
+        self.code_lines_as_set = set(code_line_info.line_to_offset)
+
+    def compute_force_stay_in_untraced_mode(self, breakpoints):
+        '''
+        :param breakpoints:
+            set(breakpoint_lines) or dict(breakpoint_line->breakpoint info)
+        :return tuple(breakpoint_found, force_stay_in_untraced_mode)
+        '''
+        force_stay_in_untraced_mode = False
+
+        target_breakpoints = self.code_lines_as_set.intersection(breakpoints)
+        breakpoint_found = bool(target_breakpoints)
+
+        if not breakpoint_found:
+            force_stay_in_untraced_mode = True
+        else:
+            force_stay_in_untraced_mode = self.breakpoints_hit_at_lines.issuperset(set(breakpoints))
+
+        return breakpoint_found, force_stay_in_untraced_mode
+
+def generate_code_with_breakpoints_py(object code_obj_py, dict breakpoints):
+    return generate_code_with_breakpoints(code_obj_py, breakpoints)
+
+# DEBUG = True
+# debug_helper = DebugHelper()
+
+cdef generate_code_with_breakpoints(object code_obj_py, dict breakpoints):
+    '''
+    :param breakpoints:
+        dict where the keys are the breakpoint lines.
+    :return tuple(breakpoint_found, new_code)
+    '''
+    # The cache is needed for generator functions, because after each yield a new frame
+    # is created but the former code object is used (so, check if code_to_modify is
+    # already there and if not cache based on the new code generated).
+
+    cdef bint success
+    cdef int breakpoint_line
+    cdef bint breakpoint_found
+    cdef _CacheValue cache_value
+    cdef set breakpoints_hit_at_lines
+    cdef dict line_to_offset
+
+    assert code_obj_py not in _cache, 'If a code object is cached, that same code object must be reused.'
+
+#     if DEBUG:
+#         initial_code_obj_py = code_obj_py
+
+    code_line_info = _get_code_line_info(code_obj_py)
+
+    success = True
+
+    breakpoints_hit_at_lines = set()
+    line_to_offset = code_line_info.line_to_offset
+
+    for breakpoint_line in reversed(sorted(breakpoints)):
+        if breakpoint_line in line_to_offset:
+            breakpoints_hit_at_lines.add(breakpoint_line)
+
+            success, new_code = insert_code(
+                code_obj_py,
+                create_pydev_trace_code_wrapper(breakpoint_line),
+                breakpoint_line,
+                code_line_info
+            )
+
+            if not success:
+                code_obj_py = None
+                break
+
+            code_obj_py = new_code
+
+    breakpoint_found = bool(breakpoints_hit_at_lines)
+    if breakpoint_found and success:
+#         if DEBUG:
+#             op_number = debug_helper.write_dis(
+#                 'inserting code, breaks at: %s' % (list(breakpoints),),
+#                 initial_code_obj_py
+#             )
+#
+#             debug_helper.write_dis(
+#                 'after inserting code, breaks at: %s' % (list(breakpoints,)),
+#                 code_obj_py,
+#                 op_number=op_number,
+#             )
+
+        cache_value = _CacheValue(code_obj_py, code_line_info, breakpoints_hit_at_lines)
+        _cache[code_obj_py] = cache_value
+
+    return breakpoint_found, code_obj_py
+
+
 cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc):
     '''
     This function makes the actual evaluation and changes the bytecode to a version
@@ -323,7 +479,7 @@ cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc
             else:
                 frame.f_trace = <object> main_debugger.trace_dispatch
         else:
-            func_code_info: FuncCodeInfo = get_func_code_info(frame_obj, frame_obj.f_code)
+            func_code_info: FuncCodeInfo = get_func_code_info(thread_info, frame_obj, frame_obj.f_code)
             # if DEBUG:
             #     print('get_bytecode_while_frame_eval always skip', func_code_info.always_skip_code)
             if not func_code_info.always_skip_code:
@@ -342,22 +498,27 @@ cdef PyObject * get_bytecode_while_frame_eval(PyFrameObject * frame_obj, int exc
                 if can_skip and func_code_info.breakpoint_found:
                     # if DEBUG:
                     #     print('get_bytecode_while_frame_eval new_code', func_code_info.new_code)
-
-                    # If breakpoints are found but new_code is None,
-                    # this means we weren't able to actually add the code
-                    # where needed, so, fallback to tracing.
-                    if func_code_info.new_code is None:
-                        if thread_info.thread_trace_func is not None:
-                            frame.f_trace = thread_info.thread_trace_func
+                    if not thread_info.force_stay_in_untraced_mode:
+                        # If breakpoints are found but new_code is None,
+                        # this means we weren't able to actually add the code
+                        # where needed, so, fallback to tracing.
+                        if func_code_info.new_code is None:
+                            if thread_info.thread_trace_func is not None:
+                                frame.f_trace = thread_info.thread_trace_func
+                            else:
+                                frame.f_trace = <object> main_debugger.trace_dispatch
                         else:
-                            frame.f_trace = <object> main_debugger.trace_dispatch
+                            # print('Using frame eval break for', <object> frame_obj.f_code.co_name)
+                            update_globals_dict(<object> frame_obj.f_globals)
+                            Py_INCREF(func_code_info.new_code)
+                            old = <object> frame_obj.f_code
+                            frame_obj.f_code = <PyCodeObject *> func_code_info.new_code
+                            Py_DECREF(old)
                     else:
-                        # print('Using frame eval break for', <object> frame_obj.f_code.co_name)
+                        # When we're forcing to stay in traced mode we need to
+                        # update the globals dict (because this means that we're reusing
+                        # a previous code which had breakpoints added in a new frame).
                         update_globals_dict(<object> frame_obj.f_globals)
-                        Py_INCREF(func_code_info.new_code)
-                        old = <object> frame_obj.f_code
-                        frame_obj.f_code = <PyCodeObject *> func_code_info.new_code
-                        Py_DECREF(old)
 
     finally:
         thread_info.inside_frame_eval -= 1

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_tracing.py

@@ -60,7 +60,7 @@ def _pydev_stop_at_break(line):
             return
 
         if python_breakpoint:
-            pydev_log.debug("Suspending at breakpoint in file: {} on line {}".format(frame.f_code.co_filename, line))
+            pydev_log.debug("Setting f_trace due to frame eval mode in file: %s on line %s", frame.f_code.co_filename, line)
             t.additional_info.trace_suspend_type = 'frame_eval'
 
             pydevd_frame_eval_cython_wrapper = sys.modules['_pydevd_frame_eval.pydevd_frame_eval_cython_wrapper']

src/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_modify_bytecode.py

@@ -3,11 +3,44 @@ from opcode import opmap, EXTENDED_ARG, HAVE_ARGUMENT
 from types import CodeType
 from _pydev_bundle import pydev_log
 from _pydevd_bundle.pydevd_constants import IS_PY38_OR_GREATER
+import os.path
+import itertools
+from functools import partial
+from collections import namedtuple
 
 MAX_BYTE = 255
 RETURN_VALUE_SIZE = 2
 
 
+class DebugHelper(object):
+
+    def __init__(self):
+        self._debug_dir = os.path.join(os.path.dirname(__file__), 'debug_info')
+        try:
+            os.makedirs(self._debug_dir)
+        except:
+            pass
+        self._next = partial(next, itertools.count(0))
+
+    def write_dis(self, msg, code_to_modify, op_number=None):
+        if op_number is None:
+            op_number = self._next()
+            name = '%03d_before.txt' % op_number
+        else:
+            name = '%03d_change.txt' % op_number
+
+        filename = os.path.join(self._debug_dir, name)
+        with open(filename, 'w') as stream:
+            stream.write('-------- ')
+            stream.write(msg)
+            stream.write('\n')
+            stream.write('-------- ')
+            stream.write('id(code_to_modify): %s' % id(code_to_modify))
+            stream.write('\n\n')
+            dis.dis(code_to_modify, file=stream)
+        return op_number
+
+
 def _add_attr_values_from_insert_to_original(original_code, insert_code, insert_code_list, attribute_name, op_list):
     """
     This function appends values of the attribute `attribute_name` of the inserted code to the original values,
@@ -183,44 +216,25 @@ def add_jump_instruction(jump_arg, code_to_insert):
     return list(code_to_insert.co_code[:-RETURN_VALUE_SIZE]) + extended_arg_list + [opmap['POP_JUMP_IF_TRUE'], jump_arg]
 
 
-_created = {}
+_CodeLineInfo = namedtuple('_CodeLineInfo', 'line_to_offset, first_line, last_line')
 
 
-def insert_code(code_to_modify, code_to_insert, before_line, all_lines_with_breaks=()):
-    '''
-    :param all_lines_with_breaks:
-        tuple(int) a tuple with all the breaks in the given code object (this method is expected
-        to be called multiple times with different lines to add multiple breakpoints, so, the
-        variable `before_line` should have the current breakpoint an the all_lines_with_breaks
-        should have all the breakpoints added so far (including the `before_line`).
-    '''
-    if not all_lines_with_breaks:
-        # Backward-compatibility with signature which received only one line.
-        all_lines_with_breaks = (before_line,)
+# Note: this method has a version in cython too.
+def _get_code_line_info(code_obj):
+    line_to_offset = {}
+    first_line = None
+    last_line = None
 
-    # The cache is needed for generator functions, because after each yield a new frame
-    # is created but the former code object is used (so, check if code_to_modify is
-    # already there and if not cache based on the new code generated).
+    for offset, line in dis.findlinestarts(code_obj):
+        line_to_offset[line] = offset
 
-    # print('inserting code', before_line, all_lines_with_breaks)
-    # dis.dis(code_to_modify)
-
-    ok_and_new_code = _created.get((code_to_modify, all_lines_with_breaks))
-    if ok_and_new_code is not None:
-        return ok_and_new_code
-
-    ok, new_code = _insert_code(code_to_modify, code_to_insert, before_line)
-
-    # print('insert code ok', ok)
-    # dis.dis(new_code)
-
-    # Note: caching with new code!
-    cache_key = new_code, all_lines_with_breaks
-    _created[cache_key] = (ok, new_code)
-    return _created[cache_key]
+    if line_to_offset:
+        first_line = min(line_to_offset)
+        last_line = max(line_to_offset)
+    return _CodeLineInfo(line_to_offset, first_line, last_line)
 
 
-def _insert_code(code_to_modify, code_to_insert, before_line):
+def insert_code(code_to_modify, code_to_insert, before_line, code_line_info=None):
     """
     Insert piece of code `code_to_insert` to `code_to_modify` right inside the line `before_line` before the
     instruction on this line by modifying original bytecode
@@ -230,26 +244,23 @@ def _insert_code(code_to_modify, code_to_insert, before_line):
     :param before_line: Number of line for code insertion
     :return: boolean flag whether insertion was successful, modified code
     """
-    linestarts = dict(dis.findlinestarts(code_to_modify))
-    if not linestarts:
+    if code_line_info is None:
+        code_line_info = _get_code_line_info(code_to_modify)
+
+    if not code_line_info.line_to_offset:
         return False, code_to_modify
 
     if code_to_modify.co_name == '<module>':
         # There's a peculiarity here: if a breakpoint is added in the first line of a module, we
         # can't replace the code because we require a line event to stop and the line event
         # was already generated, so, fallback to tracing.
-        if before_line == min(linestarts.values()):
+        if before_line == code_line_info.first_line:
             return False, code_to_modify
 
-    if before_line not in linestarts.values():
+    offset = code_line_info.line_to_offset.get(before_line)
+    if offset is None:
         return False, code_to_modify
 
-    offset = None
-    for off, line_no in linestarts.items():
-        if line_no == before_line:
-            offset = off
-            break
-
     code_to_insert_list = add_jump_instruction(offset, code_to_insert)
     try:
         code_to_insert_list, new_names = \

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case17.py

@@ -1,38 +1,44 @@
 def get_here():
     a = 10
 
-def foo(func): 
+
+def foo(func):
     return func
 
-def m1(): # @DontTrace
+
+def m1():  # @DontTrace
     get_here()
 
+
 # @DontTrace
 def m2():
     get_here()
 
+
 # @DontTrace
 @foo
 def m3():
     get_here()
 
+
 @foo
 @foo
-def m4(): # @DontTrace
+def m4():  # @DontTrace
     get_here()
 
 
 def main():
 
-    m1()
-    
-    m2()
-    
-    m3()
-    
-    m4()
+    m1()  # break1
+
+    m2()  # break2
+
+    m3()  # break3
+
+    m4()  # break4
+
 
 if __name__ == '__main__':
     main()
-    
+
     print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_yield_from.py

@@ -0,0 +1,12 @@
+def method():
+    a = [1, 2]
+
+    def b():  # break1
+        yield from [j for j in a if j % 2 == 0]  # break2
+
+    for j in b():
+        print(j)
+
+
+method()
+print('TEST SUCEEDED')

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -807,19 +807,35 @@ def test_case_17(case_setup):
     # Check dont trace
     with case_setup.test_file('_debugger_case17.py') as writer:
         writer.write_enable_dont_trace(True)
-        writer.write_add_breakpoint(27, 'main')
-        writer.write_add_breakpoint(29, 'main')
-        writer.write_add_breakpoint(31, 'main')
-        writer.write_add_breakpoint(33, 'main')
+        writer.write_add_breakpoint(writer.get_line_index_with_content('break1'), 'main')
+        writer.write_add_breakpoint(writer.get_line_index_with_content('break2'), 'main')
+        writer.write_add_breakpoint(writer.get_line_index_with_content('break3'), 'main')
+        writer.write_add_breakpoint(writer.get_line_index_with_content('break4'), 'main')
         writer.write_make_initial_run()
 
-        for _i in range(4):
-            hit = writer.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT)
+        hit = writer.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT)
+        writer.write_step_in(hit.thread_id)
+        hit = writer.wait_for_breakpoint_hit('107', line=2)
+        # Should Skip step into properties setter
+        writer.write_run_thread(hit.thread_id)
 
-            writer.write_step_in(hit.thread_id)
-            hit = writer.wait_for_breakpoint_hit('107', line=2)
-            # Should Skip step into properties setter
-            writer.write_run_thread(hit.thread_id)
+        hit = writer.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT)
+        writer.write_step_in(hit.thread_id)
+        hit = writer.wait_for_breakpoint_hit('107', line=2)
+        # Should Skip step into properties setter
+        writer.write_run_thread(hit.thread_id)
+
+        hit = writer.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT)
+        writer.write_step_in(hit.thread_id)
+        hit = writer.wait_for_breakpoint_hit('107', line=2)
+        # Should Skip step into properties setter
+        writer.write_run_thread(hit.thread_id)
+
+        hit = writer.wait_for_breakpoint_hit(REASON_STOP_ON_BREAKPOINT)
+        writer.write_step_in(hit.thread_id)
+        hit = writer.wait_for_breakpoint_hit('107', line=2)
+        # Should Skip step into properties setter
+        writer.write_run_thread(hit.thread_id)
 
         writer.finished_ok = True
 

src/debugpy/_vendored/pydevd/tests_python/test_frame_eval_and_tracing.py

@@ -236,3 +236,24 @@ def test_frame_eval_change_breakpoints(case_setup_force_frame_eval):
 
         writer.finished_ok = True
 
+
+def test_generator_code_cache(case_setup_force_frame_eval):
+
+    with case_setup_force_frame_eval.test_file('_debugger_case_yield_from.py') as writer:
+        writer.write_add_breakpoint(writer.get_line_index_with_content('break1'))
+        writer.write_add_breakpoint(writer.get_line_index_with_content('break2'))
+        writer.write_make_initial_run()
+
+        hit = writer.wait_for_breakpoint_hit()
+        writer.write_run_thread(hit.thread_id)
+
+        hit = writer.wait_for_breakpoint_hit()
+        writer.write_run_thread(hit.thread_id)
+
+        hit = writer.wait_for_breakpoint_hit()
+        writer.write_run_thread(hit.thread_id)
+
+        hit = writer.wait_for_breakpoint_hit()
+        writer.write_run_thread(hit.thread_id)
+
+        writer.finished_ok = True

src/debugpy/_vendored/pydevd/tests_python/test_frame_evaluator.py

@@ -58,19 +58,74 @@ def _custom_global_dbg():
 def test_func_code_info(_times, _custom_global_dbg):
     from _pydevd_frame_eval import pydevd_frame_evaluator
     # Must be called before get_func_code_info_py to initialize the _code_extra_index.
-    pydevd_frame_evaluator.get_thread_info_py()
+    thread_info = pydevd_frame_evaluator.get_thread_info_py()
 
-    func_info = pydevd_frame_evaluator.get_func_code_info_py(method(), method.__code__)
+    func_info = pydevd_frame_evaluator.get_func_code_info_py(thread_info, method(), method.__code__)
     assert func_info.co_filename is method.__code__.co_filename
-    func_info2 = pydevd_frame_evaluator.get_func_code_info_py(method(), method.__code__)
+    func_info2 = pydevd_frame_evaluator.get_func_code_info_py(thread_info, method(), method.__code__)
     assert func_info is func_info2
 
     some_func = eval('lambda:sys._getframe()')
-    func_info3 = pydevd_frame_evaluator.get_func_code_info_py(some_func(), some_func.__code__)
+    func_info3 = pydevd_frame_evaluator.get_func_code_info_py(thread_info, some_func(), some_func.__code__)
     del some_func
     del func_info3
 
     some_func = eval('lambda:sys._getframe()')
-    pydevd_frame_evaluator.get_func_code_info_py(some_func(), some_func.__code__)
-    func_info = pydevd_frame_evaluator.get_func_code_info_py(some_func(), some_func.__code__)
-    assert pydevd_frame_evaluator.get_func_code_info_py(some_func(), some_func.__code__) is func_info
+    pydevd_frame_evaluator.get_func_code_info_py(thread_info, some_func(), some_func.__code__)
+    func_info = pydevd_frame_evaluator.get_func_code_info_py(thread_info, some_func(), some_func.__code__)
+    assert pydevd_frame_evaluator.get_func_code_info_py(thread_info, some_func(), some_func.__code__) is func_info
+
+
+def test_generate_code_with_breakpoints():
+    from _pydevd_frame_eval.pydevd_frame_evaluator import generate_code_with_breakpoints_py
+    from _pydevd_frame_eval.pydevd_frame_evaluator import get_cached_code_obj_info_py
+
+    def create_breakpoints_dict(lines):
+        return dict((line, None) for line in lines)
+
+    def method():
+        a = 1
+        a = 2
+        a = 3
+
+    breakpoint_found, new_code = generate_code_with_breakpoints_py(
+        method.__code__,
+        create_breakpoints_dict([method.__code__.co_firstlineno + 1, method.__code__.co_firstlineno + 2])
+    )
+
+    assert breakpoint_found
+    with pytest.raises(AssertionError):
+        # We must use the cached one directly (in the real-world, this would indicate a reuse
+        # of the code object -- which is related to generator handling).
+        generate_code_with_breakpoints_py(
+            new_code,
+            create_breakpoints_dict([method.__code__.co_firstlineno + 1])
+        )
+
+    cached_value = get_cached_code_obj_info_py(new_code)
+    breakpoint_found, force_stay_in_untraced_mode = cached_value.compute_force_stay_in_untraced_mode(
+        create_breakpoints_dict([method.__code__.co_firstlineno + 1]))
+
+    assert breakpoint_found
+    assert force_stay_in_untraced_mode
+
+    # i.e.: no breakpoints match (stay in untraced mode)
+    breakpoint_found, force_stay_in_untraced_mode = cached_value.compute_force_stay_in_untraced_mode(
+        create_breakpoints_dict([method.__code__.co_firstlineno + 10]))
+
+    assert not breakpoint_found
+    assert force_stay_in_untraced_mode
+
+    # i.e.: one of the breakpoints match (stay in untraced mode)
+    breakpoint_found, force_stay_in_untraced_mode = cached_value.compute_force_stay_in_untraced_mode(
+        create_breakpoints_dict([method.__code__.co_firstlineno + 2]))
+
+    assert breakpoint_found
+    assert force_stay_in_untraced_mode
+
+    # i.e.: one of the breakpoints doesn't match (leave untraced mode)
+    breakpoint_found, force_stay_in_untraced_mode = cached_value.compute_force_stay_in_untraced_mode(
+        create_breakpoints_dict([method.__code__.co_firstlineno + 3]))
+
+    assert breakpoint_found
+    assert not force_stay_in_untraced_mode

src/debugpy/_vendored/pydevd/third_party/cython_json.py

@@ -8,61 +8,88 @@ import os
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+# Note: Cython has some recursive structures in some classes, so, parsing only what we really
+# expect may be a bit better (although our recursion check should get that too).
+accepted_info = {
+    'PyClassDef': set(['name', 'doc', 'body', 'bases', 'decorators', 'pos'])
+}
 
-def node_to_dict(node, _recurse_level=0):
-    _recurse_level += 1
-    assert _recurse_level < 5000, "It seems we are recursing..."
+def node_to_dict(node, _recurse_level=0, memo=None):
+    nodeid = id(node)  # just to be sure it's checked by identity in the memo
+    if memo is None:
+        memo = {}
+    else:
+        if nodeid in memo:
+            # i.e.: prevent Nodes recursion.
+            return None
+    memo[nodeid] = 1
+    try:
+        _recurse_level += 1
+        assert _recurse_level < 500, "It seems we are recursing..."
 
-    node_name = node.__class__.__name__
-    # print((' ' * _recurse_level) + node_name)
-    if node_name.endswith("Node"):
-        node_name = node_name[:-4]
-    data = {"__node__": node_name}
-    if _recurse_level == 1:
-        data['__version__'] = Cython.__version__
+        node_name = node.__class__.__name__
+        # print((' ' * _recurse_level) + node_name)
+        if node_name.endswith("Node"):
+            node_name = node_name[:-4]
+        data = {"__node__": node_name}
+        if _recurse_level == 1:
+            data['__version__'] = Cython.__version__
 
-    for attr_name, attr in [(key, value) for key, value in node.__dict__.items()]:
-        if attr_name in ("pos", "position"):
-            data["line"] = attr[1]
-            data["col"] = attr[2]
-            continue
-
-        if isinstance(attr, Nodes.Node):
-            data[attr_name] = node_to_dict(attr, _recurse_level)
-
-        elif isinstance(attr, (list, tuple)):
-            lst = []
-
-            for x in attr:
-                if isinstance(x, Nodes.Node):
-                    lst.append(node_to_dict(x, _recurse_level))
-
-                elif isinstance(x, (bytes, str)):
-                    lst.append(x)
-
-                elif hasattr(x, 'encode'):
-                    lst.append(x.encode('utf-8', 'replace'))
-
-                elif isinstance(x, (list, tuple)):
-                    tup = []
-
-                    for y in x:
-                        if isinstance(y, (str, bytes)):
-                            tup.append(y)
-                        elif isinstance(y, Nodes.Node):
-                            tup.append(node_to_dict(y, _recurse_level))
-
-                    lst.append(tup)
-
-            data[attr_name] = lst
 
+        dct = node.__dict__
+        accepted = accepted_info.get(node_name)
+        if accepted is None:
+            items = [(key, value) for key, value in dct.items()]
         else:
-            data[attr_name] = str(attr)
+            # for key in dct.keys():
+            #     if key not in accepted:
+            #         print('Skipped: %s' % (key,))
+            items = [(key, dct[key]) for key in accepted]
+
+
+        for attr_name, attr in items:
+            if attr_name in ("pos", "position"):
+                data["line"] = attr[1]
+                data["col"] = attr[2]
+                continue
+
+            if isinstance(attr, Nodes.Node):
+                data[attr_name] = node_to_dict(attr, _recurse_level, memo)
+
+            elif isinstance(attr, (list, tuple)):
+                lst = []
+
+                for x in attr:
+                    if isinstance(x, Nodes.Node):
+                        lst.append(node_to_dict(x, _recurse_level, memo))
+
+                    elif isinstance(x, (bytes, str)):
+                        lst.append(x)
+
+                    elif hasattr(x, 'encode'):
+                        lst.append(x.encode('utf-8', 'replace'))
+
+                    elif isinstance(x, (list, tuple)):
+                        tup = []
+
+                        for y in x:
+                            if isinstance(y, (str, bytes)):
+                                tup.append(y)
+                            elif isinstance(y, Nodes.Node):
+                                tup.append(node_to_dict(y, _recurse_level, memo))
+
+                        lst.append(tup)
+
+                data[attr_name] = lst
+
+            else:
+                data[attr_name] = str(attr)
+    finally:
+        memo.pop(nodeid, None)
 
     return data
 
 
-
 def source_to_dict(source, name=None):
     from Cython.Compiler.TreeFragment import parse_from_strings, StatListNode
     # Right now we don't collect errors, but leave the API compatible already.

src/debugpy/_vendored/pydevd/third_party/tests_cython_json.py

@@ -23,6 +23,22 @@ from distutils import sysconfig
         contents = contents.decode('utf-8')
     source_to_dict(contents)
 
+def test_dump_class():
+    contents = u'''
+class A:pass
+'''
+    if isinstance(contents, bytes):
+        contents = contents.decode('utf-8')
+    source_to_dict(contents)
+
+def test_comp():
+    contents = u'''
+{i: j for i, j in a}
+'''
+    if isinstance(contents, bytes):
+        contents = contents.decode('utf-8')
+    source_to_dict(contents)
+
 def test_global():
     contents = u'''
 def method():