From 64bb0f3fc52953cac6aae873a25f94cdfd06404b Mon Sep 17 00:00:00 2001
From: Fabio Zadrozny <fabiofz@gmail.com>
Date: Thu, 7 May 2020 13:48:26 -0300
Subject: [PATCH] Consider filenames starting with '<' library code by default.
 Fixes #209

---
 .../pydevd/_pydevd_bundle/pydevd_api.py       |  2 +-
 .../pydevd/_pydevd_bundle/pydevd_constants.py | 17 ++++++
 .../pydevd/_pydevd_bundle/pydevd_filtering.py | 17 +++---
 .../_pydevd_bundle/pydevd_source_mapping.py   | 25 ++++++++-
 src/debugpy/_vendored/pydevd/pydevd.py        | 33 ++++++-----
 .../_debugger_case_source_mapping_jmc.py      | 35 ++++++++++++
 .../pydevd/tests_python/test_debugger_json.py | 56 +++++++++++++++++--
 .../tests_python/test_pydevd_filtering.py     |  6 +-
 8 files changed, 161 insertions(+), 30 deletions(-)
 create mode 100644 src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_source_mapping_jmc.py

diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py
index ca25aabc..46205402 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py
@@ -740,7 +740,7 @@ class PyDevdAPI(object):
         if supported_type:
             py_db.has_plugin_exception_breaks = py_db.plugin.has_exception_breaks()
         else:
-            raise NameError(exception_type)
+            pydev_log.info('No exception of type: %s was previously registered.', exception_type)
 
         py_db.on_breakpoints_changed(removed=True)
 
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
index fbdea662..49d3879d 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py
@@ -46,6 +46,23 @@ class DebugInfoHolder:
     PYDEVD_DEBUG_FILE = None
 
 
+# Any filename that starts with these strings is not traced nor shown to the user.
+# In Python 3.7 "<frozen ..." appears multiple times during import and should be ignored for the user.
+# In PyPy "<builtin> ..." can appear and should be ignored for the user.
+# <attrs is used internally by attrs
+# <__array_function__ is used by numpy
+IGNORE_BASENAMES_STARTING_WITH = ('<frozen ', '<builtin', '<attrs', '<__array_function__')
+
+# Note: <string> has special heuristics to know whether it should be traced or not (it's part of
+# user code when it's the <string> used in python -c and part of the library otherwise).
+
+# Any filename that starts with these strings is considered user (project) code. Note
+# that files for which we have a source mapping are also considered as a part of the project.
+USER_CODE_BASENAMES_STARTING_WITH = ('<ipython',)
+
+# Any filename that starts with these strings is considered library code (note: checked after USER_CODE_BASENAMES_STARTING_WITH).
+LIBRARY_CODE_BASENAMES_STARTING_WITH = ('<',)
+
 IS_CPYTHON = platform.python_implementation() == 'CPython'
 
 # Hold a reference to the original _getframe (because psyco will change that as soon as it's imported)
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_filtering.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_filtering.py
index 7741b3e7..0f24b7ed 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_filtering.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_filtering.py
@@ -9,6 +9,8 @@ import json
 from collections import namedtuple
 from _pydev_imps._pydev_saved_modules import threading
 from pydevd_file_utils import normcase
+from _pydevd_bundle.pydevd_constants import USER_CODE_BASENAMES_STARTING_WITH, \
+    LIBRARY_CODE_BASENAMES_STARTING_WITH
 
 try:
     xrange  # noqa
@@ -216,14 +218,15 @@ class FilesFiltering(object):
 
     def in_project_roots(self, filename):
         '''
-        Note: don't call directly. Use PyDb.in_project_scope (no caching here).
+        Note: don't call directly. Use PyDb.in_project_scope (there's no caching here and it doesn't
+        handle all possibilities for knowing whether a project is actually in the scope, it
+        just handles the heuristics based on the filename without the actual frame).
         '''
-        if filename.startswith('<'):  # Note: always use only startswith (pypy can have: "<builtin>some other name").
-            # This is a dummy filename that is usually used for eval or exec. Assume
-            # that it is user code, with one exception: <frozen ...> is used in the
-            # standard library.
-            in_project = not filename.startswith('<frozen ')
-            return in_project
+        if filename.startswith(USER_CODE_BASENAMES_STARTING_WITH):
+            return True
+
+        if filename.startswith(LIBRARY_CODE_BASENAMES_STARTING_WITH):
+            return False
 
         project_roots = self._get_project_roots()
 
diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_source_mapping.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_source_mapping.py
index 9dbad02d..4db193cd 100644
--- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_source_mapping.py
+++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_source_mapping.py
@@ -1,5 +1,5 @@
 import bisect
-from _pydevd_bundle.pydevd_constants import dict_items
+from _pydevd_bundle.pydevd_constants import dict_items, NULL
 
 
 class SourceMappingEntry(object):
@@ -45,10 +45,11 @@ class _KeyifyList(object):
 
 class SourceMapping(object):
 
-    def __init__(self):
+    def __init__(self, on_source_mapping_changed=NULL):
         self._mappings_to_server = {}
         self._mappings_to_client = {}
         self._cache = {}
+        self._on_source_mapping_changed = on_source_mapping_changed
 
     def set_source_mapping(self, source_filename, mapping):
         '''
@@ -86,11 +87,12 @@ class SourceMapping(object):
                 self._mappings_to_client[map_entry.runtime_source] = source_filename
         finally:
             self._cache.clear()
+            self._on_source_mapping_changed()
         return ''
 
     def map_to_client(self, filename, lineno):
         # Note: the filename must be normalized to the client after this point.
-        key = (filename, lineno, 'client')
+        key = (lineno, 'client', filename)
         try:
             return self._cache[key]
         except KeyError:
@@ -104,6 +106,22 @@ class SourceMapping(object):
             self._cache[key] = (filename, lineno, False)
             return self._cache[key]
 
+    def has_mapping_entry(self, filename):
+        # Note that we're not interested in the line here, just on knowing if a given filename
+        # (from the server) has a mapping for it.
+        key = ('has_entry', filename)
+        try:
+            return self._cache[key]
+        except KeyError:
+            for _source_filename, mapping in dict_items(self._mappings_to_server):
+                for map_entry in mapping:
+                    if map_entry.runtime_source == filename:
+                        self._cache[key] = True
+                        return self._cache[key]
+
+            self._cache[key] = False
+            return self._cache[key]
+
     def map_to_server(self, filename, lineno):
         # Note: the filename must be already normalized to the server at this point.
         changed = False
@@ -132,3 +150,4 @@ class SourceMapping(object):
                 changed = True
 
         return filename, lineno, changed
+
diff --git a/src/debugpy/_vendored/pydevd/pydevd.py b/src/debugpy/_vendored/pydevd/pydevd.py
index 4e4802ce..52e2ff0e 100644
--- a/src/debugpy/_vendored/pydevd/pydevd.py
+++ b/src/debugpy/_vendored/pydevd/pydevd.py
@@ -40,7 +40,7 @@ from _pydevd_bundle.pydevd_constants import (IS_JYTH_LESS25, get_thread_id, get_
     dict_keys, dict_iter_items, DebugInfoHolder, PYTHON_SUSPEND, STATE_SUSPEND, STATE_RUN, get_frame,
     clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY2, NULL,
     NO_FTRACE, IS_IRONPYTHON, JSON_PROTOCOL, IS_CPYTHON, HTTP_JSON_PROTOCOL, USE_CUSTOM_SYS_CURRENT_FRAMES_MAP, call_only_once,
-    ForkSafeLock)
+    ForkSafeLock, IGNORE_BASENAMES_STARTING_WITH, LIBRARY_CODE_BASENAMES_STARTING_WITH)
 from _pydevd_bundle.pydevd_defaults import PydevdCustomization  # Note: import alias used on pydev_monkey.
 from _pydevd_bundle.pydevd_custom_frames import CustomFramesContainer, custom_frames_container_init
 from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE, LIB_FILE, DONT_TRACE_DIRS
@@ -466,7 +466,10 @@ class PyDB(object):
         self._cmd_queue = defaultdict(_queue.Queue)  # Key is thread id or '*', value is Queue
         self.suspended_frames_manager = SuspendedFramesManager()
         self._files_filtering = FilesFiltering()
-        self.source_mapping = SourceMapping()
+        # Note: when the source mapping is changed we also have to clear the file types cache
+        # (because if a given file is a part of the project or not may depend on it being
+        # defined in the source mapping).
+        self.source_mapping = SourceMapping(on_source_mapping_changed=self._clear_filters_caches)
 
         # Determines whether we should terminate child processes when asked to terminate.
         self.terminate_child_processes = True
@@ -746,12 +749,12 @@ class PyDB(object):
 
     def _internal_get_file_type(self, abs_real_path_and_basename):
         basename = abs_real_path_and_basename[-1]
-        if basename.startswith('<frozen '):
-            # In Python 3.7 "<frozen ..." appears multiple times during import and should be
-            # ignored for the user.
-            return self.PYDEV_FILE
-        if abs_real_path_and_basename[0].startswith(('<builtin', '<attrs')):
-            # In PyPy "<builtin> ..." can appear and should be ignored for the user.
+        if (
+                basename.startswith(IGNORE_BASENAMES_STARTING_WITH) or
+                abs_real_path_and_basename[0].startswith(IGNORE_BASENAMES_STARTING_WITH)
+            ):
+            # Note: these are the files that are completely ignored (they aren't shown to the user
+            # as user nor library code as it's usually just noise in the frame stack).
             return self.PYDEV_FILE
         file_type = self._dont_trace_get_file_type(basename)
         if file_type is not None:
@@ -1022,11 +1025,15 @@ class PyDB(object):
             if file_type == self.PYDEV_FILE:
                 cache[cache_key] = False
 
-            elif file_type == self.LIB_FILE and filename == '<string>':
-                # This means it's a <string> which should be considered to be a library file and
-                # shouldn't be considered as a part of the project.
-                # (i.e.: lib files must be traced if they're put inside a project).
-                cache[cache_key] = False
+            elif filename == '<string>':
+                # Special handling for '<string>'
+                if file_type == self.LIB_FILE:
+                    cache[cache_key] = False
+                else:
+                    cache[cache_key] = True
+
+            elif self.source_mapping.has_mapping_entry(filename):
+                cache[cache_key] = True
 
             else:
                 cache[cache_key] = self._files_filtering.in_project_roots(filename)
diff --git a/src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_source_mapping_jmc.py b/src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_source_mapping_jmc.py
new file mode 100644
index 00000000..eaf54eac
--- /dev/null
+++ b/src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_source_mapping_jmc.py
@@ -0,0 +1,35 @@
+def full_function():
+    # Note that this function is not called, it's there just to make the mapping explicit.  # map to cell1, line 1
+    import sys  # map to cell1, line 2
+    frame = sys._getframe()  # map to cell1, line 3
+    if py_db.in_project_scope(frame, '<cell1>') != expect_in_project_scope:  # map to cell1, line 4
+        raise AssertionError('Expected <cell1> to be in project scope: %s' % (expect_in_project_scope,))  # map to cell1, line 5
+    a = 1  # map to cell1, line 6
+    b = 2  # map to cell1, line 7
+
+
+def create_code():
+    cell1_code = compile(''' # line 1
+import sys # line 2
+frame = sys._getframe() # line 3
+if py_db.in_project_scope(frame, '<cell1>') != expect_in_project_scope: # line 4
+    raise AssertionError('Expected <cell1> to be in project scope: %s' % (expect_in_project_scope,)) # line 5
+a = 1  # line 6
+b = 2  # line 7
+''', '<cell1>', 'exec')
+
+    return {'cell1': cell1_code}
+
+
+if __name__ == '__main__':
+    code = create_code()
+    import pydevd
+    py_db = pydevd.get_global_debugger()
+
+    expect_in_project_scope = True
+    exec(code['cell1'])  # When executing, stop at breakpoint and then remove the source mapping.
+
+    expect_in_project_scope = False
+    exec(code['cell1'])  # Should no longer stop.
+
+    print('TEST SUCEEDED')
diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py b/src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py
index cd133e79..217349c4 100644
--- a/src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py
+++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py
@@ -2670,7 +2670,8 @@ def test_source_mapping_errors(case_setup):
     'target',
     ['_debugger_case_source_mapping.py', '_debugger_case_source_mapping_and_reference.py']
 )
-def test_source_mapping(case_setup, target):
+@pytest.mark.parametrize('jmc', [True, False])
+def test_source_mapping_base(case_setup, target, jmc):
     from _pydevd_bundle._debug_adapter.pydevd_schema import Source
     from _pydevd_bundle._debug_adapter.pydevd_schema import PydevdSourceMap
 
@@ -2679,9 +2680,7 @@ def test_source_mapping(case_setup, target):
     with case_setup.test_file(target) as writer:
         json_facade = JsonFacade(writer)
 
-        json_facade.write_launch(
-            justMyCode=False,
-        )
+        json_facade.write_launch(justMyCode=jmc)
 
         map_to_cell_1_line2 = writer.get_line_index_with_content('map to cell1, line 2')
         map_to_cell_2_line2 = writer.get_line_index_with_content('map to cell2, line 2')
@@ -2726,6 +2725,55 @@ def test_source_mapping(case_setup, target):
         writer.finished_ok = True
 
 
+def test_source_mapping_just_my_code(case_setup):
+    from _pydevd_bundle._debug_adapter.pydevd_schema import Source
+    from _pydevd_bundle._debug_adapter.pydevd_schema import PydevdSourceMap
+
+    case_setup.check_non_ascii = True
+
+    with case_setup.test_file('_debugger_case_source_mapping_jmc.py') as writer:
+        json_facade = JsonFacade(writer)
+
+        json_facade.write_launch(justMyCode=True)
+
+        map_to_cell_1_line1 = writer.get_line_index_with_content('map to cell1, line 1')
+        map_to_cell_1_line6 = writer.get_line_index_with_content('map to cell1, line 6')
+        map_to_cell_1_line7 = writer.get_line_index_with_content('map to cell1, line 7')
+
+        cell1_map = PydevdSourceMap(map_to_cell_1_line1, map_to_cell_1_line7, Source(path='<cell1>'), 1)
+        pydevd_source_maps = [cell1_map]
+
+        # Set breakpoints before setting the source map (check that we reapply them).
+        json_facade.write_set_breakpoints(map_to_cell_1_line6)
+
+        test_file = writer.TEST_FILE
+        if isinstance(test_file, bytes):
+            # file is in the filesystem encoding (needed for launch) but protocol needs it in utf-8
+            test_file = test_file.decode(file_system_encoding)
+            test_file = test_file.encode('utf-8')
+
+        json_facade.write_set_pydevd_source_map(
+            Source(path=test_file),
+            pydevd_source_maps=pydevd_source_maps,
+        )
+
+        json_facade.write_make_initial_run()
+
+        json_hit = json_facade.wait_for_thread_stopped(line=map_to_cell_1_line6, file=os.path.basename(test_file))
+        for stack_frame in json_hit.stack_trace_response.body.stackFrames:
+            assert stack_frame['source']['sourceReference'] == 0
+
+        # i.e.: Remove the source maps
+        json_facade.write_set_pydevd_source_map(
+            Source(path=test_file),
+            pydevd_source_maps=[],
+        )
+
+        json_facade.write_continue()
+
+        writer.finished_ok = True
+
+
 @pytest.mark.skipif(not TEST_CHERRYPY or IS_PY39_OR_GREATER, reason='No CherryPy available.'
                     'Must investigate support on Python 3.9')
 def test_process_autoreload_cherrypy(case_setup_multiprocessing, tmpdir):
diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_pydevd_filtering.py b/src/debugpy/_vendored/pydevd/tests_python/test_pydevd_filtering.py
index b07a83fe..d174cb45 100644
--- a/src/debugpy/_vendored/pydevd/tests_python/test_pydevd_filtering.py
+++ b/src/debugpy/_vendored/pydevd/tests_python/test_pydevd_filtering.py
@@ -50,12 +50,14 @@ def test_in_project_roots(tmpdir):
         (site_packages_inside_project_dir, False),
         (project_dir, True),
         (project_dir_inside_site_packages, False),
-        ('<foo>', True),
+        ('<foo>', False),
+        ('<ipython>', True),
         ('<frozen importlib._bootstrap>', False),
     ]
 
     for check_path, find in check:
-        assert files_filtering.in_project_roots(check_path) == find
+        assert files_filtering.in_project_roots(check_path) == find, \
+            'Expected: %s to be a part of the project: %s' % (check_path, find)
 
     sys.path.append(str(site_packages))
     try: