Fixed #481: JustMyCode debugging not working in VS Code when virtualenv is inside workspace. (#640)

.gitignore

@@ -107,3 +107,4 @@ ENV/
 # PyDev
 .project
 .pydevproject
+.settings

ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py

@@ -2,6 +2,7 @@ from __future__ import nested_scopes
 import traceback
 import os
 import warnings
+import pydevd_file_utils
 
 try:
     from urllib import quote
@@ -9,10 +10,15 @@ except:
     from urllib.parse import quote  # @UnresolvedImport
 
 import inspect
-from _pydevd_bundle.pydevd_constants import IS_PY3K
+from _pydevd_bundle.pydevd_constants import IS_PY3K, get_global_debugger
 import sys
 from _pydev_bundle import pydev_log
 
+
+def _normpath(filename):
+    return pydevd_file_utils.get_abs_path_real_path_and_base_from_file(filename)[0]
+
+
 def save_main_module(file, module_name):
     # patch provided by: Scott Schlesier - when script is run, it does not
     # use globals from pydevd:
@@ -46,8 +52,8 @@ def to_number(x):
 
         l = x.find('(')
         if l != -1:
-            y = x[0:l-1]
-            #print y
+            y = x[0:l - 1]
+            # print y
             try:
                 n = float(y)
                 return n
@@ -55,6 +61,7 @@ def to_number(x):
                 pass
     return None
 
+
 def compare_object_attrs_key(x):
     if '__len__' == x:
         as_number = to_number(x)
@@ -65,28 +72,37 @@ def compare_object_attrs_key(x):
     else:
         return (-1, to_string(x))
 
+
 if IS_PY3K:
+
     def is_string(x):
         return isinstance(x, str)
 
 else:
+
     def is_string(x):
         return isinstance(x, basestring)
 
+
 def to_string(x):
     if is_string(x):
         return x
     else:
         return str(x)
 
+
 def print_exc():
     if traceback:
         traceback.print_exc()
 
+
 if IS_PY3K:
+
     def quote_smart(s, safe='/'):
         return quote(s, safe)
+
 else:
+
     def quote_smart(s, safe='/'):
         if isinstance(s, unicode):
             s = s.encode('utf-8')
@@ -119,50 +135,109 @@ def get_clsname_for_code(code, frame):
 
     return clsname
 
+
 _PROJECT_ROOTS_CACHE = []
+_LIBRARY_ROOTS_CACHE = []
 _FILENAME_TO_IN_SCOPE_CACHE = {}
 
-def set_project_roots(project_roots):
-    from _pydevd_bundle.pydevd_comm import get_global_debugger
+
+def _convert_to_str_and_clear_empty(roots):
     if sys.version_info[0] <= 2:
         # In py2 we need bytes for the files.
-        project_roots = [
+        roots = [
             root if not isinstance(root, unicode) else root.encode(sys.getfilesystemencoding())
-            for root in project_roots
+            for root in roots
         ]
-    pydev_log.debug("IDE_PROJECT_ROOTS %s\n" % project_roots)
+    
     new_roots = []
-    for root in project_roots:
-        new_roots.append(os.path.normcase(root))
+    for root in roots:
+        assert isinstance(root, str), '%s not str (found: %s)' % (root, type(root))
+        if root:
+            new_roots.append(root)
+    return new_roots
 
-    # Leave only the last one added.
-    _PROJECT_ROOTS_CACHE.append(new_roots)
-    del _PROJECT_ROOTS_CACHE[:-1]
 
+def _clear_caches_related_to_scope_changes():
     # Clear related caches.
     _FILENAME_TO_IN_SCOPE_CACHE.clear()
     debugger = get_global_debugger()
     if debugger is not None:
         debugger.clear_skip_caches()
 
-def _get_project_roots(project_roots_cache=_PROJECT_ROOTS_CACHE):
-    # Note: the project_roots_cache is the same instance among the many calls to the method
-    if not project_roots_cache:
-        roots = os.getenv('IDE_PROJECT_ROOTS', '').split(os.pathsep)
-        set_project_roots(roots)
-    return project_roots_cache[-1] # returns the project roots with case normalized
 
-
-def _get_library_roots(library_roots_cache=[]):
-    # Note: the project_roots_cache is the same instance among the many calls to the method
-    if not library_roots_cache:
-        roots = os.getenv('LIBRARY_ROOTS', '').split(os.pathsep)
-        pydev_log.debug("LIBRARY_ROOTS %s\n" % roots)
+def _set_roots(roots, cache):
+    roots = _convert_to_str_and_clear_empty(roots)
     new_roots = []
     for root in roots:
-            new_roots.append(os.path.normcase(root))
-        library_roots_cache.append(new_roots)
-    return library_roots_cache[-1] # returns the project roots with case normalized
+        new_roots.append(_normpath(root))
+    cache.append(new_roots)
+    # Leave only the last one added.
+    del cache[:-1]
+    _clear_caches_related_to_scope_changes()
+    return new_roots
+
+
+def _get_roots(cache, env_var, set_when_not_cached, get_default_val=None):
+    if not cache:
+        roots = os.getenv(env_var, None)
+        if roots is not None:
+            roots = roots.split(os.pathsep)
+        else:
+            if not get_default_val:
+                roots = []
+            else:
+                roots = get_default_val()
+        if not roots:
+            pydev_log.warn('%s being set to empty list.' % (env_var,))
+        set_when_not_cached(roots)
+    return cache[-1]  # returns the roots with case normalized
+
+
+def _get_default_library_roots():
+    # Provide sensible defaults if not in env vars.
+    import site
+    roots = [sys.prefix]
+    if hasattr(sys, 'base_prefix'):
+        roots.append(sys.base_prefix)
+    if hasattr(sys, 'real_prefix'):
+        roots.append(sys.real_prefix)
+
+    if hasattr(site, 'getusersitepackages'):
+        site_paths = site.getusersitepackages()
+        if isinstance(site_paths, (list, tuple)):
+            for site_path in site_paths:
+                roots.append(site_path)
+        else:
+            roots.append(site_paths)
+
+    if hasattr(site, 'getsitepackages'):
+        site_paths = site.getsitepackages()
+        if isinstance(site_paths, (list, tuple)):
+            for site_path in site_paths:
+                roots.append(site_path)
+        else:
+            roots.append(site_paths)
+    return roots
+
+
+# --- Project roots
+def set_project_roots(project_roots):
+    project_roots = _set_roots(project_roots, _PROJECT_ROOTS_CACHE)
+    pydev_log.debug("IDE_PROJECT_ROOTS %s\n" % project_roots)
+
+
+def _get_project_roots(project_roots_cache=_PROJECT_ROOTS_CACHE):
+    return _get_roots(project_roots_cache, 'IDE_PROJECT_ROOTS', set_project_roots)
+
+
+# --- Library roots
+def set_library_roots(roots):
+    roots = _set_roots(roots, _LIBRARY_ROOTS_CACHE)
+    pydev_log.debug("LIBRARY_ROOTS %s\n" % roots)
+
+    
+def _get_library_roots(library_roots_cache=_LIBRARY_ROOTS_CACHE):
+    return _get_roots(library_roots_cache, 'LIBRARY_ROOTS', set_library_roots, _get_default_library_roots)
 
 
 def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_CACHE):
@@ -172,26 +247,31 @@ def in_project_roots(filename, filename_to_in_scope_cache=_FILENAME_TO_IN_SCOPE_
     except:
         project_roots = _get_project_roots()
         original_filename = filename
-        if not os.path.isabs(filename) and not filename.startswith('<'):
-            filename = os.path.abspath(filename)
-        filename = os.path.normcase(filename)
+        if not filename.startswith('<'):
+            filename = _normpath(filename)
+        
+        found_in_project = []
         for root in project_roots:
             if root and filename.startswith(root):
-                filename_to_in_scope_cache[original_filename] = True
-                break
-        else: # for else (only called if the break wasn't reached).
-            filename_to_in_scope_cache[original_filename] = False
+                found_in_project.append(root)
 
-        if filename_to_in_scope_cache[original_filename]:
-            # additional check if interpreter is situated in a project directory
+        found_in_library = []
         library_roots = _get_library_roots()
         for root in library_roots:
             if root and filename.startswith(root):
-                    filename_to_in_scope_cache[original_filename] = False
-                    break
+                found_in_library.append(root)
         
-        # at this point it must be loaded.
-        return filename_to_in_scope_cache[original_filename]
+        in_project = False
+        if found_in_project:
+            if not found_in_library:
+                in_project = True
+            else:
+                # Found in both, let's see which one has the bigger path matched.
+                if max(len(x) for x in found_in_project) > max(len(x) for x in found_in_library):
+                    in_project = True
+                    
+        filename_to_in_scope_cache[original_filename] = in_project
+        return in_project
 
 
 def is_filter_enabled():

ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -101,6 +101,8 @@ except:
 #=======================================================================================================================
 class ReaderThread(threading.Thread):
 
+    TIMEOUT = 15
+    
     def __init__(self, sock):
         threading.Thread.__init__(self)
         try:
@@ -114,11 +116,14 @@ class ReaderThread(threading.Thread):
         self.all_received = []
         self._kill = False
         
+    def set_timeout(self, timeout):
+        self.TIMEOUT = timeout
+
     def get_next_message(self, context_messag):
         try:
-            msg = self._queue.get(block=True, timeout=15)
+            msg = self._queue.get(block=True, timeout=self.TIMEOUT)
         except:
-            raise AssertionError('No message was written in 15 seconds. Error message:\n%s' % (context_messag,))
+            raise AssertionError('No message was written in %s seconds. Error message:\n%s' % (self.TIMEOUT, context_messag,))
         else:
             frame = sys._getframe().f_back
             frame_info = ' --  File "%s", line %s, in %s\n' % (frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name)

ptvsd/_vendored/pydevd/tests_python/test_debugger.py

@@ -1641,6 +1641,7 @@ class WriterThreadCaseScapy(debugger_unittest.AbstractWriterThread):
     
     def run(self):
         self.start_socket()
+        self.reader_thread.set_timeout(30)  # Starting scapy may be slow (timed out with 15 seconds on appveyor).
         self.write_add_breakpoint(2, None)
         self.write_make_initial_run()
         

ptvsd/_vendored/pydevd/tests_python/test_in_project_roots.py

@@ -0,0 +1,33 @@
+def test_in_project_roots(tmpdir):
+    from _pydevd_bundle import pydevd_utils
+    import os.path
+    assert pydevd_utils._get_library_roots() == [
+        os.path.normcase(x) for x in pydevd_utils._get_default_library_roots()]
+    
+    site_packages = tmpdir.mkdir('site-packages')
+    project_dir = tmpdir.mkdir('project')
+    
+    project_dir_inside_site_packages = str(site_packages.mkdir('project'))
+    site_packages_inside_project_dir = str(project_dir.mkdir('site-packages'))
+    
+    # Convert from pytest paths to str.
+    site_packages = str(site_packages)
+    project_dir = str(project_dir)
+    tmpdir = str(tmpdir)
+    
+    # Test permutations of project dir inside site packages and vice-versa.
+    pydevd_utils.set_project_roots([project_dir, project_dir_inside_site_packages])
+    pydevd_utils.set_library_roots([site_packages, site_packages_inside_project_dir])
+
+    check = [
+        (tmpdir, False),
+        (site_packages, False),
+        (site_packages_inside_project_dir, False),
+        (project_dir, True),
+        (project_dir_inside_site_packages, True),
+    ]
+    for (check_path, find) in check[:]:
+        check.append((os.path.join(check_path, 'a.py'), find))
+        
+    for check_path, find in check:
+        assert pydevd_utils.in_project_roots(check_path) == find