Notify when (some) required stdlib imports are shadowed. Fixes #203

src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_imports.py

@@ -33,13 +33,7 @@ try:
 except NameError:
     from _pydev_imps._pydev_execfile import execfile
 
-try:
-    if USE_LIB_COPY:
-        from _pydev_imps._pydev_saved_modules import _queue
-    else:
-        import Queue as _queue
-except:
-    import queue as _queue  # @UnresolvedImport
+from _pydev_imps._pydev_saved_modules import _queue
 
 try:
     from _pydevd_bundle.pydevd_exec import Exec

src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py

@@ -1,23 +1,125 @@
 import sys
+import os
+
 IS_PY2 = sys.version_info < (3,)
 
-import threading
 
-import time
+def find_in_pythonpath(module_name):
+    # Check all the occurrences where we could match the given module/package in the PYTHONPATH.
+    #
+    # This is a simplistic approach, but probably covers most of the cases we're interested in
+    # (i.e.: this may fail in more elaborate cases of import customization or .zip imports, but
+    # this should be rare in general).
+    found_at = []
 
-import socket
+    parts = module_name.split('.')  # split because we need to convert mod.name to mod/name
+    for path in sys.path:
+        target = os.path.join(path, *parts)
+        target_py = target + '.py'
+        if os.path.isdir(target):
+            found_at.append(target)
+        if os.path.exists(target_py):
+            found_at.append(target_py)
+    return found_at
 
-import select
+
+class DebuggerInitializationError(BaseException):
+    pass
+
+
+class VerifyShadowedImport(object):
+
+    def __init__(self, import_name):
+        self.import_name = import_name
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None:
+            if exc_type == DebuggerInitializationError:
+                return False  # It's already an error we generated.
+
+            # We couldn't even import it...
+            found_at = find_in_pythonpath(self.import_name)
+
+            if len(found_at) <= 1:
+                # It wasn't found anywhere or there was just 1 occurrence.
+                # Let's just return to show the original error.
+                return False
+
+            # We found more than 1 occurrence of the same module in the PYTHONPATH
+            # (the user module and the standard library module).
+            # Let's notify the user as it seems that the module was shadowed.
+            msg = self._generate_shadowed_import_message(found_at)
+            raise DebuggerInitializationError(msg)
+
+    def _generate_shadowed_import_message(self, found_at):
+        msg = '''It was not possible to initialize the debugger due to a module name conflict.
+
+i.e.: the module "%(import_name)s" could not be imported because it is shadowed by:
+%(found_at)s
+Please rename this file/folder so that the original module from the standard library can be imported.''' % {
+    'import_name': self.import_name, 'found_at': found_at[0]}
+
+        return msg
+
+    def check(self, module, expected_attributes):
+        msg = ''
+        for expected_attribute in expected_attributes:
+            try:
+                getattr(module, expected_attribute)
+            except:
+                msg = self._generate_shadowed_import_message([module.__file__])
+                break
+
+        if msg:
+            raise DebuggerInitializationError(msg)
+
+
+with VerifyShadowedImport('threading') as verify_shadowed:
+    import threading;    verify_shadowed.check(threading, ['Thread', 'settrace', 'setprofile', 'Lock', 'RLock', 'current_thread'])
+
+with VerifyShadowedImport('time') as verify_shadowed:
+    import time;    verify_shadowed.check(time, ['sleep', 'time', 'mktime'])
+
+with VerifyShadowedImport('socket') as verify_shadowed:
+    import socket;    verify_shadowed.check(socket, ['socket', 'gethostname', 'getaddrinfo'])
+
+with VerifyShadowedImport('select') as verify_shadowed:
+    import select;    verify_shadowed.check(select, ['select'])
+
+with VerifyShadowedImport('code') as verify_shadowed:
+    import code as _code;    verify_shadowed.check(_code, ['compile_command', 'InteractiveInterpreter'])
 
 if IS_PY2:
-    import thread
-    import Queue as _queue
-    import xmlrpclib
-    import SimpleXMLRPCServer as _pydev_SimpleXMLRPCServer
-    import BaseHTTPServer
+    with VerifyShadowedImport('thread') as verify_shadowed:
+        import thread;    verify_shadowed.check(thread, ['start_new_thread', 'start_new', 'allocate_lock'])
+
+    with VerifyShadowedImport('Queue') as verify_shadowed:
+        import Queue as _queue;    verify_shadowed.check(_queue, ['Queue', 'LifoQueue', 'Empty', 'Full', 'deque'])
+
+    with VerifyShadowedImport('xmlrpclib') as verify_shadowed:
+        import xmlrpclib;    verify_shadowed.check(xmlrpclib, ['ServerProxy', 'Marshaller', 'Server'])
+
+    with VerifyShadowedImport('SimpleXMLRPCServer') as verify_shadowed:
+        import SimpleXMLRPCServer as _pydev_SimpleXMLRPCServer;    verify_shadowed.check(_pydev_SimpleXMLRPCServer, ['SimpleXMLRPCServer'])
+
+    with VerifyShadowedImport('BaseHTTPServer') as verify_shadowed:
+        import BaseHTTPServer;    verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler'])
 else:
-    import _thread as thread
-    import queue as _queue
-    import xmlrpc.client as xmlrpclib
-    import xmlrpc.server as _pydev_SimpleXMLRPCServer
-    import http.server as BaseHTTPServer
+    with VerifyShadowedImport('_thread') as verify_shadowed:
+        import _thread as thread;    verify_shadowed.check(thread, ['start_new_thread', 'start_new', 'allocate_lock'])
+
+    with VerifyShadowedImport('queue') as verify_shadowed:
+        import queue as _queue;    verify_shadowed.check(_queue, ['Queue', 'LifoQueue', 'Empty', 'Full', 'deque'])
+
+    with VerifyShadowedImport('xmlrpclib') as verify_shadowed:
+        import xmlrpc.client as xmlrpclib;    verify_shadowed.check(xmlrpclib, ['ServerProxy', 'Marshaller', 'Server'])
+
+    with VerifyShadowedImport('xmlrpc.server') as verify_shadowed:
+        import xmlrpc.server as _pydev_SimpleXMLRPCServer;    verify_shadowed.check(_pydev_SimpleXMLRPCServer, ['SimpleXMLRPCServer'])
+
+    with VerifyShadowedImport('http.server') as verify_shadowed:
+        import http.server as BaseHTTPServer;    verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler'])
+

src/debugpy/_vendored/pydevd/pydevconsole.py

@@ -1,8 +1,8 @@
 '''
 Entry point module to start the interactive console.
 '''
-from _pydev_imps._pydev_saved_modules import thread
-from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items
+from _pydev_imps._pydev_saved_modules import thread, _code
+from _pydevd_bundle.pydevd_constants import IS_JYTHON
 start_new_thread = thread.start_new_thread
 
 try:
@@ -10,8 +10,8 @@ try:
 except ImportError:
     from _pydevd_bundle.pydevconsole_code_for_ironpython import InteractiveConsole
 
-from code import compile_command
-from code import InteractiveInterpreter
+compile_command = _code.compile_command
+InteractiveInterpreter = _code.InteractiveInterpreter
 
 import os
 import sys
@@ -22,16 +22,16 @@ from _pydevd_bundle.pydevd_constants import INTERACTIVE_MODE_AVAILABLE, dict_key
 import traceback
 from _pydev_bundle import pydev_log
 
-from _pydevd_bundle import pydevd_vars, pydevd_save_locals
+from _pydevd_bundle import pydevd_save_locals
 
 from _pydev_bundle.pydev_imports import Exec, _queue
 
-try:
+if sys.version_info[0] >= 3:
+    import builtins as __builtin__
+else:
     import __builtin__
-except:
-    import builtins as __builtin__  # @UnresolvedImport
 
-from _pydev_bundle.pydev_console_utils import BaseInterpreterInterface, BaseStdIn
+from _pydev_bundle.pydev_console_utils import BaseInterpreterInterface, BaseStdIn  # @UnusedImport
 from _pydev_bundle.pydev_console_utils import CodeFragment
 
 IS_PYTHON_3_ONWARDS = sys.version_info[0] >= 3
@@ -81,10 +81,8 @@ except:
 # Pull in runfile, the interface to UMD that wraps execfile
 from _pydev_bundle.pydev_umd import runfile, _set_globals_function
 if sys.version_info[0] >= 3:
-    import builtins  # @UnresolvedImport
-    builtins.runfile = runfile
+    __builtin__.runfile = runfile
 else:
-    import __builtin__
     __builtin__.runfile = runfile
 
 

src/debugpy/_vendored/pydevd/pydevd.py

@@ -6,33 +6,40 @@ This module starts the debugger.
 import sys  # @NoMove
 if sys.version_info[:2] < (2, 6):
     raise RuntimeError('The PyDev.Debugger requires Python 2.6 onwards to be run. If you need to use an older Python version, use an older version of the debugger.')
+import os
+
+try:
+    # Just empty packages to check if they're in the PYTHONPATH.
+    import _pydev_imps
+    import _pydev_bundle
+except ImportError:
+    # On the first import of a pydevd module, add pydevd itself to the PYTHONPATH
+    # if its dependencies cannot be imported.
+    sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+    import _pydev_imps
+    import _pydev_bundle
+
+# Import this first as it'll check for shadowed modules and will make sure that we import
+# things as needed for gevent.
+from _pydevd_bundle import pydevd_constants
 
 import atexit
 from collections import defaultdict
 from contextlib import contextmanager
 from functools import partial
 import itertools
-import os
 import traceback
 import weakref
 import getpass as getpass_mod
 import functools
-try:
-    import pydevd_file_utils
-except ImportError:
-    # On the first import of a pydevd module, add pydevd itself to the PYTHONPATH
-    # if its dependencies cannot be imported.
-    sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-    import pydevd_file_utils
 
+import pydevd_file_utils
 from _pydev_bundle import pydev_imports, pydev_log
 from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding
 from _pydev_bundle.pydev_is_thread_alive import is_thread_alive
 from _pydev_bundle.pydev_override import overrides
-from _pydev_imps._pydev_saved_modules import thread
-from _pydev_imps._pydev_saved_modules import threading
-from _pydev_imps._pydev_saved_modules import time
-from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils, pydevd_constants
+from _pydev_imps._pydev_saved_modules import threading, time, thread
+from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils
 from _pydevd_bundle.pydevd_filtering import FilesFiltering, glob_matches_path
 from _pydevd_bundle import pydevd_io, pydevd_vm_type
 from _pydevd_bundle import pydevd_utils
@@ -2795,7 +2802,7 @@ def _locked_settrace(
 
         py_db.wait_for_ready_to_run()
         py_db.start_auxiliary_daemon_threads()
-        
+
         try:
             if INTERACTIVE_MODE_AVAILABLE:
                 py_db.init_matplotlib_support()

src/debugpy/_vendored/pydevd/tests_python/debugger_fixtures.py

@@ -256,6 +256,8 @@ def case_setup(tmpdir, debugger_runner_simple):
         def test_file(
                 self,
                 filename,
+                wait_for_port=True,
+                wait_for_initialization=True,
                 **kwargs
             ):
             import shutil
@@ -282,7 +284,11 @@ def case_setup(tmpdir, debugger_runner_simple):
                 assert hasattr(WriterThread, key)
                 setattr(WriterThread, key, value)
 
-            with runner.check_case(WriterThread) as writer:
+            with runner.check_case(
+                    WriterThread,
+                    wait_for_port=wait_for_port,
+                    wait_for_initialization=wait_for_initialization
+                ) as writer:
                 yield writer
 
     return CaseSetup()

src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -430,7 +430,7 @@ class DebuggerRunner(object):
         return args + ret
 
     @contextmanager
-    def check_case(self, writer_class, wait_for_port=True):
+    def check_case(self, writer_class, wait_for_port=True, wait_for_initialization=True):
         try:
             if callable(writer_class):
                 writer = writer_class()
@@ -451,7 +451,11 @@ class DebuggerRunner(object):
 
                 with self.run_process(args, writer) as dct_with_stdout_stder:
                     try:
-                        if wait_for_port:
+                        if not wait_for_initialization:
+                            # The use-case for this is that the debugger can't even start-up in this
+                            # scenario, as such, sleep a bit so that the output can be collected.
+                            time.sleep(1)
+                        elif wait_for_port:
                             wait_for_condition(lambda: writer.finished_initialization)
                     except TimeoutError:
                         sys.stderr.write('Timed out waiting for initialization\n')

src/debugpy/_vendored/pydevd/tests_python/test_debugger.py

@@ -4281,6 +4281,83 @@ def test_frame_eval_mode_corner_case_many(case_setup, break_name):
 
             writer.finished_ok = True
 
+
+if IS_PY3K:
+    check_shadowed = [
+    (
+        u'''
+if __name__ == '__main__':
+    import queue
+    print(queue)
+''',
+        'queue.py',
+        u'shadowed = True\n'
+    ),
+
+    (
+        u'''
+if __name__ == '__main__':
+    import queue
+    print(queue)
+''',
+        'queue.py',
+        u'raise AssertionError("error on import")'
+    )
+    ]
+
+else:
+    check_shadowed = [
+    (
+        u'''
+if __name__ == '__main__':
+    import Queue
+    print(Queue)
+''',
+        'Queue.py',
+        u'shadowed = True\n'
+    ),
+
+    (
+        u'''
+if __name__ == '__main__':
+    import Queue
+    print(Queue)
+''',
+        'Queue.py',
+        u'raise AssertionError("error on import")'
+    )
+    ]
+
+
+@pytest.mark.parametrize('module_name_and_content', check_shadowed)
+def test_debugger_shadowed_imports(case_setup, tmpdir, module_name_and_content):
+    main_content, module_name, content = module_name_and_content
+    target = tmpdir.join('main.py')
+    shadowed = tmpdir.join(module_name)
+
+    target.write_text(main_content, encoding='utf-8')
+
+    shadowed.write_text(content, encoding='utf-8')
+
+    def get_environ(writer):
+        env = os.environ.copy()
+        env.update({
+            'PYTHONPATH': str(tmpdir),
+        })
+        return env
+
+    try:
+        with case_setup.test_file(
+                str(target),
+                get_environ=get_environ,
+                wait_for_initialization=False,
+            ) as writer:
+            writer.write_make_initial_run()
+    except AssertionError:
+        pass  # This is expected as pydevd didn't start-up.
+
+    assert ('the module "%s" could not be imported because it is shadowed by:' % (module_name.split('.')[0])) in writer.get_stderr()
+
 # Jython needs some vars to be set locally.
 # set JAVA_HOME=c:\bin\jdk1.8.0_172
 # set PATH=%PATH%;C:\bin\jython2.7.0\bin