From 623c503b58fd8e28ed89d8be66fddd0f45868c0e Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Sat, 26 Jun 2021 14:34:24 -0300 Subject: [PATCH] Notify when (some) required stdlib imports are shadowed. Fixes #203 --- .../pydevd/_pydev_bundle/pydev_imports.py | 8 +- .../_pydev_imps/_pydev_saved_modules.py | 130 ++++++++++++++++-- src/debugpy/_vendored/pydevd/pydevconsole.py | 22 ++- src/debugpy/_vendored/pydevd/pydevd.py | 33 +++-- .../pydevd/tests_python/debugger_fixtures.py | 8 +- .../pydevd/tests_python/debugger_unittest.py | 8 +- .../pydevd/tests_python/test_debugger.py | 77 +++++++++++ 7 files changed, 237 insertions(+), 49 deletions(-) diff --git a/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_imports.py b/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_imports.py index 0e0f21f2..9fb17b94 100644 --- a/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_imports.py +++ b/src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_imports.py @@ -33,13 +33,7 @@ try: except NameError: from _pydev_imps._pydev_execfile import execfile -try: - if USE_LIB_COPY: - from _pydev_imps._pydev_saved_modules import _queue - else: - import Queue as _queue -except: - import queue as _queue # @UnresolvedImport +from _pydev_imps._pydev_saved_modules import _queue try: from _pydevd_bundle.pydevd_exec import Exec diff --git a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py index 6ff3939d..eb95eccf 100644 --- a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py +++ b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_saved_modules.py @@ -1,23 +1,125 @@ import sys +import os + IS_PY2 = sys.version_info < (3,) -import threading -import time +def find_in_pythonpath(module_name): + # Check all the occurrences where we could match the given module/package in the PYTHONPATH. + # + # This is a simplistic approach, but probably covers most of the cases we're interested in + # (i.e.: this may fail in more elaborate cases of import customization or .zip imports, but + # this should be rare in general). + found_at = [] -import socket + parts = module_name.split('.') # split because we need to convert mod.name to mod/name + for path in sys.path: + target = os.path.join(path, *parts) + target_py = target + '.py' + if os.path.isdir(target): + found_at.append(target) + if os.path.exists(target_py): + found_at.append(target_py) + return found_at -import select + +class DebuggerInitializationError(BaseException): + pass + + +class VerifyShadowedImport(object): + + def __init__(self, import_name): + self.import_name = import_name + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + if exc_type == DebuggerInitializationError: + return False # It's already an error we generated. + + # We couldn't even import it... + found_at = find_in_pythonpath(self.import_name) + + if len(found_at) <= 1: + # It wasn't found anywhere or there was just 1 occurrence. + # Let's just return to show the original error. + return False + + # We found more than 1 occurrence of the same module in the PYTHONPATH + # (the user module and the standard library module). + # Let's notify the user as it seems that the module was shadowed. + msg = self._generate_shadowed_import_message(found_at) + raise DebuggerInitializationError(msg) + + def _generate_shadowed_import_message(self, found_at): + msg = '''It was not possible to initialize the debugger due to a module name conflict. + +i.e.: the module "%(import_name)s" could not be imported because it is shadowed by: +%(found_at)s +Please rename this file/folder so that the original module from the standard library can be imported.''' % { + 'import_name': self.import_name, 'found_at': found_at[0]} + + return msg + + def check(self, module, expected_attributes): + msg = '' + for expected_attribute in expected_attributes: + try: + getattr(module, expected_attribute) + except: + msg = self._generate_shadowed_import_message([module.__file__]) + break + + if msg: + raise DebuggerInitializationError(msg) + + +with VerifyShadowedImport('threading') as verify_shadowed: + import threading; verify_shadowed.check(threading, ['Thread', 'settrace', 'setprofile', 'Lock', 'RLock', 'current_thread']) + +with VerifyShadowedImport('time') as verify_shadowed: + import time; verify_shadowed.check(time, ['sleep', 'time', 'mktime']) + +with VerifyShadowedImport('socket') as verify_shadowed: + import socket; verify_shadowed.check(socket, ['socket', 'gethostname', 'getaddrinfo']) + +with VerifyShadowedImport('select') as verify_shadowed: + import select; verify_shadowed.check(select, ['select']) + +with VerifyShadowedImport('code') as verify_shadowed: + import code as _code; verify_shadowed.check(_code, ['compile_command', 'InteractiveInterpreter']) if IS_PY2: - import thread - import Queue as _queue - import xmlrpclib - import SimpleXMLRPCServer as _pydev_SimpleXMLRPCServer - import BaseHTTPServer + with VerifyShadowedImport('thread') as verify_shadowed: + import thread; verify_shadowed.check(thread, ['start_new_thread', 'start_new', 'allocate_lock']) + + with VerifyShadowedImport('Queue') as verify_shadowed: + import Queue as _queue; verify_shadowed.check(_queue, ['Queue', 'LifoQueue', 'Empty', 'Full', 'deque']) + + with VerifyShadowedImport('xmlrpclib') as verify_shadowed: + import xmlrpclib; verify_shadowed.check(xmlrpclib, ['ServerProxy', 'Marshaller', 'Server']) + + with VerifyShadowedImport('SimpleXMLRPCServer') as verify_shadowed: + import SimpleXMLRPCServer as _pydev_SimpleXMLRPCServer; verify_shadowed.check(_pydev_SimpleXMLRPCServer, ['SimpleXMLRPCServer']) + + with VerifyShadowedImport('BaseHTTPServer') as verify_shadowed: + import BaseHTTPServer; verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler']) else: - import _thread as thread - import queue as _queue - import xmlrpc.client as xmlrpclib - import xmlrpc.server as _pydev_SimpleXMLRPCServer - import http.server as BaseHTTPServer \ No newline at end of file + with VerifyShadowedImport('_thread') as verify_shadowed: + import _thread as thread; verify_shadowed.check(thread, ['start_new_thread', 'start_new', 'allocate_lock']) + + with VerifyShadowedImport('queue') as verify_shadowed: + import queue as _queue; verify_shadowed.check(_queue, ['Queue', 'LifoQueue', 'Empty', 'Full', 'deque']) + + with VerifyShadowedImport('xmlrpclib') as verify_shadowed: + import xmlrpc.client as xmlrpclib; verify_shadowed.check(xmlrpclib, ['ServerProxy', 'Marshaller', 'Server']) + + with VerifyShadowedImport('xmlrpc.server') as verify_shadowed: + import xmlrpc.server as _pydev_SimpleXMLRPCServer; verify_shadowed.check(_pydev_SimpleXMLRPCServer, ['SimpleXMLRPCServer']) + + with VerifyShadowedImport('http.server') as verify_shadowed: + import http.server as BaseHTTPServer; verify_shadowed.check(BaseHTTPServer, ['BaseHTTPRequestHandler']) + diff --git a/src/debugpy/_vendored/pydevd/pydevconsole.py b/src/debugpy/_vendored/pydevd/pydevconsole.py index 0bdebdf7..caccb480 100644 --- a/src/debugpy/_vendored/pydevd/pydevconsole.py +++ b/src/debugpy/_vendored/pydevd/pydevconsole.py @@ -1,8 +1,8 @@ ''' Entry point module to start the interactive console. ''' -from _pydev_imps._pydev_saved_modules import thread -from _pydevd_bundle.pydevd_constants import IS_JYTHON, dict_iter_items +from _pydev_imps._pydev_saved_modules import thread, _code +from _pydevd_bundle.pydevd_constants import IS_JYTHON start_new_thread = thread.start_new_thread try: @@ -10,8 +10,8 @@ try: except ImportError: from _pydevd_bundle.pydevconsole_code_for_ironpython import InteractiveConsole -from code import compile_command -from code import InteractiveInterpreter +compile_command = _code.compile_command +InteractiveInterpreter = _code.InteractiveInterpreter import os import sys @@ -22,16 +22,16 @@ from _pydevd_bundle.pydevd_constants import INTERACTIVE_MODE_AVAILABLE, dict_key import traceback from _pydev_bundle import pydev_log -from _pydevd_bundle import pydevd_vars, pydevd_save_locals +from _pydevd_bundle import pydevd_save_locals from _pydev_bundle.pydev_imports import Exec, _queue -try: +if sys.version_info[0] >= 3: + import builtins as __builtin__ +else: import __builtin__ -except: - import builtins as __builtin__ # @UnresolvedImport -from _pydev_bundle.pydev_console_utils import BaseInterpreterInterface, BaseStdIn +from _pydev_bundle.pydev_console_utils import BaseInterpreterInterface, BaseStdIn # @UnusedImport from _pydev_bundle.pydev_console_utils import CodeFragment IS_PYTHON_3_ONWARDS = sys.version_info[0] >= 3 @@ -81,10 +81,8 @@ except: # Pull in runfile, the interface to UMD that wraps execfile from _pydev_bundle.pydev_umd import runfile, _set_globals_function if sys.version_info[0] >= 3: - import builtins # @UnresolvedImport - builtins.runfile = runfile + __builtin__.runfile = runfile else: - import __builtin__ __builtin__.runfile = runfile diff --git a/src/debugpy/_vendored/pydevd/pydevd.py b/src/debugpy/_vendored/pydevd/pydevd.py index d65be5e6..adb96dc4 100644 --- a/src/debugpy/_vendored/pydevd/pydevd.py +++ b/src/debugpy/_vendored/pydevd/pydevd.py @@ -6,33 +6,40 @@ This module starts the debugger. import sys # @NoMove if sys.version_info[:2] < (2, 6): raise RuntimeError('The PyDev.Debugger requires Python 2.6 onwards to be run. If you need to use an older Python version, use an older version of the debugger.') +import os + +try: + # Just empty packages to check if they're in the PYTHONPATH. + import _pydev_imps + import _pydev_bundle +except ImportError: + # On the first import of a pydevd module, add pydevd itself to the PYTHONPATH + # if its dependencies cannot be imported. + sys.path.append(os.path.dirname(os.path.abspath(__file__))) + import _pydev_imps + import _pydev_bundle + +# Import this first as it'll check for shadowed modules and will make sure that we import +# things as needed for gevent. +from _pydevd_bundle import pydevd_constants import atexit from collections import defaultdict from contextlib import contextmanager from functools import partial import itertools -import os import traceback import weakref import getpass as getpass_mod import functools -try: - import pydevd_file_utils -except ImportError: - # On the first import of a pydevd module, add pydevd itself to the PYTHONPATH - # if its dependencies cannot be imported. - sys.path.append(os.path.dirname(os.path.abspath(__file__))) - import pydevd_file_utils +import pydevd_file_utils from _pydev_bundle import pydev_imports, pydev_log from _pydev_bundle._pydev_filesystem_encoding import getfilesystemencoding from _pydev_bundle.pydev_is_thread_alive import is_thread_alive from _pydev_bundle.pydev_override import overrides -from _pydev_imps._pydev_saved_modules import thread -from _pydev_imps._pydev_saved_modules import threading -from _pydev_imps._pydev_saved_modules import time -from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils, pydevd_constants +from _pydev_imps._pydev_saved_modules import threading, time, thread +from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils from _pydevd_bundle.pydevd_filtering import FilesFiltering, glob_matches_path from _pydevd_bundle import pydevd_io, pydevd_vm_type from _pydevd_bundle import pydevd_utils @@ -2795,7 +2802,7 @@ def _locked_settrace( py_db.wait_for_ready_to_run() py_db.start_auxiliary_daemon_threads() - + try: if INTERACTIVE_MODE_AVAILABLE: py_db.init_matplotlib_support() diff --git a/src/debugpy/_vendored/pydevd/tests_python/debugger_fixtures.py b/src/debugpy/_vendored/pydevd/tests_python/debugger_fixtures.py index 9663f0c2..5e21a620 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/debugger_fixtures.py +++ b/src/debugpy/_vendored/pydevd/tests_python/debugger_fixtures.py @@ -256,6 +256,8 @@ def case_setup(tmpdir, debugger_runner_simple): def test_file( self, filename, + wait_for_port=True, + wait_for_initialization=True, **kwargs ): import shutil @@ -282,7 +284,11 @@ def case_setup(tmpdir, debugger_runner_simple): assert hasattr(WriterThread, key) setattr(WriterThread, key, value) - with runner.check_case(WriterThread) as writer: + with runner.check_case( + WriterThread, + wait_for_port=wait_for_port, + wait_for_initialization=wait_for_initialization + ) as writer: yield writer return CaseSetup() diff --git a/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py b/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py index d433b058..0a9967c0 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py +++ b/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py @@ -430,7 +430,7 @@ class DebuggerRunner(object): return args + ret @contextmanager - def check_case(self, writer_class, wait_for_port=True): + def check_case(self, writer_class, wait_for_port=True, wait_for_initialization=True): try: if callable(writer_class): writer = writer_class() @@ -451,7 +451,11 @@ class DebuggerRunner(object): with self.run_process(args, writer) as dct_with_stdout_stder: try: - if wait_for_port: + if not wait_for_initialization: + # The use-case for this is that the debugger can't even start-up in this + # scenario, as such, sleep a bit so that the output can be collected. + time.sleep(1) + elif wait_for_port: wait_for_condition(lambda: writer.finished_initialization) except TimeoutError: sys.stderr.write('Timed out waiting for initialization\n') diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py index b14eb1f8..af4aea4b 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py @@ -4281,6 +4281,83 @@ def test_frame_eval_mode_corner_case_many(case_setup, break_name): writer.finished_ok = True + +if IS_PY3K: + check_shadowed = [ + ( + u''' +if __name__ == '__main__': + import queue + print(queue) +''', + 'queue.py', + u'shadowed = True\n' + ), + + ( + u''' +if __name__ == '__main__': + import queue + print(queue) +''', + 'queue.py', + u'raise AssertionError("error on import")' + ) + ] + +else: + check_shadowed = [ + ( + u''' +if __name__ == '__main__': + import Queue + print(Queue) +''', + 'Queue.py', + u'shadowed = True\n' + ), + + ( + u''' +if __name__ == '__main__': + import Queue + print(Queue) +''', + 'Queue.py', + u'raise AssertionError("error on import")' + ) + ] + + +@pytest.mark.parametrize('module_name_and_content', check_shadowed) +def test_debugger_shadowed_imports(case_setup, tmpdir, module_name_and_content): + main_content, module_name, content = module_name_and_content + target = tmpdir.join('main.py') + shadowed = tmpdir.join(module_name) + + target.write_text(main_content, encoding='utf-8') + + shadowed.write_text(content, encoding='utf-8') + + def get_environ(writer): + env = os.environ.copy() + env.update({ + 'PYTHONPATH': str(tmpdir), + }) + return env + + try: + with case_setup.test_file( + str(target), + get_environ=get_environ, + wait_for_initialization=False, + ) as writer: + writer.write_make_initial_run() + except AssertionError: + pass # This is expected as pydevd didn't start-up. + + assert ('the module "%s" could not be imported because it is shadowed by:' % (module_name.split('.')[0])) in writer.get_stderr() + # Jython needs some vars to be set locally. # set JAVA_HOME=c:\bin\jdk1.8.0_172 # set PATH=%PATH%;C:\bin\jython2.7.0\bin