From 608803cb99b450aedecc45167a7339b9b7b93b75 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Tue, 29 Oct 2019 10:35:12 -0300 Subject: [PATCH] Subprocesses should inherit PydevdCustomization. Fixes #1874 --- .../pydevd/_pydev_bundle/pydev_monkey.py | 40 +++++++-- .../pydevd_command_line_handling.py | 5 ++ .../pydevd/_pydevd_bundle/pydevd_constants.py | 4 + src/ptvsd/_vendored/pydevd/pydevd.py | 21 +++-- .../pydevd/tests_python/debugger_fixtures.py | 3 + .../pydevd/tests_python/debugger_unittest.py | 1 + .../_debugger_case_pydevd_customization.py | 51 +++++++++++ .../pydevd/tests_python/test_debugger_json.py | 90 +++++++++++++++++++ .../pydevd/tests_python/test_pydev_monkey.py | 13 ++- 9 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py diff --git a/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py b/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py index 638b84e5..1c1d28f0 100644 --- a/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py +++ b/src/ptvsd/_vendored/pydevd/_pydev_bundle/pydev_monkey.py @@ -36,14 +36,44 @@ def _get_apply_arg_patching(): return getattr(_arg_patch, 'apply_arg_patching', True) +def _get_setup_updated_with_protocol(setup): + if setup is None: + setup = {} + setup = setup.copy() + # Discard anything related to the protocol (we'll set the the protocol based on the one + # currently set). + setup.pop(pydevd_constants.ARGUMENT_HTTP_JSON_PROTOCOL, None) + setup.pop(pydevd_constants.ARGUMENT_JSON_PROTOCOL, None) + setup.pop(pydevd_constants.ARGUMENT_QUOTED_LINE_PROTOCOL, None) + setup.pop(pydevd_constants.ARGUMENT_HTTP_PROTOCOL, None) + + protocol = pydevd_constants.get_protocol() + if protocol == pydevd_constants.HTTP_JSON_PROTOCOL: + setup[pydevd_constants.ARGUMENT_HTTP_JSON_PROTOCOL] = True + + elif protocol == pydevd_constants.JSON_PROTOCOL: + setup[pydevd_constants.ARGUMENT_JSON_PROTOCOL] = True + + elif protocol == pydevd_constants.QUOTED_LINE_PROTOCOL: + setup[pydevd_constants.ARGUMENT_QUOTED_LINE_PROTOCOL] = True + + elif protocol == pydevd_constants.HTTP_PROTOCOL: + setup[pydevd_constants.ARGUMENT_HTTP_PROTOCOL] = True + + else: + pydev_log.debug('Unexpected protocol: %s', protocol) + return setup + + def _get_python_c_args(host, port, indC, args, setup): - host_literal = "'" + host + "'" if host is not None else 'None' - return ("import sys; sys.path.append(r'%s'); import pydevd; " - "pydevd.settrace(host=%s, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); " + setup = _get_setup_updated_with_protocol(setup) + return ("import sys; sys.path.append(r'%s'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL=%r;" + "pydevd.settrace(host=%r, port=%s, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); " "from pydevd import SetupHolder; SetupHolder.setup = %s; %s" ) % ( pydev_src_dir, - host_literal, + pydevd_constants.get_protocol(), + host, port, setup, args[indC + 1]) @@ -210,7 +240,7 @@ def patch_args(args): # ['X:\\pysrc\\pydevd.py', '--multiprocess', '--print-in-debugger-startup', # '--vm_type', 'python', '--client', '127.0.0.1', '--port', '56352', '--file', 'x:\\snippet1.py'] from _pydevd_bundle.pydevd_command_line_handling import setup_to_argv - original = setup_to_argv(SetupHolder.setup) + ['--file'] + original = setup_to_argv(_get_setup_updated_with_protocol(SetupHolder.setup)) + ['--file'] while i < len(args): if args[i] == '-m': # Always insert at pos == 1 (i.e.: pydevd "--module" --multiprocess ...) diff --git a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_command_line_handling.py b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_command_line_handling.py index f7f684da..682ad684 100644 --- a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_command_line_handling.py +++ b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_command_line_handling.py @@ -65,8 +65,13 @@ ACCEPTED_ARG_HANDLERS = [ ArgHandlerBool('print-in-debugger-startup'), ArgHandlerBool('cmd-line'), ArgHandlerBool('module'), + + # The ones below should've been just one setting to specify the protocol, but for compatibility + # reasons they're passed as a flag but are mutually exclusive. ArgHandlerBool('json-dap'), # Protocol used by ptvsd to communicate with pydevd (a single json message in each read) ArgHandlerBool('json-dap-http'), # Actual DAP (json messages over http protocol). + ArgHandlerBool('protocol-quoted-line'), # Custom protocol with quoted lines. + ArgHandlerBool('protocol-http'), # Custom protocol with http. ] ARGV_REP_TO_HANDLER = {} diff --git a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py index 3813b315..0ba6c737 100644 --- a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py +++ b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py @@ -586,19 +586,23 @@ def call_only_once(func): # Protocol where each line is a new message (text is quoted to prevent new lines). # payload is xml QUOTED_LINE_PROTOCOL = 'quoted-line' +ARGUMENT_QUOTED_LINE_PROTOCOL = 'protocol-quoted-line' # Uses http protocol to provide a new message. # i.e.: Content-Length:xxx\r\n\r\npayload # payload is xml HTTP_PROTOCOL = 'http' +ARGUMENT_HTTP_PROTOCOL = 'protocol-http' # Message is sent without any header. # payload is json JSON_PROTOCOL = 'json' +ARGUMENT_JSON_PROTOCOL = 'json-dap' # Same header as the HTTP_PROTOCOL # payload is json HTTP_JSON_PROTOCOL = 'http_json' +ARGUMENT_HTTP_JSON_PROTOCOL = 'json-dap-http' class _GlobalSettings: diff --git a/src/ptvsd/_vendored/pydevd/pydevd.py b/src/ptvsd/_vendored/pydevd/pydevd.py index 2c4b8c80..4025f333 100644 --- a/src/ptvsd/_vendored/pydevd/pydevd.py +++ b/src/ptvsd/_vendored/pydevd/pydevd.py @@ -25,7 +25,7 @@ from _pydev_bundle.pydev_override import overrides from _pydev_imps._pydev_saved_modules import thread from _pydev_imps._pydev_saved_modules import threading from _pydev_imps._pydev_saved_modules import time -from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils +from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils, pydevd_constants from _pydevd_bundle.pydevd_filtering import FilesFiltering from _pydevd_bundle import pydevd_io, pydevd_vm_type from _pydevd_bundle import pydevd_utils @@ -41,7 +41,7 @@ from _pydevd_bundle.pydevd_constants import (IS_JYTH_LESS25, get_thread_id, get_ clear_cached_thread_id, INTERACTIVE_MODE_AVAILABLE, SHOW_DEBUG_INFO_ENV, IS_PY34_OR_GREATER, IS_PY2, NULL, NO_FTRACE, IS_IRONPYTHON, JSON_PROTOCOL, IS_CPYTHON, HTTP_JSON_PROTOCOL, USE_CUSTOM_SYS_CURRENT_FRAMES_MAP, call_only_once, ForkSafeLock) -from _pydevd_bundle.pydevd_defaults import PydevdCustomization +from _pydevd_bundle.pydevd_defaults import PydevdCustomization # Note: import alias used on pydev_monkey. from _pydevd_bundle.pydevd_custom_frames import CustomFramesContainer, custom_frames_container_init from _pydevd_bundle.pydevd_dont_trace_files import DONT_TRACE, PYDEV_FILE, LIB_FILE from _pydevd_bundle.pydevd_extension_api import DebuggerEventHandler @@ -2910,6 +2910,7 @@ for handler in pydevd_extension_utils.extensions_of_type(DebuggerEventHandler): def main(): # parse the command line. --file is our last argument that is required + pydev_log.debug("Initial arguments: %s", (sys.argv,)) try: from _pydevd_bundle.pydevd_command_line_handling import process_command_line setup = process_command_line(sys.argv) @@ -2925,8 +2926,8 @@ def main(): pid = '' sys.stderr.write("pydev debugger: starting%s\n" % pid) - pydev_log.debug("Executing file %s" % setup['file']) - pydev_log.debug("arguments: %s" % str(sys.argv)) + pydev_log.debug("Executing file %s", setup['file']) + pydev_log.debug("arguments: %s", (sys.argv,)) pydevd_vm_type.setup_type(setup.get('vm_type', None)) @@ -2964,7 +2965,7 @@ def main(): dispatcher.connect(host, port) if dispatcher.port is not None: port = dispatcher.port - pydev_log.debug("Received port %d\n" % port) + pydev_log.debug("Received port %d\n", port) pydev_log.info("pydev debugger: process %d is connecting\n" % os.getpid()) try: @@ -3036,12 +3037,18 @@ def main(): is_module = setup['module'] patch_stdin() - if setup['json-dap']: + if setup[pydevd_constants.ARGUMENT_JSON_PROTOCOL]: PyDevdAPI().set_protocol(debugger, 0, JSON_PROTOCOL) - elif setup['json-dap-http']: + elif setup[pydevd_constants.ARGUMENT_HTTP_JSON_PROTOCOL]: PyDevdAPI().set_protocol(debugger, 0, HTTP_JSON_PROTOCOL) + elif setup[pydevd_constants.ARGUMENT_HTTP_PROTOCOL]: + PyDevdAPI().set_protocol(debugger, 0, pydevd_constants.HTTP_PROTOCOL) + + elif setup[pydevd_constants.ARGUMENT_QUOTED_LINE_PROTOCOL]: + PyDevdAPI().set_protocol(debugger, 0, pydevd_constants.QUOTED_LINE_PROTOCOL) + access_token = setup['access-token'] if access_token: debugger.authentication.access_token = access_token diff --git a/src/ptvsd/_vendored/pydevd/tests_python/debugger_fixtures.py b/src/ptvsd/_vendored/pydevd/tests_python/debugger_fixtures.py index ee6fdeb8..04b73d2f 100644 --- a/src/ptvsd/_vendored/pydevd/tests_python/debugger_fixtures.py +++ b/src/ptvsd/_vendored/pydevd/tests_python/debugger_fixtures.py @@ -324,6 +324,7 @@ def case_setup_remote(debugger_runner_remote): wait_for_port=True, access_token=None, ide_access_token=None, + append_command_line_args=(), **kwargs ): @@ -338,6 +339,8 @@ def case_setup_remote(debugger_runner_remote): if ide_access_token is not None: ret.append('--ide-access-token') ret.append(ide_access_token) + + ret.extend(append_command_line_args) return ret WriterThread.TEST_FILE = debugger_unittest._get_debugger_test_file(filename) diff --git a/src/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py b/src/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py index c62aa3f1..cec805b5 100644 --- a/src/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py +++ b/src/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py @@ -459,6 +459,7 @@ class DebuggerRunner(object): env['PYDEVD_DEBUG'] = 'True' env['PYDEVD_DEBUG_FILE'] = self.pydevd_debug_file + print('Logging to: %s' % (self.pydevd_debug_file,)) process = subprocess.Popen( args, stdout=subprocess.PIPE, diff --git a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py new file mode 100644 index 00000000..ccf604ba --- /dev/null +++ b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_pydevd_customization.py @@ -0,0 +1,51 @@ +import sys +import os + + +def main(): + env = os.environ.copy() + pythonpath = env.get('PYTHONPATH', '') + env['PYTHONPATH'] = os.path.dirname(__file__) + os.pathsep + \ + os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + + from _pydevd_bundle.pydevd_constants import HTTP_JSON_PROTOCOL + from _pydevd_bundle.pydevd_defaults import PydevdCustomization + PydevdCustomization.DEFAULT_PROTOCOL = HTTP_JSON_PROTOCOL + + import pydevd + from _pydev_bundle import pydev_log + pydev_log.debug('Argv received: %s', sys.argv) + port = int(sys.argv[1]) + print('before pydevd.settrace') + pydevd.settrace(port=port, patch_multiprocessing=True, suspend=True) + print('after pydevd.settrace') + + import subprocess + if '--use-c-switch' in sys.argv: + p = subprocess.Popen( + [sys.executable, '-u', '-c', 'import _debugger_case_pydevd_customization;_debugger_case_pydevd_customization.call()'], + stdout=subprocess.PIPE, + env=env, + ) + else: + p = subprocess.Popen( + [sys.executable, '-u', '_debugger_case_pydevd_customization.py', '--simple-call'], + cwd=os.path.dirname(__file__), + stdout=subprocess.PIPE, + env=env, + ) + + stdout, stderr = p.communicate() + assert b'called' in stdout, 'Did not find b"called" in: %s' % (stdout,) + print('TEST SUCEEDED!') # break 2 here + + +def call(): + print("called") # break 1 here + + +if __name__ == '__main__': + if '--simple-call' in sys.argv: + call() + else: + main() diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py index 6a19669c..3c5c1c4f 100644 --- a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py +++ b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py @@ -2961,6 +2961,96 @@ def test_attach_to_pid(case_setup_remote, reattach): writer.finished_ok = True +def test_remote_debugger_basic(case_setup_remote): + with case_setup_remote.test_file('_debugger_case_remote.py') as writer: + json_facade = JsonFacade(writer) + json_facade.write_launch() + json_facade.write_make_initial_run() + json_facade.wait_for_thread_stopped() + json_facade.write_continue() + + writer.finished_ok = True + + +@pytest.mark.parametrize('use_c_switch', [True, False]) +def test_subprocess_pydevd_customization(case_setup_remote, use_c_switch): + import threading + from tests_python.debugger_unittest import AbstractWriterThread + + with case_setup_remote.test_file( + '_debugger_case_pydevd_customization.py', + append_command_line_args=['--use-c-switch'] if use_c_switch else [] + ) as writer: + json_facade = JsonFacade(writer, send_json_startup_messages=False) + json_facade.writer.write_multi_threads_single_notification(True) + json_facade.write_launch() + + break1_line = writer.get_line_index_with_content('break 1 here') + break2_line = writer.get_line_index_with_content('break 2 here') + json_facade.write_set_breakpoints([break1_line, break2_line]) + + server_socket = writer.server_socket + + class SecondaryProcessWriterThread(AbstractWriterThread): + + TEST_FILE = writer.get_main_filename() + _sequence = -1 + + class SecondaryProcessThreadCommunication(threading.Thread): + + def run(self): + from tests_python.debugger_unittest import ReaderThread + expected_connections = 1 + if sys.platform != 'win32' and IS_PY2: + # Note: on linux on Python 2 CPython subprocess.call will actually + # create a fork first (at which point it'll connect) and then, later on it'll + # call the main (as if it was a clean process as if PyDB wasn't created + # the first time -- the debugger will still work, but it'll do an additional + # connection). + expected_connections = 2 + + for _ in range(expected_connections): + server_socket.listen(1) + self.server_socket = server_socket + writer.log.append(' *** Multiprocess waiting on server_socket.accept()') + new_sock, addr = server_socket.accept() + writer.log.append(' *** Multiprocess completed server_socket.accept()') + + reader_thread = ReaderThread(new_sock) + reader_thread.name = ' *** Multiprocess Reader Thread' + reader_thread.start() + writer.log.append(' *** Multiprocess started ReaderThread') + + writer2 = SecondaryProcessWriterThread() + writer2._WRITE_LOG_PREFIX = ' *** Multiprocess write: ' + writer2.reader_thread = reader_thread + writer2.sock = new_sock + json_facade2 = JsonFacade(writer2, send_json_startup_messages=False) + json_facade2.writer.write_multi_threads_single_notification(True) + + json_facade2.write_set_breakpoints([break1_line, break2_line]) + json_facade2.write_make_initial_run() + + json_facade2.wait_for_thread_stopped() + json_facade2.write_continue() + + secondary_process_thread_communication = SecondaryProcessThreadCommunication() + secondary_process_thread_communication.start() + time.sleep(.1) + + json_facade.write_make_initial_run() + json_facade.wait_for_thread_stopped() + + json_facade.write_continue() + json_facade.wait_for_thread_stopped() + json_facade.write_continue() + + secondary_process_thread_communication.join(5) + if secondary_process_thread_communication.is_alive(): + raise AssertionError('The SecondaryProcessThreadCommunication did not finish') + writer.finished_ok = True + + @pytest.mark.parametrize('apply_multiprocessing_patch', [True, False]) def test_no_subprocess_patching(case_setup_multiprocessing, apply_multiprocessing_patch): import threading diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_pydev_monkey.py b/src/ptvsd/_vendored/pydevd/tests_python/test_pydev_monkey.py index aa512244..fddd6bf0 100644 --- a/src/ptvsd/_vendored/pydevd/tests_python/test_pydev_monkey.py +++ b/src/ptvsd/_vendored/pydevd/tests_python/test_pydev_monkey.py @@ -17,12 +17,13 @@ class TestCase(unittest.TestCase): original = SetupHolder.setup try: - SetupHolder.setup = {'client': '127.0.0.1', 'port': '0'} + SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'protocol-quoted-line': True} check = '''C:\\bin\\python.exe -u -c connect(\\"127.0.0.1\\")''' debug_command = ( 'import sys; ' 'sys.path.append(r\'%s\'); ' - "import pydevd; pydevd.settrace(host='127.0.0.1', port=0, suspend=False, " + "import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line';" + "pydevd.settrace(host='127.0.0.1', port=0, suspend=False, " 'trace_only_current_thread=False, patch_multiprocessing=True); ' '' "from pydevd import SetupHolder; " @@ -46,10 +47,10 @@ class TestCase(unittest.TestCase): original = SetupHolder.setup try: - SetupHolder.setup = {'client': '127.0.0.1', 'port': '0'} + SetupHolder.setup = {'client': '127.0.0.1', 'port': '0', 'protocol-quoted-line': True} check = ['C:\\bin\\python.exe', '-u', '-c', 'connect("127.0.0.1")'] debug_command = ( - 'import sys; sys.path.append(r\'%s\'); import pydevd; ' + "import sys; sys.path.append(r\'%s\'); import pydevd; pydevd.PydevdCustomization.DEFAULT_PROTOCOL='quoted-line';" 'pydevd.settrace(host=\'127.0.0.1\', port=0, suspend=False, trace_only_current_thread=False, patch_multiprocessing=True); ' '' "from pydevd import SetupHolder; " @@ -85,6 +86,7 @@ class TestCase(unittest.TestCase): '--client', '127.0.0.1', '--multiprocess', + '--protocol-quoted-line', '--file', 'test', ]) @@ -105,6 +107,7 @@ class TestCase(unittest.TestCase): '0', '--client', '127.0.0.1', + '--protocol-quoted-line', '--file', '"connect(\\\\\\"127.0.0.1\\\\\\")"' if sys.platform == 'win32' else 'connect(\\"127.0.0.1\\")', '"with spaces"' if sys.platform == 'win32' else 'with spaces', @@ -138,6 +141,7 @@ class TestCase(unittest.TestCase): '0', '--client', '127.0.0.1', + '--protocol-quoted-line', '--file', 'target.py', '"connect(\\\\\\"127.0.0.1\\\\\\")"' if sys.platform == 'win32' else 'connect(\\"127.0.0.1\\")', @@ -162,6 +166,7 @@ class TestCase(unittest.TestCase): '0', '--client', '127.0.0.1', + '--protocol-quoted-line', '--file', 'target.py', '-c',