Enable pydevd to be used in DAP mode directly. WIP: #532

src/debugpy/_vendored/force_pydevd.py

@@ -60,3 +60,14 @@ pydevd.install_breakpointhook(debugpy_breakpointhook)
 from _pydevd_bundle import pydevd_constants
 from _pydevd_bundle import pydevd_defaults
 pydevd_defaults.PydevdCustomization.DEFAULT_PROTOCOL = pydevd_constants.HTTP_JSON_PROTOCOL
+
+# Enable some defaults related to debugpy such as sending a single notification when
+# threads pause and stopping on any exception.
+pydevd_defaults.PydevdCustomization.DEBUG_MODE = 'debugpy-dap'
+
+# This is important when pydevd attaches automatically to a subprocess. In this case, we have to
+# make sure that debugpy is properly put back in the game for users to be able to use it.
+pydevd_defaults.PydevdCustomization.PREIMPORT = '%r;%s' % (
+    os.path.dirname(os.path.dirname(debugpy.__file__)), 
+    'debugpy._vendored.force_pydevd'
+)

src/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py

@@ -7,7 +7,7 @@ from _pydevd_bundle.pydevd_constants import get_global_debugger, IS_WINDOWS, IS_
     sorted_dict_repr, set_global_debugger, DebugInfoHolder
 from _pydev_bundle import pydev_log
 from contextlib import contextmanager
-from _pydevd_bundle import pydevd_constants
+from _pydevd_bundle import pydevd_constants, pydevd_defaults
 from _pydevd_bundle.pydevd_defaults import PydevdCustomization
 import ast
 
@@ -69,6 +69,14 @@ def _get_setup_updated_with_protocol_and_ppid(setup, is_exec=False):
     else:
         pydev_log.debug('Unexpected protocol: %s', protocol)
 
+    mode = pydevd_defaults.PydevdCustomization.DEBUG_MODE
+    if mode:
+        setup['debug-mode'] = mode
+
+    preimport = pydevd_defaults.PydevdCustomization.PREIMPORT
+    if preimport:
+        setup['preimport'] = preimport
+
     if DebugInfoHolder.PYDEVD_DEBUG_FILE:
         setup['log-file'] = DebugInfoHolder.PYDEVD_DEBUG_FILE
 

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_command_line_handling.py

@@ -68,6 +68,8 @@ ACCEPTED_ARG_HANDLERS = [
     ArgHandlerWithParam('client'),
     ArgHandlerWithParam('access-token'),
     ArgHandlerWithParam('client-access-token'),
+    ArgHandlerWithParam('debug-mode'),
+    ArgHandlerWithParam('preimport'),
 
     # Logging
     ArgHandlerWithParam('log-file'),

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_defaults.py

@@ -1,8 +1,60 @@
 '''
 This module holds the customization settings for the debugger.
 '''
+
 from _pydevd_bundle.pydevd_constants import QUOTED_LINE_PROTOCOL
+from _pydev_bundle import pydev_log
+import sys
 
 
 class PydevdCustomization(object):
-    DEFAULT_PROTOCOL = QUOTED_LINE_PROTOCOL
+    DEFAULT_PROTOCOL: str = QUOTED_LINE_PROTOCOL
+
+    # Debug mode may be set to 'debugpy-dap'.
+    #
+    # In 'debugpy-dap' mode the following settings are done to PyDB:
+    #
+    # py_db.skip_suspend_on_breakpoint_exception = (BaseException,)
+    # py_db.skip_print_breakpoint_exception = (NameError,)
+    # py_db.multi_threads_single_notification = True
+    DEBUG_MODE: str = ''
+
+    # This may be a <sys_path_entry>;<module_name> to be pre-imported
+    # Something as: 'c:/temp/foo;my_module.bar'
+    #
+    # What's done in this case is something as:
+    #
+    # sys.path.insert(0, <sys_path_entry>)
+    # try:
+    #     import <module_name>
+    # finally:
+    #     del sys.path[0]
+    #
+    # If the pre-import fails an output message is
+    # sent (but apart from that debugger execution
+    # should continue).
+    PREIMPORT: str = ''
+
+
+def on_pydb_init(py_db):
+    if PydevdCustomization.DEBUG_MODE == 'debugpy-dap':
+        py_db.skip_suspend_on_breakpoint_exception = (BaseException,)
+        py_db.skip_print_breakpoint_exception = (NameError,)
+        py_db.multi_threads_single_notification = True
+
+    if PydevdCustomization.PREIMPORT:
+        try:
+            sys_path_entry, module_name = PydevdCustomization.PREIMPORT.rsplit(';', maxsplit=1)
+        except Exception:
+            pydev_log.exception("Expected ';' in %s" % (PydevdCustomization.PREIMPORT,))
+        else:
+            try:
+                sys.path.insert(0, sys_path_entry)
+                try:
+                    __import__(module_name)
+                finally:
+                    sys.path.remove(sys_path_entry)
+            except Exception:
+                pydev_log.exception(
+                    "Error importing %s (with sys.path entry: %s)" % (module_name, sys_path_entry))
+

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_json_debug_options.py

@@ -14,6 +14,7 @@ class DebugOptions(object):
         'stop_on_entry',
         'max_exception_stack_frames',
         'gui_event_loop',
+        'client_os',
     ]
 
     def __init__(self):
@@ -26,6 +27,7 @@ class DebugOptions(object):
         self.stop_on_entry = False
         self.max_exception_stack_frames = 0
         self.gui_event_loop = 'matplotlib'
+        self.client_os = None
 
     def to_json(self):
         dct = {}
@@ -55,6 +57,9 @@ class DebugOptions(object):
         if 'STOP_ON_ENTRY' in debug_options:
             self.stop_on_entry = debug_options.get('STOP_ON_ENTRY')
 
+        if 'CLIENT_OS_TYPE' in debug_options:
+            self.client_os = debug_options.get('CLIENT_OS_TYPE')
+
         # Note: _max_exception_stack_frames cannot be set by debug options.
 
     def update_from_args(self, args):
@@ -91,6 +96,9 @@ class DebugOptions(object):
         if 'guiEventLoop' in args:
             self.gui_event_loop = str(args['guiEventLoop'])
 
+        if 'clientOS' in args:
+            self.client_os = str(args['clientOS']).upper()
+
 
 def int_parser(s, default_value=0):
     try:

src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py

@@ -17,7 +17,7 @@ from _pydevd_bundle._debug_adapter.pydevd_schema import (
     VariablesResponseBody, SetBreakpointsResponseBody, Response,
     Capabilities, PydevdAuthorizeRequest, Request,
     StepInTargetsResponseBody, SetFunctionBreakpointsResponseBody, BreakpointEvent,
-    BreakpointEventBody)
+    BreakpointEventBody, InitializedEvent)
 from _pydevd_bundle.pydevd_api import PyDevdAPI
 from _pydevd_bundle.pydevd_breakpoints import get_exception_class, FunctionBreakpoint
 from _pydevd_bundle.pydevd_comm_constants import (
@@ -380,6 +380,9 @@ class PyDevJsonCommandProcessor(object):
 
         self.api.set_use_libraries_filter(py_db, self._options.just_my_code)
 
+        if self._options.client_os:
+            self.api.set_ide_os(self._options.client_os)
+
         path_mappings = []
         for pathMapping in args.get('pathMappings', []):
             localRoot = pathMapping.get('localRoot', '')
@@ -496,6 +499,9 @@ class PyDevJsonCommandProcessor(object):
         self.api.set_enable_thread_notifications(py_db, True)
         self._set_debug_options(py_db, request.arguments.kwargs, start_reason=start_reason)
         response = pydevd_base_schema.build_response(request)
+
+        initialized_event = InitializedEvent()
+        py_db.writer.add_command(NetCommand(CMD_RETURN, 0, initialized_event, is_json=True))
         return NetCommand(CMD_RETURN, 0, response, is_json=True)
 
     def on_launch_request(self, py_db, request):

src/debugpy/_vendored/pydevd/pydevd.py

@@ -41,7 +41,7 @@ from _pydev_bundle.pydev_override import overrides
 from _pydev_bundle._pydev_saved_modules import threading, time, thread
 from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils
 from _pydevd_bundle.pydevd_filtering import FilesFiltering, glob_matches_path
-from _pydevd_bundle import pydevd_io, pydevd_vm_type
+from _pydevd_bundle import pydevd_io, pydevd_vm_type, pydevd_defaults
 from _pydevd_bundle import pydevd_utils
 from _pydevd_bundle import pydevd_runpy
 from _pydev_bundle.pydev_console_utils import DebugConsoleStdIn
@@ -715,6 +715,7 @@ class PyDB(object):
             # Set as the global instance only after it's initialized.
             set_global_debugger(self)
 
+        pydevd_defaults.on_pydb_init(self)
         # Stop the tracing as the last thing before the actual shutdown for a clean exit.
         atexit.register(stoptrace)
 
@@ -3279,6 +3280,14 @@ def main():
         pydev_log.exception()
         usage(1)
 
+    preimport = setup.get('preimport')
+    if preimport:
+        pydevd_defaults.PydevdCustomization.PREIMPORT = preimport
+
+    debug_mode = setup.get('debug-mode')
+    if debug_mode:
+        pydevd_defaults.PydevdCustomization.DEBUG_MODE = debug_mode
+
     log_trace_level = setup.get('log-level')
 
     # Note: the logging info could've been changed (this would happen if this is a

src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -1367,7 +1367,10 @@ class AbstractWriterThread(threading.Thread):
                 else:
                     return last
             if prev != last:
-                print('Ignored message: %r' % (last,))
+                sys.stderr.write('Ignored message: %r\n' % (last,))
+                # Uncomment to know where in the stack it was ignored.
+                # import traceback
+                # traceback.print_stack(limit=7)
 
             prev = last
 

src/debugpy/_vendored/pydevd/tests_python/resources/_debugger_case_wait_for_attach_debugpy_mode.py

@@ -0,0 +1,45 @@
+import os
+import sys
+port = int(sys.argv[1])
+root_dirname = os.path.dirname(os.path.dirname(__file__))
+
+if root_dirname not in sys.path:
+    sys.path.append(root_dirname)
+
+import pydevd
+
+# Ensure that pydevd uses JSON protocol
+from _pydevd_bundle import pydevd_constants
+from _pydevd_bundle import pydevd_defaults
+pydevd_defaults.PydevdCustomization.DEFAULT_PROTOCOL = pydevd_constants.HTTP_JSON_PROTOCOL
+
+# Enable some defaults related to debugpy such as sending a single notification when
+# threads pause and stopping on any exception.
+pydevd_defaults.PydevdCustomization.DEBUG_MODE = 'debugpy-dap'
+
+import tempfile
+with tempfile.TemporaryDirectory('w') as tempdir:
+    with open(os.path.join(tempdir, 'my_custom_module.py'), 'w') as stream:
+        stream.write("print('Loaded my_custom_module')")
+
+    pydevd_defaults.PydevdCustomization.PREIMPORT = '%s;my_custom_module' % (tempdir,)
+    assert 'my_custom_module' not in sys.modules
+
+    assert sys.gettrace() is None
+    print('enable attach to port: %s' % (port,))
+    pydevd._enable_attach(('127.0.0.1', port))
+
+    assert pydevd.get_global_debugger() is not None
+    # Set as a part of debugpy-dap
+    assert pydevd.get_global_debugger().multi_threads_single_notification
+    assert sys.gettrace() is not None
+
+    assert 'my_custom_module' in sys.modules
+
+    a = 10  # Break 1
+    print('wait for attach')
+    pydevd._wait_for_attach()
+
+    a = 20  # Break 2
+
+    print('TEST SUCEEDED!')

src/debugpy/_vendored/pydevd/tests_python/test_debugger_json.py

@@ -15,7 +15,7 @@ from _pydevd_bundle._debug_adapter.pydevd_schema import (ThreadEvent, ModuleEven
     ExceptionOptions, Response, StoppedEvent, ContinuedEvent, ProcessEvent, InitializeRequest,
     InitializeRequestArguments, TerminateArguments, TerminateRequest, TerminatedEvent,
     FunctionBreakpoint, SetFunctionBreakpointsRequest, SetFunctionBreakpointsArguments,
-    BreakpointEvent)
+    BreakpointEvent, InitializedEvent)
 from _pydevd_bundle.pydevd_comm_constants import file_system_encoding
 from _pydevd_bundle.pydevd_constants import (int_types, IS_64BIT_PROCESS,
     PY_VERSION_STR, PY_IMPL_VERSION_STR, PY_IMPL_NAME, IS_PY36_OR_GREATER,
@@ -3920,6 +3920,30 @@ cherrypy.quickstart(HelloWorld())
         writer.finished_ok = True
 
 
+def test_wait_for_attach_debugpy_mode(case_setup_remote_attach_to):
+    host_port = get_socket_name(close=True)
+
+    with case_setup_remote_attach_to.test_file('_debugger_case_wait_for_attach_debugpy_mode.py', host_port[1]) as writer:
+        time.sleep(1)  # Give some time for it to pass the first breakpoint and wait in 'wait_for_attach'.
+        writer.start_socket_client(*host_port)
+
+        # We don't send initial messages because everything should be pre-configured to
+        # the DAP mode already (i.e.: making sure it works).
+        json_facade = JsonFacade(writer, send_json_startup_messages=False)
+        break2_line = writer.get_line_index_with_content('Break 2')
+
+        json_facade.write_attach()
+        # Make sure we also received the initialized in the attach.
+        assert len(json_facade.mark_messages(InitializedEvent)) == 1
+
+        json_facade.write_set_breakpoints([break2_line])
+
+        json_facade.write_make_initial_run()
+        json_facade.wait_for_thread_stopped(line=break2_line)
+        json_facade.write_continue()
+        writer.finished_ok = True
+
+
 def test_wait_for_attach(case_setup_remote_attach_to):
     host_port = get_socket_name(close=True)
 
@@ -5411,6 +5435,7 @@ def test_debug_options(case_setup, val):
             stopOnEntry=val,
             maxExceptionStackFrames=4 if val else 5,
             guiEventLoop=gui_event_loop,
+            clientOS='UNIX' if val else 'WINDOWS'
         )
         json_facade.write_launch(**args)
 
@@ -5434,6 +5459,7 @@ def test_debug_options(case_setup, val):
             'stopOnEntry': 'stop_on_entry',
             'maxExceptionStackFrames': 'max_exception_stack_frames',
             'guiEventLoop': 'gui_event_loop',
+            'clientOS': 'client_os',
         }
 
         assert json.loads(output.body.output) == dict((translation[key], val) for key, val in args.items())

src/debugpy/adapter/clients.py

@@ -266,24 +266,6 @@ class Client(components.Component):
                 self._propagate_deferred_events()
                 return
 
-            if "clientOS" in request:
-                client_os = request("clientOS", json.enum("windows", "unix")).upper()
-            elif {"WindowsClient", "Windows"} & debug_options:
-                client_os = "WINDOWS"
-            elif {"UnixClient", "UNIX"} & debug_options:
-                client_os = "UNIX"
-            else:
-                client_os = "WINDOWS" if sys.platform == "win32" else "UNIX"
-            self.server.channel.request(
-                "setDebuggerProperty",
-                {
-                    "skipSuspendOnBreakpointException": ("BaseException",),
-                    "skipPrintBreakpointException": ("NameError",),
-                    "multiThreadsSingleNotification": True,
-                    "ideOS": client_os,
-                },
-            )
-
             # Let the client know that it can begin configuring the adapter.
             self.channel.send_event("initialized")
 

src/debugpy/adapter/servers.py

@@ -84,45 +84,6 @@ class Connection(object):
                 self.ppid = None
             self.channel.name = stream.name = str(self)
 
-            debugpy_dir = os.path.dirname(os.path.dirname(debugpy.__file__))
-            # Note: we must check if 'debugpy' is not already in sys.modules because the
-            # evaluation of an import at the wrong time could deadlock Python due to
-            # its import lock.
-            #
-            # So, in general this evaluation shouldn't do anything. It's only
-            # important when pydevd attaches automatically to a subprocess. In this
-            # case, we have to make sure that debugpy is properly put back in the game
-            # for users to be able to use it.v
-            #
-            # In this case (when the import is needed), this evaluation *must* be done
-            # before the configurationDone request is sent -- if this is not respected
-            # it's possible that pydevd already started secondary threads to handle
-            # commands, in which case it's very likely that this command would be
-            # evaluated at the wrong thread and the import could potentially deadlock
-            # the program.
-            #
-            # Note 2: the sys module is guaranteed to be in the frame globals and
-            # doesn't need to be imported.
-            inject_debugpy = """
-if 'debugpy' not in sys.modules:
-    sys.path.insert(0, {debugpy_dir!r})
-    try:
-        import debugpy
-    finally:
-        del sys.path[0]
-"""
-            inject_debugpy = inject_debugpy.format(debugpy_dir=debugpy_dir)
-
-            try:
-                self.channel.request("evaluate", {"expression": inject_debugpy})
-            except messaging.MessageHandlingError:
-                # Failure to inject is not a fatal error - such a subprocess can
-                # still be debugged, it just won't support "import debugpy" in user
-                # code - so don't terminate the session.
-                log.swallow_exception(
-                    "Failed to inject debugpy into {0}:", self, level="warning"
-                )
-
             with _lock:
                 # The server can disconnect concurrently before we get here, e.g. if
                 # it was force-killed. If the disconnect() handler has already run,

src/debugpy/public_api.py

@@ -91,7 +91,9 @@ def configure(__properties: dict[str, typing.Any] | None = None, **kwargs) -> No
 
 
 @_api()
-def listen(__endpoint: Endpoint | int) -> Endpoint:
+def listen(
+    __endpoint: Endpoint | int, *, in_process_debug_adapter: bool = False
+) -> Endpoint:
     """Starts a debug adapter debugging this process, that listens for
     incoming socket connections from clients on the specified address.
 
@@ -99,6 +101,13 @@ def listen(__endpoint: Endpoint | int) -> Endpoint:
     standard `socket` module for the `AF_INET` address family, or a port
     number. If only the port is specified, host is "127.0.0.1".
 
+    `in_process_debug_adapter`: by default a separate python process is
+    spawned and used to communicate with the client as the debug adapter.
+    By setting the value of `in_process_debug_adapter` to True a new 
+    python process is not spawned. Note: the con of setting 
+    `in_process_debug_adapter` to True is that subprocesses won't be 
+    automatically debugged.
+        
     Returns the interface and the port on which the debug adapter is
     actually listening, in the same format as `__endpoint`. This may be
     different from address if port was 0 in the latter, in which case

src/debugpy/server/api.py

@@ -146,10 +146,23 @@ def _starts_debugging(func):
 
 
 @_starts_debugging
-def listen(address, settrace_kwargs):
+def listen(address, settrace_kwargs, in_process_debug_adapter=False):
     # Errors below are logged with level="info", because the caller might be catching
     # and handling exceptions, and we don't want to spam their stderr unnecessarily.
 
+    if in_process_debug_adapter:
+        host, port = address
+        log.info("Listening: pydevd without debugpy adapter: {0}:{1}", host, port)
+        settrace_kwargs['patch_multiprocessing'] = False
+        _settrace(
+            host=host,
+            port=port,
+            wait_for_ready_to_run=False,
+            block_until_connected=False,
+            **settrace_kwargs
+        )
+        return
+
     import subprocess
 
     server_access_token = codecs.encode(os.urandom(32), "hex").decode("ascii")