Fix #966: Sub process debugging not working with PTVSD

Allow launching ptvsd as script rather than as a module.

Use script mode in multiproc implementation, and in tests, to avoid explicit PYTHONPATH manipulation.

.flake8

@@ -1,7 +1,8 @@
 [flake8]
 ignore = W,
     E24,E121,E123,E125,E126,E221,E226,E266,E704,
-    E265,E722,E501,E731,E306,E401,E302,E222,E303
+    E265,E722,E501,E731,E306,E401,E302,E222,E303,
+    E402
 exclude =
     ptvsd/_vendored/pydevd,
     ./.eggs,

ptvsd/__main__.py

@@ -2,10 +2,37 @@
 # Licensed under the MIT License. See LICENSE in the project root
 # for license information.
 
+from __future__ import print_function, with_statement, absolute_import
+
 import argparse
 import os.path
 import sys
 
+
+# ptvsd can also be invoked directly rather than via -m. In this case, the
+# first entry on sys.path is the one added automatically by Python for the
+# directory containing this file. This means that 1) import ptvsd will not
+# work, since we need the parent directory of ptvsd/ to be on path, rather
+# than ptvsd/ itself, and 2) many other absolute imports will break, because
+# they will be resolved relative to ptvsd/ - e.g. import socket will then
+# try to import ptvsd/socket.py!
+#
+# To fix this, we need to replace the automatically added entry such that it
+# points at the parent directory instead, import ptvsd from that directory,
+# and then remove than entry altogether so that it doesn't affect any further
+# imports. For example, suppose the user did:
+#
+#   python /foo/bar/ptvsd ...
+#
+# At the beginning of this script, sys.path will contain '/foo/bar/ptvsd' as
+# the first entry. What we want is to replace it with '/foo/bar', then import
+# ptvsd with that in effect, and then remove it before continuing execution.
+if __name__ == '__main__' and 'ptvsd' not in sys.modules:
+    sys.path[0] = os.path.dirname(sys.path[0])
+    import ptvsd # noqa
+    del sys.path[0]
+
+
 from ptvsd import multiproc, options
 from ptvsd._attach import attach_main
 from ptvsd._local import debug_main, run_main
@@ -13,6 +40,17 @@ from ptvsd.socket import Address
 from ptvsd.version import __version__, __author__  # noqa
 
 
+# When forming the command line involving __main__.py, it might be tempting to
+# import it as a module, and then use its __file__. However, that does not work
+# reliably, because __file__ can be a relative path - and when it is relative,
+# that's relative to the current directory at the time import was done, which
+# may be different from the current directory at the time the path is used.
+#
+# So, to be able to correctly locate the script at any point, we compute the
+# absolute path at import time.
+__file__ = os.path.abspath(__file__)
+
+
 ##################################
 # the script
 

ptvsd/multiproc.py

@@ -18,10 +18,10 @@ try:
 except ImportError:
     import Queue as queue
 
-from . import options
-from .socket import create_server, create_client
-from .messaging import JsonIOStream, JsonMessageChannel
-from ._util import new_hidden_thread, debug
+from ptvsd import options
+from ptvsd.socket import create_server, create_client
+from ptvsd.messaging import JsonIOStream, JsonMessageChannel
+from ptvsd._util import new_hidden_thread, debug
 
 from _pydev_bundle import pydev_monkey
 from _pydevd_bundle.pydevd_comm import get_global_debugger
@@ -168,9 +168,7 @@ def patch_args(args):
 
     the result should be:
 
-        python -R -Q warn -m ptvsd --host localhost --port 0 ... -m app
-
-    Note that the first -m above is interpreted by Python, and the second by ptvsd.
+        python -R -Q warn .../ptvsd/__main__.py --host localhost --port 0 ... -m app
     """
     if not options.multiprocess:
         return args
@@ -242,8 +240,9 @@ def patch_args(args):
 
     # Now we need to inject the ptvsd invocation right before the target. The target
     # itself can remain as is, because ptvsd is compatible with Python in that respect.
+    from ptvsd import __main__
     args[i:i] = [
-        '-m', 'ptvsd',
+        __main__.__file__,
         '--host', 'localhost',
         '--port', '0',
         '--wait',

pytests/func/test_multiproc.py

@@ -43,7 +43,7 @@ def test_multiprocessing(debug_session, pyfile):
             print('leaving child')
 
         if __name__ == '__main__':
-            import pytests.helpers.backchannel as backchannel
+            import backchannel
             if sys.version_info >= (3, 4):
                 multiprocessing.set_start_method('spawn')
             else:

pytests/func/test_run.py

@@ -20,7 +20,7 @@ def test_run(debug_session, pyfile, run_as):
     def code_to_debug():
         import os
         import sys
-        from pytests.helpers import backchannel
+        import backchannel
 
         print('begin')
         assert backchannel.read_json() == 'continue'

pytests/helpers/debuggee/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root
+# for license information.
+
+from __future__ import print_function, with_statement, absolute_import
+
+# This dummy package contains modules that are only supposed to be imported from
+# the code that is executed under debugger as part of the test (e.g. via @pyfile).
+# PYTHONPATH has an entry appended to it that allows these modules to be imported
+# directly from such code, i.e. "import backchannel". Consequently, these modules
+# should not assume that any other code from pytests/ is importable.
+
+
+# Ensure that __file__ is always absolute.
+import os
+__file__ = os.path.abspath(__file__)

pytests/helpers/session.py

@@ -14,17 +14,15 @@ import time
 import traceback
 
 import ptvsd
+import ptvsd.__main__
 from ptvsd.messaging import JsonIOStream, JsonMessageChannel, MessageHandlers
-from . import colors, print, watchdog
+
+from . import colors, debuggee, print, watchdog
 from .messaging import LoggingJsonStream
 from .pattern import ANY
 from .timeline import Timeline, Event, Response
 
 
-# ptvsd.__file__ will be <dir>/ptvsd/__main__.py - we want <dir>.
-PTVSD_SYS_PATH = os.path.dirname(os.path.dirname(ptvsd.__file__))
-
-
 class DebugSession(object):
     WAIT_FOR_EXIT_TIMEOUT = 5
     BACKCHANNEL_TIMEOUT = 15
@@ -41,7 +39,7 @@ class DebugSession(object):
         self.multiprocess_port_range = None
         self.debug_options = ['RedirectOutput']
         self.env = os.environ.copy()
-        self.env['PYTHONPATH'] = PTVSD_SYS_PATH
+        self.env['PYTHONPATH'] = os.path.dirname(debuggee.__file__)
         self.cwd = None
         self.expected_returncode = 0
         self.program_args = []
@@ -131,7 +129,7 @@ class DebugSession(object):
 
         argv = [sys.executable]
         if self.method != 'attach_pid':
-            argv += ['-m', 'ptvsd']
+            argv += [ptvsd.__main__.__file__]
 
         if self.method == 'attach_socket':
             argv += ['--wait']