Fix #886: ptvsd command line should support the equivalent of python -c

Plus some test fixes.

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 ignore = W,
     E24,E121,E123,E125,E126,E221,E226,E266,E704,
-    E265,E722,E501,E731,E306,E401,E302,E222
+    E265,E722,E501,E731,E306,E401,E302,E222,E303
 exclude =
     ptvsd/_vendored/pydevd,
     ./.eggs,

ptvsd/__main__.py

@@ -6,7 +6,7 @@ import argparse
 import os.path
 import sys
 
-from ptvsd import multiproc
+from ptvsd import multiproc, options
 from ptvsd._attach import attach_main
 from ptvsd._local import debug_main, run_main
 from ptvsd.socket import Address
@@ -124,8 +124,8 @@ def _group_args(argv):
             supported.append(arg)
 
         # ptvsd support
-        elif arg in ('--host', '--server-host', '--port', '--pid', '-m', '--multiprocess-port-range'):
-            if arg in ('-m', '--pid'):
+        elif arg in ('--host', '--server-host', '--port', '--pid', '-m', '-c', '--multiprocess-port-range'):
+            if arg in ('-m', '-c', '--pid'):
                 gottarget = True
             supported.append(arg)
             if nextarg is not None:
@@ -169,6 +169,7 @@ def _parse_args(prog, argv):
 
     target = parser.add_mutually_exclusive_group(required=True)
     target.add_argument('-m', dest='module')
+    target.add_argument('-c', dest='code')
     target.add_argument('--pid', type=int)
     target.add_argument('filename', nargs='?')
 
@@ -205,12 +206,17 @@ def _parse_args(prog, argv):
     pid = ns.pop('pid')
     module = ns.pop('module')
     filename = ns.pop('filename')
+    code = ns.pop('code')
     if pid is not None:
         args.name = pid
         args.kind = 'pid'
     elif module is not None:
         args.name = module
         args.kind = 'module'
+    elif code is not None:
+        options.code = code
+        args.name = 'ptvsd.run_code'
+        args.kind = 'module'
     else:
         args.name = filename
         args.kind = 'script'

ptvsd/multiproc.py

@@ -55,6 +55,7 @@ def disable():
     except Exception:
         pass
 
+
 def _listener():
     counter = itertools.count(1)
     while listener_port:

ptvsd/options.py

@@ -0,0 +1,13 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root
+# for license information.
+
+from __future__ import print_function, with_statement, absolute_import
+
+
+"""ptvsd command-line options that need to be globally available.
+"""
+
+code = None
+"""When running with -c, specifies the code that needs to be run.
+"""

ptvsd/run_code.py

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See LICENSE in the project root
+# for license information.
+
+# This module is used to implement -c support, since pydevd doesn't support
+# it directly. So we tell it to run this module instead, and it just does
+# exec on the code.
+
+# It is crucial that this module does *not* do "from __future__ import ...",
+# because we want exec below to use the defaults as defined by the flags that
+# were passed to the Python interpreter when it was launched.
+
+if __name__ == '__main__':
+    from ptvsd.options import code
+    exec(code, {})

pytests/conftest.py

@@ -126,8 +126,8 @@ def pyfile(request, tmpdir):
 ])
 def debug_session(request):
     session = DebugSession(request.param)
-    yield session
     try:
+        yield session
         try:
             failed = request.node.call_result.failed
         except AttributeError:

pytests/func/test_run.py

@@ -5,13 +5,17 @@
 from __future__ import print_function, with_statement, absolute_import
 
 import os
+import pytest
+
 import ptvsd
 
-from ..helpers.pattern import ANY
-from ..helpers.timeline import Event
+from pytests.helpers import print
+from pytests.helpers.pattern import ANY
+from pytests.helpers.timeline import Event
 
 
-def test_run(debug_session, pyfile):
+@pytest.mark.parametrize('run_as', ['file', 'module', 'code'])
+def test_run(debug_session, pyfile, run_as):
     @pyfile
     def code_to_debug():
         import os
@@ -23,13 +27,24 @@ def test_run(debug_session, pyfile):
         backchannel.write_json(os.path.abspath(sys.modules['ptvsd'].__file__))
         print('end')
 
-    debug_session.prepare_to_run(filename=code_to_debug, backchannel=True)
+    if run_as == 'file':
+        debug_session.prepare_to_run(filename=code_to_debug, backchannel=True)
+    elif run_as == 'module':
+        debug_session.add_file_to_pythonpath(code_to_debug)
+        debug_session.prepare_to_run(module='code_to_debug', backchannel=True)
+    elif run_as == 'code':
+        with open(code_to_debug, 'r') as f:
+            code = f.read()
+        debug_session.prepare_to_run(code=code, backchannel=True)
+    else:
+        pytest.fail()
+
     debug_session.start_debugging()
     assert debug_session.timeline.is_frozen
 
     process_event, = debug_session.all_occurrences_of(Event('process'))
     assert process_event == Event('process', ANY.dict_with({
-        'name': code_to_debug,
+        'name': ANY if run_as == 'code' else code_to_debug,
     }))
 
     debug_session.write_json('continue')

pytests/helpers/colors.py

@@ -4,54 +4,89 @@
 
 from __future__ import print_function, with_statement, absolute_import
 
-from colorama import Fore
-from pygments import highlight, lexers, formatters, token
+import platform
 
 
-# Colors that are commented out don't work with PowerShell.
-RESET = Fore.RESET
-BLACK = Fore.BLACK
-BLUE = Fore.BLUE
-CYAN = Fore.CYAN
-GREEN = Fore.GREEN
-# MAGENTA = Fore.MAGENTA
-RED = Fore.RED
-WHITE = Fore.WHITE
-# YELLOW = Fore.YELLOW
-LIGHT_BLACK = Fore.LIGHTBLACK_EX
-LIGHT_BLUE = Fore.LIGHTBLUE_EX
-LIGHT_CYAN = Fore.LIGHTCYAN_EX
-LIGHT_GREEN = Fore.LIGHTGREEN_EX
-LIGHT_MAGENTA = Fore.LIGHTMAGENTA_EX
-LIGHT_RED = Fore.LIGHTRED_EX
-LIGHT_WHITE = Fore.LIGHTWHITE_EX
-LIGHT_YELLOW = Fore.LIGHTYELLOW_EX
+if platform.system() == 'Windows':
+    # pytest-timeout seems to be buggy wrt colorama when capturing output.
+    #
+    # TODO: re-enable after enabling proper ANSI sequence handling:
+    # https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
+
+    RESET = ''
+    BLACK = ''
+    BLUE = ''
+    CYAN = ''
+    GREEN = ''
+    RED = ''
+    WHITE = ''
+    LIGHT_BLACK = ''
+    LIGHT_BLUE = ''
+    LIGHT_CYAN = ''
+    LIGHT_GREEN = ''
+    LIGHT_MAGENTA = ''
+    LIGHT_RED = ''
+    LIGHT_WHITE = ''
+    LIGHT_YELLOW = ''
 
 
-color_scheme = {
-    token.Token: ('white', 'white'),
-    token.Punctuation: ('', ''),
-    token.Operator: ('', ''),
-    token.Literal: ('brown', 'brown'),
-    token.Keyword: ('brown', 'brown'),
-    token.Name: ('white', 'white'),
-    token.Name.Constant: ('brown', 'brown'),
-    token.Name.Attribute: ('brown', 'brown'),
-    # token.Name.Tag: ('white', 'white'),
-    # token.Name.Function: ('white', 'white'),
-    # token.Name.Variable: ('white', 'white'),
-}
-
-formatter = formatters.TerminalFormatter(colorscheme=color_scheme)
-json_lexer = lexers.JsonLexer()
-python_lexer = lexers.PythonLexer()
+    def colorize_json(s):
+        return s
 
 
-def colorize_json(s):
-    return highlight(s, json_lexer, formatter).rstrip()
+    def color_repr(obj):
+        return repr(obj)
 
 
-def color_repr(obj):
-    return highlight(repr(obj), python_lexer, formatter).rstrip()
+else:
+    from colorama import Fore
+    from pygments import highlight, lexers, formatters, token
+
+
+    # Colors that are commented out don't work with PowerShell.
+    RESET = Fore.RESET
+    BLACK = Fore.BLACK
+    BLUE = Fore.BLUE
+    CYAN = Fore.CYAN
+    GREEN = Fore.GREEN
+    # MAGENTA = Fore.MAGENTA
+    RED = Fore.RED
+    WHITE = Fore.WHITE
+    # YELLOW = Fore.YELLOW
+    LIGHT_BLACK = Fore.LIGHTBLACK_EX
+    LIGHT_BLUE = Fore.LIGHTBLUE_EX
+    LIGHT_CYAN = Fore.LIGHTCYAN_EX
+    LIGHT_GREEN = Fore.LIGHTGREEN_EX
+    LIGHT_MAGENTA = Fore.LIGHTMAGENTA_EX
+    LIGHT_RED = Fore.LIGHTRED_EX
+    LIGHT_WHITE = Fore.LIGHTWHITE_EX
+    LIGHT_YELLOW = Fore.LIGHTYELLOW_EX
+
+
+    color_scheme = {
+        token.Token: ('white', 'white'),
+        token.Punctuation: ('', ''),
+        token.Operator: ('', ''),
+        token.Literal: ('brown', 'brown'),
+        token.Keyword: ('brown', 'brown'),
+        token.Name: ('white', 'white'),
+        token.Name.Constant: ('brown', 'brown'),
+        token.Name.Attribute: ('brown', 'brown'),
+        # token.Name.Tag: ('white', 'white'),
+        # token.Name.Function: ('white', 'white'),
+        # token.Name.Variable: ('white', 'white'),
+    }
+
+    formatter = formatters.TerminalFormatter(colorscheme=color_scheme)
+    json_lexer = lexers.JsonLexer()
+    python_lexer = lexers.PythonLexer()
+
+
+    def colorize_json(s):
+        return highlight(s, json_lexer, formatter).rstrip()
+
+
+    def color_repr(obj):
+        return highlight(repr(obj), python_lexer, formatter).rstrip()
 
 

pytests/helpers/session.py

@@ -20,11 +20,11 @@ from .timeline import Timeline, Event, Response
 
 
 # ptvsd.__file__ will be <dir>/ptvsd/__main__.py - we want <dir>.
-PTVSD_SYS_PATH = os.path.basename(os.path.basename(ptvsd.__file__))
+PTVSD_SYS_PATH = os.path.dirname(os.path.dirname(ptvsd.__file__))
 
 
 class DebugSession(object):
-    WAIT_FOR_EXIT_TIMEOUT = 3
+    WAIT_FOR_EXIT_TIMEOUT = 5
     BACKCHANNEL_TIMEOUT = 10
 
     def __init__(self, method='launch', ptvsd_port=None):
@@ -37,6 +37,9 @@ class DebugSession(object):
         self.ptvsd_port = ptvsd_port or 5678
         self.multiprocess = False
         self.multiprocess_port_range = None
+        self.debug_options = ['RedirectOutput']
+        self.env = os.environ.copy()
+        self.env['PYTHONPATH'] = PTVSD_SYS_PATH
 
         self.is_running = False
         self.process = None
@@ -46,7 +49,7 @@ class DebugSession(object):
         self.backchannel_socket = None
         self.backchannel_port = None
         self.backchannel_established = threading.Event()
-        self.debug_options = ['RedirectOutput']
+        self._output_capture_threads = []
 
         self.timeline = Timeline(ignore_unobserved=[
             Event('output'),
@@ -66,6 +69,9 @@ class DebugSession(object):
         self.expect_realized = self.timeline.expect_realized
         self.all_occurrences_of = self.timeline.all_occurrences_of
 
+    def add_file_to_pythonpath(self, filename):
+        self.env['PYTHONPATH'] += os.pathsep + os.path.dirname(filename)
+
     def __contains__(self, expectation):
         return expectation in self.timeline
 
@@ -98,10 +104,12 @@ class DebugSession(object):
                 self.backchannel_socket.shutdown(socket.SHUT_RDWR)
             except:
                 self.backchannel_socket = None
+        self._wait_for_remaining_output()
 
-    def prepare_to_run(self, perform_handshake=True, filename=None, module=None, backchannel=False):
+    def prepare_to_run(self, perform_handshake=True, filename=None, module=None, code=None, backchannel=False):
         """Spawns ptvsd using the configured method, telling it to execute the
-        provided Python file or module, and establishes a message channel to it.
+        provided Python file, module, or code, and establishes a message channel
+        to it.
 
         If backchannel is True, calls self.setup_backchannel() before returning.
 
@@ -125,22 +133,24 @@ class DebugSession(object):
             argv += ['--multiprocess-port-range', '%d-%d' % self.multiprocess_port_range]
 
         if filename:
-            assert not module
+            assert not module and not code
             argv += [filename]
         elif module:
-            assert not filename
+            assert not filename and not code
             argv += ['-m', module]
-
-        env = os.environ.copy()
-        env.update({'PYTHONPATH': PTVSD_SYS_PATH})
+        elif code:
+            assert not filename and not module
+            argv += ['-c', code]
 
         if backchannel:
             self.setup_backchannel()
         if self.backchannel_port:
-            env['PTVSD_BACKCHANNEL_PORT'] = str(self.backchannel_port)
+            self.env['PTVSD_BACKCHANNEL_PORT'] = str(self.backchannel_port)
 
+        print('Current directory: %s' % os.getcwd())
+        print('PYTHONPATH: %s' % self.env['PYTHONPATH'])
         print('Spawning %r' % argv)
-        self.process = subprocess.Popen(argv, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.process = subprocess.Popen(argv, env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         self.is_running = True
         watchdog.create(self.process.pid)
 
@@ -170,7 +180,7 @@ class DebugSession(object):
     def __exit__(self, *args):
         self.wait_for_exit()
 
-    def wait_for_disconnect(self):
+    def wait_for_disconnect(self, close=True):
         """Waits for the connected ptvsd process to disconnect.
         """
 
@@ -180,18 +190,26 @@ class DebugSession(object):
         self.channel.close()
 
         self.timeline.finalize()
-        self.timeline.close()
+        if close:
+            self.timeline.close()
 
     def wait_for_termination(self, expected_returncode=0):
         print(colors.LIGHT_MAGENTA + 'Waiting for ptvsd#%d to terminate' % self.ptvsd_port + colors.RESET)
 
-        self.wait_for_next(Event('terminated'))
-        self.expect_realized(
-            Event('exited', {'exitCode': expected_returncode}) >>
-            Event('terminated', {})
-        )
+        if sys.version_info < (3,):
+            # On 3.x, ptvsd sometimes exits without sending this, likely due to
+            # https://github.com/Microsoft/ptvsd/issues/530
+            self.wait_for_next(Event('terminated'))
 
-        self.wait_for_disconnect()
+        self.wait_for_disconnect(close=False)
+
+        if sys.version_info < (3,) or Event('exited') in self:
+            self.expect_realized(Event('exited', {'exitCode': expected_returncode}))
+
+        if sys.version_info < (3,) or Event('terminated') in self:
+            self.expect_realized(Event('exited') >> Event('terminated', {}))
+
+        self.timeline.close()
 
     def wait_for_exit(self, expected_returncode=0):
         """Waits for the spawned ptvsd process to exit. If it doesn't exit within
@@ -326,8 +344,6 @@ class DebugSession(object):
 
     def _process_event(self, channel, event, body):
         self.timeline.record_event(event, body, block=False)
-        # if event == 'terminated':
-        #     self.channel.close()
 
     def _process_response(self, request, response):
         body = response.body if response.success else RequestFailure(response.error_message)
@@ -393,3 +409,8 @@ class DebugSession(object):
         thread = threading.Thread(target=_output_worker, name='ptvsd#%r %s' % (self.ptvsd_port, name))
         thread.daemon = True
         thread.start()
+        self._output_capture_threads.append(thread)
+
+    def _wait_for_remaining_output(self):
+        for thread in self._output_capture_threads:
+            thread.join()

pytests/helpers/test_timeline.py

@@ -318,7 +318,7 @@ def test_conditional(make_timeline):
         timeline.expect_realized(t >> something_exciting)
 
 
-@pytest.mark.timeout(1)
+@pytest.mark.timeout(3)
 def test_frozen(make_timeline, daemon):
     timeline, initial_history = make_timeline()
     assert not timeline.is_frozen
@@ -361,7 +361,7 @@ def test_frozen(make_timeline, daemon):
     assert Mark('dee') in timeline
 
 
-@pytest.mark.timeout(1)
+@pytest.mark.timeout(3)
 def test_unobserved(make_timeline, daemon):
     timeline, initial_history = make_timeline()
 
@@ -423,7 +423,7 @@ def test_unobserved(make_timeline, daemon):
     timeline.proceed()
 
 
-@pytest.mark.timeout(1)
+@pytest.mark.timeout(3)
 @pytest.mark.parametrize('order', ['mark_then_wait', 'wait_then_mark'])
 def test_concurrency(make_timeline, daemon, order):
     timeline, initial_history = make_timeline()