diff --git a/.flake8 b/.flake8 index 62aba857..b1da661b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] ignore = W, E24,E121,E123,E125,E126,E221,E226,E266,E704, - E265,E722,E501,E731,E306,E401,E302,E222 + E265,E722,E501,E731,E306,E401,E302,E222,E303 exclude = ptvsd/_vendored/pydevd, ./.eggs, diff --git a/ptvsd/__main__.py b/ptvsd/__main__.py index 2053c5ef..17b21121 100644 --- a/ptvsd/__main__.py +++ b/ptvsd/__main__.py @@ -6,7 +6,7 @@ import argparse import os.path import sys -from ptvsd import multiproc +from ptvsd import multiproc, options from ptvsd._attach import attach_main from ptvsd._local import debug_main, run_main from ptvsd.socket import Address @@ -124,8 +124,8 @@ def _group_args(argv): supported.append(arg) # ptvsd support - elif arg in ('--host', '--server-host', '--port', '--pid', '-m', '--multiprocess-port-range'): - if arg in ('-m', '--pid'): + elif arg in ('--host', '--server-host', '--port', '--pid', '-m', '-c', '--multiprocess-port-range'): + if arg in ('-m', '-c', '--pid'): gottarget = True supported.append(arg) if nextarg is not None: @@ -169,6 +169,7 @@ def _parse_args(prog, argv): target = parser.add_mutually_exclusive_group(required=True) target.add_argument('-m', dest='module') + target.add_argument('-c', dest='code') target.add_argument('--pid', type=int) target.add_argument('filename', nargs='?') @@ -205,12 +206,17 @@ def _parse_args(prog, argv): pid = ns.pop('pid') module = ns.pop('module') filename = ns.pop('filename') + code = ns.pop('code') if pid is not None: args.name = pid args.kind = 'pid' elif module is not None: args.name = module args.kind = 'module' + elif code is not None: + options.code = code + args.name = 'ptvsd.run_code' + args.kind = 'module' else: args.name = filename args.kind = 'script' diff --git a/ptvsd/multiproc.py b/ptvsd/multiproc.py index 6f1a2a30..a610fe7d 100644 --- a/ptvsd/multiproc.py +++ b/ptvsd/multiproc.py @@ -55,6 +55,7 @@ def disable(): except Exception: pass + def _listener(): counter = itertools.count(1) while listener_port: diff --git a/ptvsd/options.py b/ptvsd/options.py new file mode 100644 index 00000000..c554bd8b --- /dev/null +++ b/ptvsd/options.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in the project root +# for license information. + +from __future__ import print_function, with_statement, absolute_import + + +"""ptvsd command-line options that need to be globally available. +""" + +code = None +"""When running with -c, specifies the code that needs to be run. +""" diff --git a/ptvsd/run_code.py b/ptvsd/run_code.py new file mode 100644 index 00000000..a0adb982 --- /dev/null +++ b/ptvsd/run_code.py @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in the project root +# for license information. + +# This module is used to implement -c support, since pydevd doesn't support +# it directly. So we tell it to run this module instead, and it just does +# exec on the code. + +# It is crucial that this module does *not* do "from __future__ import ...", +# because we want exec below to use the defaults as defined by the flags that +# were passed to the Python interpreter when it was launched. + +if __name__ == '__main__': + from ptvsd.options import code + exec(code, {}) diff --git a/pytests/conftest.py b/pytests/conftest.py index 3382ecb3..b520ee84 100644 --- a/pytests/conftest.py +++ b/pytests/conftest.py @@ -126,8 +126,8 @@ def pyfile(request, tmpdir): ]) def debug_session(request): session = DebugSession(request.param) - yield session try: + yield session try: failed = request.node.call_result.failed except AttributeError: diff --git a/pytests/func/test_run.py b/pytests/func/test_run.py index cdc1ec39..dc8f483c 100644 --- a/pytests/func/test_run.py +++ b/pytests/func/test_run.py @@ -5,13 +5,17 @@ from __future__ import print_function, with_statement, absolute_import import os +import pytest + import ptvsd -from ..helpers.pattern import ANY -from ..helpers.timeline import Event +from pytests.helpers import print +from pytests.helpers.pattern import ANY +from pytests.helpers.timeline import Event -def test_run(debug_session, pyfile): +@pytest.mark.parametrize('run_as', ['file', 'module', 'code']) +def test_run(debug_session, pyfile, run_as): @pyfile def code_to_debug(): import os @@ -23,13 +27,24 @@ def test_run(debug_session, pyfile): backchannel.write_json(os.path.abspath(sys.modules['ptvsd'].__file__)) print('end') - debug_session.prepare_to_run(filename=code_to_debug, backchannel=True) + if run_as == 'file': + debug_session.prepare_to_run(filename=code_to_debug, backchannel=True) + elif run_as == 'module': + debug_session.add_file_to_pythonpath(code_to_debug) + debug_session.prepare_to_run(module='code_to_debug', backchannel=True) + elif run_as == 'code': + with open(code_to_debug, 'r') as f: + code = f.read() + debug_session.prepare_to_run(code=code, backchannel=True) + else: + pytest.fail() + debug_session.start_debugging() assert debug_session.timeline.is_frozen process_event, = debug_session.all_occurrences_of(Event('process')) assert process_event == Event('process', ANY.dict_with({ - 'name': code_to_debug, + 'name': ANY if run_as == 'code' else code_to_debug, })) debug_session.write_json('continue') diff --git a/pytests/helpers/colors.py b/pytests/helpers/colors.py index 220e1a3c..bac77319 100644 --- a/pytests/helpers/colors.py +++ b/pytests/helpers/colors.py @@ -4,54 +4,89 @@ from __future__ import print_function, with_statement, absolute_import -from colorama import Fore -from pygments import highlight, lexers, formatters, token +import platform -# Colors that are commented out don't work with PowerShell. -RESET = Fore.RESET -BLACK = Fore.BLACK -BLUE = Fore.BLUE -CYAN = Fore.CYAN -GREEN = Fore.GREEN -# MAGENTA = Fore.MAGENTA -RED = Fore.RED -WHITE = Fore.WHITE -# YELLOW = Fore.YELLOW -LIGHT_BLACK = Fore.LIGHTBLACK_EX -LIGHT_BLUE = Fore.LIGHTBLUE_EX -LIGHT_CYAN = Fore.LIGHTCYAN_EX -LIGHT_GREEN = Fore.LIGHTGREEN_EX -LIGHT_MAGENTA = Fore.LIGHTMAGENTA_EX -LIGHT_RED = Fore.LIGHTRED_EX -LIGHT_WHITE = Fore.LIGHTWHITE_EX -LIGHT_YELLOW = Fore.LIGHTYELLOW_EX +if platform.system() == 'Windows': + # pytest-timeout seems to be buggy wrt colorama when capturing output. + # + # TODO: re-enable after enabling proper ANSI sequence handling: + # https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences + + RESET = '' + BLACK = '' + BLUE = '' + CYAN = '' + GREEN = '' + RED = '' + WHITE = '' + LIGHT_BLACK = '' + LIGHT_BLUE = '' + LIGHT_CYAN = '' + LIGHT_GREEN = '' + LIGHT_MAGENTA = '' + LIGHT_RED = '' + LIGHT_WHITE = '' + LIGHT_YELLOW = '' -color_scheme = { - token.Token: ('white', 'white'), - token.Punctuation: ('', ''), - token.Operator: ('', ''), - token.Literal: ('brown', 'brown'), - token.Keyword: ('brown', 'brown'), - token.Name: ('white', 'white'), - token.Name.Constant: ('brown', 'brown'), - token.Name.Attribute: ('brown', 'brown'), - # token.Name.Tag: ('white', 'white'), - # token.Name.Function: ('white', 'white'), - # token.Name.Variable: ('white', 'white'), -} - -formatter = formatters.TerminalFormatter(colorscheme=color_scheme) -json_lexer = lexers.JsonLexer() -python_lexer = lexers.PythonLexer() + def colorize_json(s): + return s -def colorize_json(s): - return highlight(s, json_lexer, formatter).rstrip() + def color_repr(obj): + return repr(obj) -def color_repr(obj): - return highlight(repr(obj), python_lexer, formatter).rstrip() +else: + from colorama import Fore + from pygments import highlight, lexers, formatters, token + + + # Colors that are commented out don't work with PowerShell. + RESET = Fore.RESET + BLACK = Fore.BLACK + BLUE = Fore.BLUE + CYAN = Fore.CYAN + GREEN = Fore.GREEN + # MAGENTA = Fore.MAGENTA + RED = Fore.RED + WHITE = Fore.WHITE + # YELLOW = Fore.YELLOW + LIGHT_BLACK = Fore.LIGHTBLACK_EX + LIGHT_BLUE = Fore.LIGHTBLUE_EX + LIGHT_CYAN = Fore.LIGHTCYAN_EX + LIGHT_GREEN = Fore.LIGHTGREEN_EX + LIGHT_MAGENTA = Fore.LIGHTMAGENTA_EX + LIGHT_RED = Fore.LIGHTRED_EX + LIGHT_WHITE = Fore.LIGHTWHITE_EX + LIGHT_YELLOW = Fore.LIGHTYELLOW_EX + + + color_scheme = { + token.Token: ('white', 'white'), + token.Punctuation: ('', ''), + token.Operator: ('', ''), + token.Literal: ('brown', 'brown'), + token.Keyword: ('brown', 'brown'), + token.Name: ('white', 'white'), + token.Name.Constant: ('brown', 'brown'), + token.Name.Attribute: ('brown', 'brown'), + # token.Name.Tag: ('white', 'white'), + # token.Name.Function: ('white', 'white'), + # token.Name.Variable: ('white', 'white'), + } + + formatter = formatters.TerminalFormatter(colorscheme=color_scheme) + json_lexer = lexers.JsonLexer() + python_lexer = lexers.PythonLexer() + + + def colorize_json(s): + return highlight(s, json_lexer, formatter).rstrip() + + + def color_repr(obj): + return highlight(repr(obj), python_lexer, formatter).rstrip() diff --git a/pytests/helpers/session.py b/pytests/helpers/session.py index 5305afbb..5e07382f 100644 --- a/pytests/helpers/session.py +++ b/pytests/helpers/session.py @@ -20,11 +20,11 @@ from .timeline import Timeline, Event, Response # ptvsd.__file__ will be /ptvsd/__main__.py - we want . -PTVSD_SYS_PATH = os.path.basename(os.path.basename(ptvsd.__file__)) +PTVSD_SYS_PATH = os.path.dirname(os.path.dirname(ptvsd.__file__)) class DebugSession(object): - WAIT_FOR_EXIT_TIMEOUT = 3 + WAIT_FOR_EXIT_TIMEOUT = 5 BACKCHANNEL_TIMEOUT = 10 def __init__(self, method='launch', ptvsd_port=None): @@ -37,6 +37,9 @@ class DebugSession(object): self.ptvsd_port = ptvsd_port or 5678 self.multiprocess = False self.multiprocess_port_range = None + self.debug_options = ['RedirectOutput'] + self.env = os.environ.copy() + self.env['PYTHONPATH'] = PTVSD_SYS_PATH self.is_running = False self.process = None @@ -46,7 +49,7 @@ class DebugSession(object): self.backchannel_socket = None self.backchannel_port = None self.backchannel_established = threading.Event() - self.debug_options = ['RedirectOutput'] + self._output_capture_threads = [] self.timeline = Timeline(ignore_unobserved=[ Event('output'), @@ -66,6 +69,9 @@ class DebugSession(object): self.expect_realized = self.timeline.expect_realized self.all_occurrences_of = self.timeline.all_occurrences_of + def add_file_to_pythonpath(self, filename): + self.env['PYTHONPATH'] += os.pathsep + os.path.dirname(filename) + def __contains__(self, expectation): return expectation in self.timeline @@ -98,10 +104,12 @@ class DebugSession(object): self.backchannel_socket.shutdown(socket.SHUT_RDWR) except: self.backchannel_socket = None + self._wait_for_remaining_output() - def prepare_to_run(self, perform_handshake=True, filename=None, module=None, backchannel=False): + def prepare_to_run(self, perform_handshake=True, filename=None, module=None, code=None, backchannel=False): """Spawns ptvsd using the configured method, telling it to execute the - provided Python file or module, and establishes a message channel to it. + provided Python file, module, or code, and establishes a message channel + to it. If backchannel is True, calls self.setup_backchannel() before returning. @@ -125,22 +133,24 @@ class DebugSession(object): argv += ['--multiprocess-port-range', '%d-%d' % self.multiprocess_port_range] if filename: - assert not module + assert not module and not code argv += [filename] elif module: - assert not filename + assert not filename and not code argv += ['-m', module] - - env = os.environ.copy() - env.update({'PYTHONPATH': PTVSD_SYS_PATH}) + elif code: + assert not filename and not module + argv += ['-c', code] if backchannel: self.setup_backchannel() if self.backchannel_port: - env['PTVSD_BACKCHANNEL_PORT'] = str(self.backchannel_port) + self.env['PTVSD_BACKCHANNEL_PORT'] = str(self.backchannel_port) + print('Current directory: %s' % os.getcwd()) + print('PYTHONPATH: %s' % self.env['PYTHONPATH']) print('Spawning %r' % argv) - self.process = subprocess.Popen(argv, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.process = subprocess.Popen(argv, env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.is_running = True watchdog.create(self.process.pid) @@ -170,7 +180,7 @@ class DebugSession(object): def __exit__(self, *args): self.wait_for_exit() - def wait_for_disconnect(self): + def wait_for_disconnect(self, close=True): """Waits for the connected ptvsd process to disconnect. """ @@ -180,18 +190,26 @@ class DebugSession(object): self.channel.close() self.timeline.finalize() - self.timeline.close() + if close: + self.timeline.close() def wait_for_termination(self, expected_returncode=0): print(colors.LIGHT_MAGENTA + 'Waiting for ptvsd#%d to terminate' % self.ptvsd_port + colors.RESET) - self.wait_for_next(Event('terminated')) - self.expect_realized( - Event('exited', {'exitCode': expected_returncode}) >> - Event('terminated', {}) - ) + if sys.version_info < (3,): + # On 3.x, ptvsd sometimes exits without sending this, likely due to + # https://github.com/Microsoft/ptvsd/issues/530 + self.wait_for_next(Event('terminated')) - self.wait_for_disconnect() + self.wait_for_disconnect(close=False) + + if sys.version_info < (3,) or Event('exited') in self: + self.expect_realized(Event('exited', {'exitCode': expected_returncode})) + + if sys.version_info < (3,) or Event('terminated') in self: + self.expect_realized(Event('exited') >> Event('terminated', {})) + + self.timeline.close() def wait_for_exit(self, expected_returncode=0): """Waits for the spawned ptvsd process to exit. If it doesn't exit within @@ -326,8 +344,6 @@ class DebugSession(object): def _process_event(self, channel, event, body): self.timeline.record_event(event, body, block=False) - # if event == 'terminated': - # self.channel.close() def _process_response(self, request, response): body = response.body if response.success else RequestFailure(response.error_message) @@ -393,3 +409,8 @@ class DebugSession(object): thread = threading.Thread(target=_output_worker, name='ptvsd#%r %s' % (self.ptvsd_port, name)) thread.daemon = True thread.start() + self._output_capture_threads.append(thread) + + def _wait_for_remaining_output(self): + for thread in self._output_capture_threads: + thread.join() diff --git a/pytests/helpers/test_timeline.py b/pytests/helpers/test_timeline.py index 9c9c3d60..248cd645 100644 --- a/pytests/helpers/test_timeline.py +++ b/pytests/helpers/test_timeline.py @@ -318,7 +318,7 @@ def test_conditional(make_timeline): timeline.expect_realized(t >> something_exciting) -@pytest.mark.timeout(1) +@pytest.mark.timeout(3) def test_frozen(make_timeline, daemon): timeline, initial_history = make_timeline() assert not timeline.is_frozen @@ -361,7 +361,7 @@ def test_frozen(make_timeline, daemon): assert Mark('dee') in timeline -@pytest.mark.timeout(1) +@pytest.mark.timeout(3) def test_unobserved(make_timeline, daemon): timeline, initial_history = make_timeline() @@ -423,7 +423,7 @@ def test_unobserved(make_timeline, daemon): timeline.proceed() -@pytest.mark.timeout(1) +@pytest.mark.timeout(3) @pytest.mark.parametrize('order', ['mark_then_wait', 'wait_then_mark']) def test_concurrency(make_timeline, daemon, order): timeline, initial_history = make_timeline()