Merge pull request #160 from ericsnowcurrently/fix-highlevel-tests

Fix highlevel tests.

ptvsd/ipcjson.py

@@ -287,7 +287,7 @@ class IpcChannel(object):
                 self._wait_for_message()
             except OSError as exc:
                 if exc.errno == errno.EBADF or self.__exit:  # socket closed
-                    return self.__exit                
+                    return self.__exit
                 raise
             except Exception:
                 if self.__exit:

ptvsd/wrapper.py

@@ -41,6 +41,7 @@ ptvsd_sys_exit_code = 0
 WAIT_FOR_DISCONNECT_REQUEST_TIMEOUT = 2
 WAIT_FOR_THREAD_FINISH_TIMEOUT = 1
 
+
 def unquote(s):
     if s is None:
         return None
@@ -326,7 +327,7 @@ class ExceptionsManager(object):
             notify_on_terminate,
             ignore_libraries,
         )
-        break_mode = 'never' 
+        break_mode = 'never'
         if break_raised:
             break_mode = 'always'
         elif break_uncaught:
@@ -401,12 +402,13 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
     protocol.
     """
 
-    def __init__(self, socket, pydevd, logfile=None):
+    def __init__(self, socket, pydevd, logfile=None, killonclose=True):
         super(VSCodeMessageProcessor, self).__init__(socket=socket,
                                                      own_socket=False,
                                                      logfile=logfile)
         self.socket = socket
         self.pydevd = pydevd
+        self.killonclose = killonclose
         self.stack_traces = {}
         self.stack_traces_lock = threading.Lock()
         self.active_exceptions = {}
@@ -425,7 +427,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
         self._closed = False
 
         self.event_loop_thread = threading.Thread(target=self.loop.run_forever,
-                             name='ptvsd.EventLoop')
+                                                  name='ptvsd.EventLoop')
         self.event_loop_thread.daemon = True
         self.event_loop_thread.start()
 
@@ -449,7 +451,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
             self.send_response(self.disconnect_request)
             self.disconnect_request = None
 
-        self.set_exit()    
+        self.set_exit()
         self.loop.stop()
         self.event_loop_thread.join(WAIT_FOR_THREAD_FINISH_TIMEOUT)
 
@@ -543,13 +545,16 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
         self.process_launch_arguments()
         self.pydevd_request(pydevd_comm.CMD_RUN, '')
         self.send_process_event(self.start_reason)
-    
+
     def process_launch_arguments(self):
         """Process the launch arguments to configure the debugger"""
         if self.launch_arguments is None:
             return
 
-        redirect_stdout = 'STDOUT\tSTDERR' if self.launch_arguments.get('redirectOutput', False) == True else ''
+        if self.launch_arguments.get('redirectOutput', False):
+            redirect_stdout = 'STDOUT\tSTDERR'
+        else:
+            redirect_stdout = ''
         self.pydevd_request(pydevd_comm.CMD_REDIRECT_OUTPUT, redirect_stdout)
 
     def on_disconnect(self, request, args):
@@ -559,10 +564,10 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
             self.disconnect_request_event.set()
             killProcess = not self._closed
             self.close()
-            if killProcess:
+            if killProcess and self.killonclose:
                 os.kill(os.getpid(), signal.SIGTERM)
         else:
-            self.send_response(request)    
+            self.send_response(request)
 
     @async_handler
     def on_attach(self, request, args):
@@ -612,7 +617,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
                 name = unquote(xthread['name'])
             except KeyError:
                 name = None
-            
+
             if not self.is_debugger_internal_thread(name):
                 pyd_tid = xthread['id']
                 try:
@@ -620,7 +625,8 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
                 except KeyError:
                     # This is a previously unseen thread
                     vsc_tid = self.thread_map.to_vscode(pyd_tid, autogen=True)
-                    self.send_event('thread', reason='started', threadId=vsc_tid)
+                    self.send_event('thread', reason='started',
+                                    threadId=vsc_tid)
 
                 threads.append({'id': vsc_tid, 'name': name})
 
@@ -638,7 +644,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
             try:
                 xframes = self.stack_traces[pyd_tid]
             except KeyError:
-                # This means the stack was requested before the 
+                # This means the stack was requested before the
                 # thread was suspended
                 xframes = []
         totalFrames = len(xframes)
@@ -712,7 +718,8 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
             }
             if bool(xvar['isContainer']):
                 pyd_child = pyd_var + (var['name'],)
-                var['variablesReference'] = self.var_map.to_vscode(pyd_child, autogen=True)
+                var['variablesReference'] = self.var_map.to_vscode(
+                    pyd_child, autogen=True)
             variables.append(var)
 
         self.send_response(request, variables=variables)
@@ -872,7 +879,9 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
         )
 
     def get_exceptionInfo(self, pyd_tid):
-        """Gets the exception name and description for the given PyDev Thread id"""
+        """
+        Gets the exception name and description for the given PyDev Thread id.
+        """
         with self.active_exceptions_lock:
             try:
                 exc = self.active_exceptions[pyd_tid]
@@ -880,7 +889,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
                 exc = ExceptionInfo('BaseException',
                                     'exception: no description')
         return (exc.name, exc.description)
-        
+
     @pydevd_events.handler(pydevd_comm.CMD_THREAD_CREATE)
     def on_pydevd_thread_create(self, seq, args):
         # TODO: docstring
@@ -926,7 +935,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
             vsc_tid = self.thread_map.to_vscode(pyd_tid, autogen=False)
         except KeyError:
             return
-        
+
         text = None
         if reason in STEP_REASONS:
             reason = 'step'
@@ -941,7 +950,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
 
         with self.stack_traces_lock:
             self.stack_traces[pyd_tid] = xml.thread.frame
-        
+
         self.send_event('stopped', reason=reason, threadId=vsc_tid, text=text)
 
     @pydevd_events.handler(pydevd_comm.CMD_THREAD_RUN)
@@ -965,7 +974,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
         for pyd_var, vsc_var in self.var_map.pairs():
             if pyd_var[0] == pyd_tid:
                 self.var_map.remove(pyd_var, vsc_var)
-        
+
         try:
             vsc_tid = self.thread_map.to_vscode(pyd_tid, autogen=False)
         except KeyError:
@@ -1000,7 +1009,7 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel):
         category = 'stdout' if ctx == '1' else 'stderr'
         content = unquote(xml.io['s'])
         self.send_event('output', category=category, output=content)
-        
+
 
 ########################
 # lifecycle
@@ -1024,11 +1033,12 @@ def _new_sock():
     return sock
 
 
-def _start(client, server):
+def _start(client, server, killonclose=True):
     name = 'ptvsd.Client' if server is None else 'ptvsd.Server'
 
     pydevd = PydevdSocket(lambda *args: proc.on_pydevd_event(*args))
-    proc = VSCodeMessageProcessor(client, pydevd)
+    proc = VSCodeMessageProcessor(client, pydevd,
+                                  killonclose=killonclose)
 
     server_thread = threading.Thread(target=proc.process_messages,
                                      name=name)
@@ -1046,10 +1056,12 @@ def exit_handler(proc, server_thread):
     if server_thread.is_alive():
         server_thread.join(WAIT_FOR_THREAD_FINISH_TIMEOUT)
 
+
 def signal_handler(signum, frame, proc):
     proc.close()
     sys.exit(0)
 
+
 def start_server(port):
     """Return a socket to a (new) local pydevd-handling daemon.
 
@@ -1063,7 +1075,10 @@ def start_server(port):
     pydevd, proc, server_thread = _start(client, server)
     atexit.register(lambda: exit_handler(proc, server_thread))
     if platform.system() != 'Windows':
-        signal.signal(signal.SIGHUP, lambda signum, frame: signal_handler(signum, frame, proc))
+        signal.signal(
+            signal.SIGHUP,
+            (lambda signum, frame: signal_handler(signum, frame, proc)),
+        )
     return pydevd
 
 
@@ -1080,7 +1095,10 @@ def start_client(host, port):
     pydevd, proc, server_thread = _start(client, None)
     atexit.register(lambda: exit_handler(proc, server_thread))
     if platform.system() != 'Windows':
-        signal.signal(signal.SIGHUP, lambda signum, frame: signal_handler(signum, frame, proc))
+        signal.signal(
+            signal.SIGHUP,
+            (lambda signum, frame: signal_handler(signum, frame, proc)),
+        )
     return pydevd
 
 

tests/__main__.py

@@ -58,6 +58,15 @@ def convert_argv(argv):
     return cmd + args
 
 
+def fix_sys_path():
+    pydevdroot = os.path.join(PROJECT_ROOT, 'ptvsd', 'pydevd')
+    if not sys.path[0] or sys.path[0] == '.':
+        sys.path.insert(1, pydevdroot)
+    else:
+        sys.path.insert(0, pydevdroot)
+
+
 if __name__ == '__main__':
     argv = convert_argv(sys.argv[1:])
+    fix_sys_path()
     unittest.main(module=None, argv=argv)

tests/helpers/counter.py

@@ -0,0 +1,56 @@
+
+class Counter(object):
+    """An introspectable, dynamic alternative to itertools.count()."""
+
+    def __init__(self, start=0, step=1):
+        self._start = int(start)
+        self._step = int(step)
+
+    def __repr__(self):
+        return '{}(start={}, step={})'.format(
+            type(self).__name__,
+            self.peek(),
+            self._step,
+        )
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            self._last += self._step
+        except AttributeError:
+            self._last = self._start
+        return self._last
+
+    @property
+    def start(self):
+        return self._start
+
+    @property
+    def step(self):
+        return self._step
+
+    @property
+    def last(self):
+        try:
+            return self._last
+        except AttributeError:
+            return None
+
+    def peek(self, iterations=1):
+        """Return the value that will be used next."""
+        try:
+            last = self._last
+        except AttributeError:
+            last = self._start - self._step
+        return last + self._step * iterations
+
+    def reset(self, start=None):
+        """Set the next value to the given one.
+
+        If no value is provided then the previous start value is used.
+        """
+        if start is not None:
+            self._start = int(start)
+        self._next = self._start

tests/helpers/protocol.py

@@ -3,6 +3,7 @@ import errno
 import threading
 
 from . import socket
+from .counter import Counter
 
 
 class StreamFailure(Exception):
@@ -40,6 +41,42 @@ class MessageProtocol(namedtuple('Protocol', 'parse encode iter')):
             yield self.parse(msg)
 
 
+class MessageCounters(namedtuple('MessageCounters',
+                                 'request response event')):
+    """Track the next "seq" value for the protocol message types."""
+
+    REQUEST_INC = 1
+    RESPONSE_INC = 1
+    EVENT_INC = 1
+
+    def __new__(cls, request=0, response=0, event=None):
+        request = Counter(request, cls.REQUEST_INC)
+        if response is None:
+            response = request
+        else:
+            response = Counter(response, cls.RESPONSE_INC)
+        if event is None:
+            event = response
+        else:
+            event = Counter(event, cls.EVENT_INC)
+        self = super(MessageCounters, cls).__new__(
+            cls,
+            request,
+            response,
+            event,
+        )
+        return self
+
+    def next_request(self):
+        return next(self.request)
+
+    def next_response(self):
+        return next(self.response)
+
+    def next_event(self):
+        return next(self.event)
+
+
 class Started(object):
     """A simple wrapper around a started message protocol daemon."""
 

tests/helpers/pydevd/_fake.py

@@ -19,7 +19,7 @@ def _bind(address):
 
     def connect(_connect=connect):
         client, server = _connect()
-        pydevd, _, _ = _ptvsd._start(client, server)
+        pydevd, _, _ = _ptvsd._start(client, server, killonclose=False)
         return socket.Connection(pydevd, server)
     return connect, remote
 

tests/helpers/vsc/_fake.py

@@ -89,18 +89,26 @@ class FakeVSC(protocol.Daemon):
         command = req['command']
 
         def match(msg):
-            msg = msg.data
-            if msg['type'] != 'response' or msg['request_seq'] != reqseq:
+            #msg = parse_message(msg)
+            try:
+                actual = msg.request_seq
+            except AttributeError:
                 return False
-            assert(msg['command'] == command)
+            if actual != reqseq:
+                return False
+            assert(msg.command == command)
             return True
 
         return self._wait_for_message(match, req, **kwargs)
 
     def wait_for_event(self, event, **kwargs):
         def match(msg):
-            msg = msg.data
-            if msg['type'] != 'event' or msg['event'] != event:
+            #msg = parse_message(msg)
+            try:
+                actual = msg.event
+            except AttributeError:
+                return False
+            if actual != event:
                 return False
             return True
 

tests/helpers/vsc/_vsc.py

@@ -7,6 +7,12 @@ from tests.helpers.protocol import StreamFailure
 # TODO: Use more of the code from debugger_protocol.
 
 
+class ProtocolMessageError(Exception): pass  # noqa
+class MalformedMessageError(ProtocolMessageError): pass  # noqa
+class IncompleteMessageError(MalformedMessageError): pass  # noqa
+class UnsupportedMessageTypeError(ProtocolMessageError): pass  # noqa
+
+
 def parse_message(msg):
     """Return a message object for the given "msg" data."""
     if type(msg) is str:
@@ -14,10 +20,29 @@ def parse_message(msg):
     elif isinstance(msg, bytes):
         data = json.loads(msg.decode('utf-8'))
     elif type(msg) is RawMessage:
-        return msg
+        try:
+            msg.data['seq']
+            msg.data['type']
+        except KeyError:
+            return msg
+        return parse_message(msg.data)
+    elif isinstance(msg, ProtocolMessage):
+        if msg.TYPE is not None:
+            return msg
+        try:
+            ProtocolMessage._look_up(msg.type)
+        except UnsupportedMessageTypeError:
+            return msg
+        data = msg.as_data()
     else:
         data = msg
-    return RawMessage.from_data(**data)
+
+    cls = look_up(data)
+    try:
+        return cls.from_data(**data)
+    except IncompleteMessageError:
+        # TODO: simply fail?
+        return RawMessage.from_data(**data)
 
 
 def encode_message(msg):
@@ -29,7 +54,8 @@ def iter_messages(stream, stop=lambda: False):
     """Yield the correct message for each line-formatted one found."""
     while not stop():
         try:
-            msg = wireformat.read(stream, lambda _: RawMessage)
+            #msg = wireformat.read(stream, lambda _: RawMessage)
+            msg = wireformat.read(stream, look_up)
             if msg is None:  # EOF
                 break
             yield msg
@@ -37,6 +63,20 @@ def iter_messages(stream, stop=lambda: False):
             yield StreamFailure('recv', None, exc)
 
 
+def look_up(data):
+    """Return the message type to use."""
+    try:
+        msgtype = data['type']
+    except KeyError:
+        # TODO: return RawMessage?
+        ProtocolMessage._check_data(data)
+    try:
+        return ProtocolMessage._look_up(msgtype)
+    except UnsupportedMessageTypeError:
+        # TODO: return Message?
+        raise
+
+
 class RawMessage(namedtuple('RawMessage', 'data')):
     """A wrapper around a line-formatted debugger protocol message."""
 
@@ -54,3 +94,142 @@ class RawMessage(namedtuple('RawMessage', 'data')):
     def as_data(self):
         """Return the corresponding data, ready to be JSON-encoded."""
         return self.data
+
+
+class ProtocolMessage(object):
+    """The base type for VSC debug adapter protocol message."""
+
+    TYPE = None
+
+    @classmethod
+    def from_data(cls, **data):
+        """Return a message for the given JSON-decoded data."""
+        try:
+            return cls(**data)
+        except TypeError:
+            cls._check_data(data)
+            raise
+
+    @classmethod
+    def _check_data(cls, data):
+        missing = set(cls._fields) - set(data)
+        if missing:
+            raise IncompleteMessageError(','.join(missing))
+
+    @classmethod
+    def _look_up(cls, msgtype):
+        if msgtype == 'request':
+            return Request
+        elif msgtype == 'response':
+            return Response
+        elif msgtype == 'event':
+            return Event
+        else:
+            raise UnsupportedMessageTypeError(msgtype)
+
+    def __new__(cls, seq, type, **kwargs):
+        if cls is ProtocolMessage:
+            return Message(seq, type, **kwargs)
+        seq = int(seq)
+        type = str(type) if type else None
+        unused = {k: kwargs.pop(k)
+                  for k in tuple(kwargs)
+                  if k not in cls._fields}
+        self = super(ProtocolMessage, cls).__new__(cls, seq, type, **kwargs)
+        self._unused = unused
+        return self
+
+    def __init__(self, *args, **kwargs):
+        if self.TYPE is None:
+            if self.type is None:
+                raise TypeError('missing type')
+        elif self.type != self.TYPE:
+            msg = 'wrong type (expected {!r}, go {!r}'
+            raise ValueError(msg.format(self.TYPE, self.type))
+
+    def __repr__(self):
+        raw = super(ProtocolMessage, self).__repr__()
+        if self.TYPE is None:
+            return raw
+        return ', '.join(part
+                         for part in raw.split(', ')
+                         if not part.startswith('type='))
+
+    @property
+    def unused(self):
+        return dict(self._unused)
+
+    def as_data(self):
+        """Return the corresponding data, ready to be JSON-encoded."""
+        data = self._asdict()
+        data.update(self._unused)
+        return data
+
+
+class Message(ProtocolMessage, namedtuple('Message', 'seq type')):
+    """A generic DAP message."""
+
+    def __getattr__(self, name):
+        try:
+            return self._unused[name]
+        except KeyError:
+            raise AttributeError(name)
+
+
+class Request(ProtocolMessage,
+              namedtuple('Request', 'seq type command arguments')):
+    """A DAP request message."""
+
+    TYPE = 'request'
+
+    def __new__(cls, seq, type, command, arguments, **unused):
+        # TODO: Make "arguments" immutable?
+        return super(Request, cls).__new__(
+            cls,
+            seq,
+            type,
+            command=command,
+            arguments=arguments,
+            **unused
+        )
+
+
+class Response(ProtocolMessage,
+               namedtuple('Response',
+                          'seq type request_seq command success message body'),
+               ):
+    """A DAP response message."""
+
+    TYPE = 'response'
+
+    def __new__(cls, seq, type, request_seq, command, success, message, body,
+                **unused):
+        # TODO: Make "body" immutable?
+        return super(Response, cls).__new__(
+            cls,
+            seq,
+            type,
+            request_seq=request_seq,
+            command=command,
+            success=success,
+            message=message,
+            body=body,
+            **unused
+        )
+
+
+class Event(ProtocolMessage, namedtuple('Event', 'seq type event body')):
+    """A DAP event message."""
+
+    TYPE = 'event'
+
+    def __new__(cls, seq, type, event, body, **unused):
+        # TODO: Make "body" immutable?
+        return super(Event, cls).__new__(
+            cls,
+            seq,
+            type,
+            event=event,
+            body=body,
+            **unused
+        )

tests/ptvsd/highlevel/__init__.py

@@ -11,12 +11,14 @@ from _pydevd_bundle.pydevd_comm import (
     CMD_VERSION,
     CMD_LIST_THREADS,
     CMD_THREAD_SUSPEND,
+    CMD_REDIRECT_OUTPUT,
     CMD_RETURN,
     CMD_RUN,
     CMD_STEP_CAUGHT_EXCEPTION,
     CMD_SEND_CURR_EXCEPTION_TRACE,
 )
 
+from tests.helpers.protocol import MessageCounters
 from tests.helpers.pydevd import FakePyDevd
 from tests.helpers.vsc import FakeVSC
 
@@ -33,18 +35,20 @@ class PyDevdMessages(object):
                  response_seq=0,  # PyDevd responses/events to ptvsd
                  event_seq=None,
                  ):
-        self.request_seq = itertools.count(request_seq)
-        self.response_seq = itertools.count(response_seq)
-        if event_seq is None:
-            self.event_seq = self.response_seq
-        else:
-            self.event_seq = itertools.count(event_seq)
+        self.counters = MessageCounters(
+            request_seq,
+            response_seq,
+            event_seq,
+        )
+
+    def __getattr__(self, name):
+        return getattr(self.counters, name)
 
     def new_request(self, cmdid, *args, **kwargs):
         """Return a new PyDevd request message."""
         seq = kwargs.pop('seq', None)
         if seq is None:
-            seq = next(self.request_seq)
+            seq = self.counters.next_request()
         return self._new_message(cmdid, seq, args, **kwargs)
 
     def new_response(self, req, *args):
@@ -59,7 +63,7 @@ class PyDevdMessages(object):
         """Return a new VSC event message."""
         seq = kwargs.pop('seq', None)
         if seq is None:
-            seq = next(self.event_seq)
+            seq = self.counters.next_event()
         return self._new_message(cmdid, seq, args, **kwargs)
 
     def _new_message(self, cmdid, seq, args=()):
@@ -120,17 +124,19 @@ class VSCMessages(object):
                  response_seq=0,  # ptvsd responses/events to VSC
                  event_seq=None,
                  ):
-        self.request_seq = itertools.count(request_seq)
-        self.response_seq = itertools.count(response_seq)
-        if event_seq is None:
-            self.event_seq = self.response_seq
-        else:
-            self.event_seq = itertools.count(event_seq)
+        self.counters = MessageCounters(
+            request_seq,
+            response_seq,
+            event_seq,
+        )
+
+    def __getattr__(self, name):
+        return getattr(self.counters, name)
 
     def new_request(self, command, seq=None, **args):
         """Return a new VSC request message."""
         if seq is None:
-            seq = next(self.request_seq)
+            seq = self.counters.next_request()
         return {
             'type': 'request',
             'seq': seq,
@@ -148,7 +154,7 @@ class VSCMessages(object):
 
     def _new_response(self, req, err=None, seq=None, body=None):
         if seq is None:
-            seq = next(self.response_seq)
+            seq = self.counters.next_response()
         return {
             'type': 'response',
             'seq': seq,
@@ -162,7 +168,7 @@ class VSCMessages(object):
     def new_event(self, eventname, seq=None, **body):
         """Return a new VSC event message."""
         if seq is None:
-            seq = next(self.event_seq)
+            seq = self.counters.next_event()
         return {
             'type': 'event',
             'seq': seq,
@@ -230,13 +236,15 @@ class VSCLifecycle(object):
             self._initialize(**initargs)
             self._fix.send_request(command, **kwargs)
 
-        self._fix.set_threads(*threads or (),
-                              **dict(default_threads=default_threads))
+        if threads:
+            self._fix.set_threads(*threads,
+                                  **dict(default_threads=default_threads))
 
         self._handle_config(**config or {})
-        with self._fix.expect_debugger_command(CMD_RUN):
-            with self._fix.wait_for_event('process'):
-                self._fix.send_request('configurationDone')
+        with self._fix.expect_debugger_command(CMD_REDIRECT_OUTPUT):
+            with self._fix.expect_debugger_command(CMD_RUN):
+                with self._fix.wait_for_event('process'):
+                    self._fix.send_request('configurationDone')
 
         if reset:
             self._fix.reset()
@@ -248,7 +256,7 @@ class VSCLifecycle(object):
         See https://code.visualstudio.com/docs/extensionAPI/api-debugging#_the-vs-code-debug-protocol-in-a-nutshell
         """  # noqa
         def handle_response(resp, _):
-            self._capabilities = resp.data['body']
+            self._capabilities = resp.body
         version = self._fix.debugger.VERSION
         self._fix.set_debugger_response(CMD_VERSION, version)
         self._fix.send_request(
@@ -291,6 +299,7 @@ class HighlevelFixture(object):
         self.debugger_msgs = PyDevdMessages()
 
         self._hidden = False
+        self._default_threads = None
 
     @property
     def vsc(self):
@@ -353,25 +362,34 @@ class HighlevelFixture(object):
         with self.vsc.wait_for_response(req, handler=handler):
             yield req
         if self._hidden:
-            next(self.vsc_msgs.response_seq)
+            self.vsc_msgs.next_response()
 
     @contextlib.contextmanager
     def wait_for_event(self, event, *args, **kwargs):
         with self.vsc.wait_for_event(event, *args, **kwargs):
             yield
         if self._hidden:
-            next(self.vsc_msgs.event_seq)
+            self.vsc_msgs.next_event()
+
+    @contextlib.contextmanager
+    def _wait_for_events(self, events):
+        if not events:
+            yield
+            return
+        with self._wait_for_events(events[1:]):
+            with self.wait_for_event(events[0]):
+                yield
 
     @contextlib.contextmanager
     def expect_debugger_command(self, cmdid):
         yield
         if self._hidden:
-            next(self.debugger_msgs.request_seq)
+            self.debugger_msgs.next_request()
 
     def set_debugger_response(self, cmdid, payload, **kwargs):
         self.debugger.add_pending_response(cmdid, payload, **kwargs)
         if self._hidden:
-            next(self.debugger_msgs.request_seq)
+            self.debugger_msgs.next_request()
 
     def send_debugger_event(self, cmdid, payload):
         event = self.debugger_msgs.new_event(cmdid, payload)
@@ -385,22 +403,35 @@ class HighlevelFixture(object):
             self.send_debugger_event(cmdid, text)
             return None
 
-    def set_threads(self, *threads, **kwargs):
+    def set_threads(self, _thread, *threads, **kwargs):
+        threads = (_thread,) + threads
         return self._set_threads(threads, **kwargs)
 
     def set_thread(self, thread):
-        return self.set_threads(thread)[thread]
+        threads = (thread,)
+        return self._set_threads(threads)[thread]
 
     def _set_threads(self, threads, default_threads=True):
         request = {t[1]: t for t in threads}
         response = {t: None for t in threads}
         if default_threads:
             threads = self._add_default_threads(threads)
+        active = [name
+                  for _, name in threads
+                  if not name.startswith(('ptvsd.', 'pydevd.'))]
         text = self.debugger_msgs.format_threads(*threads)
         self.set_debugger_response(CMD_RETURN, text, reqid=CMD_LIST_THREADS)
-        self.send_request('threads')
+        with self._wait_for_events(['thread' for _ in active]):
+            self.send_request('threads')
 
-        for tinfo in self.vsc.received[-1].data['body']['threads']:
+        for msg in reversed(self.vsc.received):
+            if msg.type == 'response':
+                if msg.command == 'threads':
+                    break
+        else:
+            assert False, 'we waited for the response in send_request()'
+
+        for tinfo in msg.body['threads']:
             try:
                 thread = request[tinfo['name']]
             except KeyError:
@@ -409,6 +440,8 @@ class HighlevelFixture(object):
         return response
 
     def _add_default_threads(self, threads):
+        if self._default_threads is not None:
+            return threads
         defaults = {
             'MainThread',
             'ptvsd.Server',
@@ -427,6 +460,7 @@ class HighlevelFixture(object):
             tid = next(ids)
             thread = tid, tname
             allthreads.append(thread)
+        self._default_threads = list(allthreads)
         allthreads.extend(threads)
         return allthreads
 
@@ -524,7 +558,7 @@ class HighlevelTest(object):
 
         failure = received[-1]
         expected = self.vsc.protocol.parse(
-            self.fix.vsc_msgs.new_failure(req, failure.data['message']))
+            self.fix.vsc_msgs.new_failure(req, failure.message))
         self.assertEqual(failure, expected)
 
     def assert_received(self, daemon, expected):

tests/ptvsd/highlevel/test_lifecycle.py

@@ -3,6 +3,7 @@ import sys
 import unittest
 
 from _pydevd_bundle.pydevd_comm import (
+    CMD_REDIRECT_OUTPUT,
     CMD_RUN,
     CMD_VERSION,
 )
@@ -138,11 +139,13 @@ class LifecycleTests(HighlevelTest, unittest.TestCase):
                 isLocalProcess=True,
                 startMethod='launch',
             )),
-            self.new_response(req_disconnect),
             self.new_event('exited', exitCode=0),
+            self.new_event('terminated'),
+            self.new_response(req_disconnect),
         ])
         self.assert_received(self.debugger, [
             self.debugger_msgs.new_request(CMD_VERSION,
                                            *['1.1', OS_ID, 'ID']),
+            self.debugger_msgs.new_request(CMD_REDIRECT_OUTPUT),
             self.debugger_msgs.new_request(CMD_RUN),
         ])

tests/ptvsd/highlevel/test_messages.py

@@ -191,11 +191,13 @@ class ThreadsTests(NormalRequestTest, unittest.TestCase):
             received = self.vsc.received
 
         self.assert_vsc_received(received, [
+            self.new_event('thread', threadId=1, reason='started'),
+            self.new_event('thread', threadId=2, reason='started'),
             self.expected_response(
                 threads=[
                     {'id': 1, 'name': 'spam'},
                     # Threads named 'pydevd.*' are ignored.
-                    {'id': 3, 'name': ''},
+                    {'id': 2, 'name': ''},
                 ],
             ),
             # no events
@@ -584,14 +586,15 @@ class PauseTests(NormalRequestTest, unittest.TestCase):
     PYDEVD_RESP = None
 
     def test_pause_one(self):
+        thread = (10, 'spam')
         with self.launched():
             with self.hidden():
-                self.set_threads(
-                    (10, 'spam'),
+                tids = self.set_threads(
+                    thread,
                     (11, ''),
                 )
             self.send_request(
-                threadId=5,  # matches our first thread
+                threadId=tids[thread],
             )
             received = self.vsc.received
 
@@ -619,11 +622,11 @@ class ContinueTests(NormalRequestTest, unittest.TestCase):
         thread = (10, 'x')
         with self.launched():
             with self.hidden():
-                self.pause(thread, *[
+                tid = self.pause(thread, *[
                     (2, 'spam', 'abc.py', 10),
                 ])
             self.send_request(
-                threadId=5,  # matches our thread
+                threadId=tid,
             )
             received = self.vsc.received
 
@@ -646,11 +649,11 @@ class NextTests(NormalRequestTest, unittest.TestCase):
         thread = (10, 'x')
         with self.launched():
             with self.hidden():
-                self.pause(thread, *[
+                tid = self.pause(thread, *[
                     (2, 'spam', 'abc.py', 10),
                 ])
             self.send_request(
-                threadId=5,  # matches our thread
+                threadId=tid,
             )
             received = self.vsc.received
 
@@ -673,11 +676,11 @@ class StepInTests(NormalRequestTest, unittest.TestCase):
         thread = (10, 'x')
         with self.launched():
             with self.hidden():
-                self.pause(thread, *[
+                tid = self.pause(thread, *[
                     (2, 'spam', 'abc.py', 10),
                 ])
             self.send_request(
-                threadId=5,  # matches our thread
+                threadId=tid,
             )
             received = self.vsc.received
 
@@ -700,11 +703,11 @@ class StepOutTests(NormalRequestTest, unittest.TestCase):
         thread = (10, 'x')
         with self.launched():
             with self.hidden():
-                self.pause(thread, *[
+                tid = self.pause(thread, *[
                     (2, 'spam', 'abc.py', 10),
                 ])
             self.send_request(
-                threadId=5,  # matches our thread
+                threadId=tid,
             )
             received = self.vsc.received
 
@@ -1169,10 +1172,16 @@ class SetExceptionBreakpointsTests(NormalRequestTest, unittest.TestCase):
             self.expected_response(),
         ])
         self.PYDEVD_CMD = CMD_REMOVE_EXCEPTION_BREAK
-        removed = [
-            self.expected_pydevd_request('python-ImportError'),
-            self.expected_pydevd_request('python-BaseException'),
-        ]
+        if self.debugger.received[0].payload == 'python-ImportError':
+            removed = [
+                self.expected_pydevd_request('python-ImportError'),
+                self.expected_pydevd_request('python-BaseException'),
+            ]
+        else:
+            removed = [
+                self.expected_pydevd_request('python-BaseException'),
+                self.expected_pydevd_request('python-ImportError'),
+            ]
         self.PYDEVD_CMD = CMD_ADD_EXCEPTION_BREAK
         self.assert_received(self.debugger, removed + [
             self.expected_pydevd_request('python-ImportError\t0\t0\t0'),
@@ -1325,10 +1334,16 @@ class SetExceptionBreakpointsTests(NormalRequestTest, unittest.TestCase):
             self.expected_response(),
         ])
         self.PYDEVD_CMD = CMD_REMOVE_EXCEPTION_BREAK
-        removed = [
-            self.expected_pydevd_request('python-ImportError'),
-            self.expected_pydevd_request('python-BaseException'),
-        ]
+        if self.debugger.received[0].payload == 'python-ImportError':
+            removed = [
+                self.expected_pydevd_request('python-ImportError'),
+                self.expected_pydevd_request('python-BaseException'),
+            ]
+        else:
+            removed = [
+                self.expected_pydevd_request('python-BaseException'),
+                self.expected_pydevd_request('python-ImportError'),
+            ]
         self.PYDEVD_CMD = CMD_ADD_EXCEPTION_BREAK
         self.assert_received(self.debugger, removed + [
             self.expected_pydevd_request('python-BaseException\t3\t0\t0'),
@@ -1505,7 +1520,7 @@ class ThreadEventTest(PyDevdEventTest):
 
     def send_event(self, *args, **kwargs):
         def handler(msg, _):
-            self._tid = msg.data['body']['threadId']
+            self._tid = msg.body['threadId']
         kwargs['handler'] = handler
         super(ThreadEventTest, self).send_event(*args, **kwargs)
         return self._tid
@@ -1571,8 +1586,6 @@ class ThreadKillTests(ThreadEventTest, unittest.TestCase):
     def pydevd_payload(self, threadid):
         return str(threadid)
 
-    # TODO: https://github.com/Microsoft/ptvsd/issues/138
-    @unittest.skip('broken')
     def test_known(self):
         thread = (10, 'x')
         with self.launched():
@@ -1597,8 +1610,6 @@ class ThreadKillTests(ThreadEventTest, unittest.TestCase):
         self.assert_vsc_received(received, [])
         self.assert_received(self.debugger, [])
 
-    # TODO: https://github.com/Microsoft/ptvsd/issues/137
-    @unittest.skip('broken')
     def test_pydevd_name(self):
         thread = (10, 'pydevd.spam')
         with self.launched():
@@ -1610,8 +1621,6 @@ class ThreadKillTests(ThreadEventTest, unittest.TestCase):
         self.assert_vsc_received(received, [])
         self.assert_received(self.debugger, [])
 
-    # TODO: https://github.com/Microsoft/ptvsd/issues/137
-    @unittest.skip('broken')
     def test_ptvsd_name(self):
         thread = (10, 'ptvsd.spam')
         with self.launched():
@@ -1650,6 +1659,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='step',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1666,6 +1676,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='step',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1682,6 +1693,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='step',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1698,6 +1710,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='exception',
                 threadId=tid,
+                text='BaseException, exception: no description',
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1714,6 +1727,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='exception',
                 threadId=tid,
+                text='BaseException, exception: no description',
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1730,6 +1744,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='pause',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1747,6 +1762,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='pause',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1765,6 +1781,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='pause',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1784,6 +1801,7 @@ class ThreadSuspendTests(ThreadEventTest, unittest.TestCase):
             self.expected_event(
                 reason='pause',
                 threadId=tid,
+                text=None,
             ),
         ])
         self.assert_received(self.debugger, [])
@@ -1842,8 +1860,8 @@ class SendCurrExcTraceTests(PyDevdEventTest, unittest.TestCase):
 
         self.assert_vsc_received(received, [])
         self.assert_received(self.debugger, [])
-        self.assertTrue(resp.data['success'], resp.data['message'])
-        self.assertEqual(resp.data['body'], dict(
+        self.assertTrue(resp.success, resp.message)
+        self.assertEqual(resp.body, dict(
             exceptionId='RuntimeError',
             description='something went wrong',
             breakMode='unhandled',
@@ -1859,8 +1877,6 @@ class SendCurrExcTraceProceededTests(PyDevdEventTest, unittest.TestCase):
     CMD = CMD_SEND_CURR_EXCEPTION_TRACE_PROCEEDED
     EVENT = None
 
-    # See https://github.com/Microsoft/ptvsd/issues/141.
-
     def pydevd_payload(self, threadid):
         return str(threadid)
 
@@ -1871,10 +1887,24 @@ class SendCurrExcTraceProceededTests(PyDevdEventTest, unittest.TestCase):
         text = self.debugger_msgs.format_exception(thread[0], exc, frame)
         with self.launched():
             with self.hidden():
-                self.set_thread(thread)
+                tid = self.set_thread(thread)
                 self.fix.send_event(CMD_SEND_CURR_EXCEPTION_TRACE, text)
+                self.send_request('exceptionInfo', dict(
+                    threadId=tid,
+                ))
+                before = self.vsc.received[-1]
+
             self.send_event(10)
             received = self.vsc.received
 
+            self.send_request('exceptionInfo', dict(
+                threadId=tid,
+            ))
+            after = self.vsc.received[-1]
+
         self.assert_vsc_received(received, [])
         self.assert_received(self.debugger, [])
+        # The exception got cleared so we do not see RuntimeError.
+        self.assertEqual(after.body['exceptionId'], 'BaseException')
+        self.assertNotEqual(after.body['exceptionId'],
+                            before.body['exceptionId'])