From 105032c3ae3994837bfb2278cf86446188628b91 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Wed, 2 May 2018 18:05:13 -0600
Subject: [PATCH] Run most of the tests under Python 2. (#405)

---
 Makefile                            |  3 +-
 debugger_protocol/arg/_decl.py      |  5 +++-
 tests/__init__.py                   | 44 ++++++++++++++++++++++++++++-
 tests/__main__.py                   | 13 ++++-----
 tests/debugger_protocol/__init__.py |  6 ++--
 tests/helpers/counter.py            |  5 ++++
 tests/helpers/debugclient.py        |  2 ++
 tests/helpers/debugsession.py       |  9 +++---
 tests/helpers/http.py               | 11 +++++---
 tests/helpers/proc.py               |  6 +++-
 tests/helpers/protocol.py           |  5 ++--
 tests/helpers/pydevd/_binder.py     |  3 +-
 tests/helpers/pydevd/_live.py       |  3 +-
 tests/helpers/pydevd/_pydevd.py     |  7 ++++-
 tests/helpers/threading.py          |  7 +++--
 tests/helpers/vsc/_vsc.py           |  7 ++++-
 tests/highlevel/__init__.py         | 14 +++++----
 tests/system_tests/test_main.py     | 16 ++++++++---
 tests/system_tests/test_schema.py   |  2 ++
 tests/test_tests___main__.py        |  5 +---
 20 files changed, 127 insertions(+), 46 deletions(-)

diff --git a/Makefile b/Makefile
index 3c834f7d..1576165a 100644
--- a/Makefile
+++ b/Makefile
@@ -42,7 +42,8 @@ ci-lint: depends lint
 
 .PHONY: ci-test
 ci-test: depends
-	$(PYTHON) -m tests -v --full --no-network
+	# For now we use --quickpy2.
+	$(PYTHON) -m tests -v --full --no-network --quick-py2
 
 .PHONY: ci-coverage
 ci-coverage: depends
diff --git a/debugger_protocol/arg/_decl.py b/debugger_protocol/arg/_decl.py
index 1466c6d8..cad37ba6 100644
--- a/debugger_protocol/arg/_decl.py
+++ b/debugger_protocol/arg/_decl.py
@@ -1,5 +1,8 @@
 from collections import namedtuple
-from collections.abc import Sequence
+try:
+    from collections.abc import Sequence
+except ImportError:
+    from collections import Sequence
 
 from debugger_protocol._base import Readonly
 from ._common import sentinel, NOT_SET, ANY, SIMPLE_TYPES
diff --git a/tests/__init__.py b/tests/__init__.py
index e0c842c6..4322993f 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,3 +1,45 @@
+from __future__ import absolute_import
 
-# Trigger the pydevd vendoring.
+import os
+import os.path
+import sys
+import unittest
+
+# Importing "ptvsd" here triggers the vendoring code before any vendored
+# code ever gets imported.
 import ptvsd  # noqa
+from ptvsd._vendored import list_all as vendored
+
+
+TEST_ROOT = os.path.dirname(__file__)  # noqa
+PROJECT_ROOT = os.path.dirname(TEST_ROOT)  # noqa
+VENDORED_ROOTS = vendored(resolve=True)  # noqa
+
+
+def skip_py2(decorated=None):
+    if sys.version_info[0] > 2:
+        return decorated
+    msg = 'not tested under Python 2'
+    if decorated is None:
+        raise unittest.SkipTest(msg)
+    else:
+        decorator = unittest.skip(msg)
+    return decorator(decorated)
+
+
+if sys.version_info[0] == 2:
+    # Hack alert!!!
+    class SkippingTestSuite(unittest.TestSuite):
+        def __init__(self, tests=()):
+            if tests and type(tests[0]).__name__ == 'ModuleImportFailure':
+                _, exc, _ = sys.exc_info()
+                if isinstance(exc, unittest.SkipTest):
+                    from unittest.loader import _make_failed_load_tests
+                    suite = _make_failed_load_tests(
+                        tests[0]._testMethodName,
+                        exc,
+                        type(self),
+                    )
+                    tests = tuple(suite)
+            unittest.TestSuite.__init__(self, tests)
+    unittest.TestLoader.suiteClass = SkippingTestSuite
diff --git a/tests/__main__.py b/tests/__main__.py
index 4f82c7f0..a26c5667 100644
--- a/tests/__main__.py
+++ b/tests/__main__.py
@@ -6,17 +6,13 @@ import subprocess
 import sys
 import unittest
 
-from ptvsd._vendored import list_all as vendored
-
-
-TEST_ROOT = os.path.dirname(__file__)
-PROJECT_ROOT = os.path.dirname(TEST_ROOT)
-VENDORED_ROOTS = vendored(resolve=True)
+from . import TEST_ROOT, PROJECT_ROOT, VENDORED_ROOTS
 
 
 def convert_argv(argv):
     help  = False
     quick = False
+    quickpy2 = False
     network = True
     runtests = True
     lint = False
@@ -26,6 +22,9 @@ def convert_argv(argv):
         if arg == '--quick':
             quick = True
             continue
+        if arg == '--quick-py2':
+            quickpy2 = True
+            continue
         elif arg == '--full':
             quick = False
             continue
@@ -76,7 +75,7 @@ def convert_argv(argv):
             quickroot = os.path.join(TEST_ROOT, 'ptvsd')
             if quick:
                 start = quickroot
-            elif sys.version_info[0] != 3:
+            elif quickpy2 and sys.version_info[0] == 2:
                 start = quickroot
             else:
                 start = PROJECT_ROOT
diff --git a/tests/debugger_protocol/__init__.py b/tests/debugger_protocol/__init__.py
index 2efc3d52..65be4a92 100644
--- a/tests/debugger_protocol/__init__.py
+++ b/tests/debugger_protocol/__init__.py
@@ -1,9 +1,7 @@
-import sys
-import unittest
+from .. import skip_py2
 
 
 # The code under the debugger_protocol package isn't used
 # by the debugger (it's used by schema-related tools).  So we don't need
 # to support Python 2.
-if sys.version_info[0] == 2:
-    raise unittest.SkipTest('not tested under Python 2')
+skip_py2()
diff --git a/tests/helpers/counter.py b/tests/helpers/counter.py
index 2c4a3dd9..b0c74a3b 100644
--- a/tests/helpers/counter.py
+++ b/tests/helpers/counter.py
@@ -1,3 +1,5 @@
+import sys
+
 
 class Counter(object):
     """An introspectable, dynamic alternative to itertools.count()."""
@@ -23,6 +25,9 @@ class Counter(object):
             self._last = self._start
         return self._last
 
+    if sys.version_info[0] == 2:
+        next = __next__
+
     @property
     def start(self):
         return self._start
diff --git a/tests/helpers/debugclient.py b/tests/helpers/debugclient.py
index e8efaab7..64cce04a 100644
--- a/tests/helpers/debugclient.py
+++ b/tests/helpers/debugclient.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+
 import threading
 import warnings
 
diff --git a/tests/helpers/debugsession.py b/tests/helpers/debugsession.py
index 4f837f9f..588a3bfc 100644
--- a/tests/helpers/debugsession.py
+++ b/tests/helpers/debugsession.py
@@ -1,7 +1,8 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
 import contextlib
 import json
+import socket
 import sys
 import time
 import threading
@@ -34,10 +35,10 @@ class DebugSessionConnection(Closeable):
             for _ in range(int(timeout * 10)):
                 try:
                     sock.connect(addr)
-                except OSError:
+                except (OSError, socket.error):
                     if cls.VERBOSE:
                         print('+', end='')
-                    sys.stdout.flush()
+                        sys.stdout.flush()
                     time.sleep(0.1)
                 else:
                     break
@@ -87,7 +88,7 @@ class DebugSessionConnection(Closeable):
         read = recv_as_read(self._sock)
         for msg, _, _ in read_messages(read, stop=stop):
             if self.VERBOSE:
-                print(msg)
+                print(repr(msg))
             yield parse_message(msg)
 
     def send(self, req):
diff --git a/tests/helpers/http.py b/tests/helpers/http.py
index 04fcca61..5438af69 100644
--- a/tests/helpers/http.py
+++ b/tests/helpers/http.py
@@ -1,6 +1,9 @@
 from __future__ import absolute_import
 
-import http.server
+try:
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+except ImportError:
+    from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 import threading
 
 
@@ -23,7 +26,7 @@ class Server:
     def start(self):
         if self._server is not None:
             raise RuntimeError('already started')
-        self._server = http.server.HTTPServer(self._addr, self.handler)
+        self._server = HTTPServer(self._addr, self.handler)
         self._thread = threading.Thread(
                 target=lambda: self._server.serve_forever())
         self._thread.start()
@@ -48,7 +51,7 @@ class Server:
 def json_file_handler(data):
     """Return an HTTP handler that always serves the given JSON bytes."""
 
-    class HTTPHandler(http.server.BaseHTTPRequestHandler):
+    class HTTPHandler(BaseHTTPRequestHandler):
         def do_GET(self):
             self.send_response(200)
             self.send_header('Content-Type', b'application/json')
@@ -66,7 +69,7 @@ def json_file_handler(data):
 def error_handler(code, msg):
     """Return an HTTP handler that always returns the given error code."""
 
-    class HTTPHandler(http.server.BaseHTTPRequestHandler):
+    class HTTPHandler(BaseHTTPRequestHandler):
         def do_GET(self):
             self.send_error(code, msg)
 
diff --git a/tests/helpers/proc.py b/tests/helpers/proc.py
index 837682cc..f848ccf4 100644
--- a/tests/helpers/proc.py
+++ b/tests/helpers/proc.py
@@ -73,7 +73,11 @@ class Proc(Closeable):
 
     def _close(self):
         if self._proc is not None:
-            self._proc.kill()
+            try:
+                self._proc.kill()
+            except OSError:
+                # Already killed.
+                pass
         if self.VERBOSE:
             lines = self.output.decode('utf-8').splitlines()
             print(' + ' + '\n + '.join(lines))
diff --git a/tests/helpers/protocol.py b/tests/helpers/protocol.py
index e88a79cf..a8adf71e 100644
--- a/tests/helpers/protocol.py
+++ b/tests/helpers/protocol.py
@@ -8,6 +8,7 @@ import warnings
 
 from . import socket
 from .counter import Counter
+from .threading import acquire_with_timeout
 
 
 try:
@@ -259,7 +260,7 @@ class MessageDaemon(Daemon):
         yield req
 
         # Wait for the message to match.
-        if lock.acquire(timeout=timeout):
+        if acquire_with_timeout(lock, timeout=timeout):
             lock.release()
         else:
             msg = 'timed out after {} seconds waiting for message ({})'
@@ -278,7 +279,7 @@ class MessageDaemon(Daemon):
 
     def _listen(self):
         try:
-            with self._sock.makefile('rb') as sockfile:
+            with contextlib.closing(self._sock.makefile('rb')) as sockfile:
                 for msg in self._protocol.iter(sockfile, lambda: self._closed):
                     if isinstance(msg, StreamFailure):
                         self._failures.append(msg)
diff --git a/tests/helpers/pydevd/_binder.py b/tests/helpers/pydevd/_binder.py
index fa88daf4..64d021e9 100644
--- a/tests/helpers/pydevd/_binder.py
+++ b/tests/helpers/pydevd/_binder.py
@@ -3,6 +3,7 @@ import time
 
 import ptvsd.daemon
 from tests.helpers import socket
+from tests.helpers.threading import acquire_with_timeout
 
 
 class PTVSD(ptvsd.daemon.Daemon):
@@ -112,7 +113,7 @@ class BinderBase(object):
             self._thread = threading.Thread(target=self._run)
             self._thread.start()
             # Wait for ptvsd to start up.
-            if self._waiter.acquire(timeout=1):
+            if acquire_with_timeout(self._waiter, timeout=1):
                 self._waiter.release()
             else:
                 raise RuntimeError('timed out')
diff --git a/tests/helpers/pydevd/_live.py b/tests/helpers/pydevd/_live.py
index 5739ff8e..01673cea 100644
--- a/tests/helpers/pydevd/_live.py
+++ b/tests/helpers/pydevd/_live.py
@@ -5,6 +5,7 @@ import warnings
 
 import ptvsd._main
 from tests.helpers import protocol
+from tests.helpers.threading import acquire_with_timeout
 from ._binder import BinderBase
 
 
@@ -38,7 +39,7 @@ class Binder(BinderBase):
         )
 
         # Block until "done" debugging.
-        if not self._lock.acquire(timeout=3):
+        if not acquire_with_timeout(self._lock, timeout=3):
             # This shouldn't happen since the timeout on event waiting
             # is this long.
             warnings.warn('timeout out waiting for "done"')
diff --git a/tests/helpers/pydevd/_pydevd.py b/tests/helpers/pydevd/_pydevd.py
index 85537eb5..c9121ba0 100644
--- a/tests/helpers/pydevd/_pydevd.py
+++ b/tests/helpers/pydevd/_pydevd.py
@@ -1,4 +1,5 @@
 from collections import namedtuple
+import sys
 try:
     from urllib.parse import quote, unquote
 except ImportError:
@@ -11,6 +12,10 @@ from tests.helpers.protocol import StreamFailure
 # TODO: Everything here belongs in a proper pydevd package.
 
 
+if sys.version_info[0] > 2:
+    basestring = str
+
+
 def parse_message(msg):
     """Return a message object for the given "msg" data."""
     if type(msg) is bytes:
@@ -112,7 +117,7 @@ class Message(namedtuple('Message', 'cmdid seq payload')):
         """Return the de-serialized payload."""
         if isinstance(payload, bytes):
             payload = payload.decode('utf-8')
-        if isinstance(payload, str):
+        if isinstance(payload, basestring):
             text = unquote(payload)
             return cls._parse_payload_text(text)
         elif hasattr(payload, 'as_text'):
diff --git a/tests/helpers/threading.py b/tests/helpers/threading.py
index 155ed334..47e214bd 100644
--- a/tests/helpers/threading.py
+++ b/tests/helpers/threading.py
@@ -8,11 +8,12 @@ import warnings
 
 if sys.version_info < (3,):
     def acquire_with_timeout(lock, timeout):
-        segments = int(timeout * 10) + 1
-        for _ in range(segments):
+        if lock.acquire(False):
+            return True
+        for _ in range(int(timeout * 10)):
+            time.sleep(0.1)
             if lock.acquire(False):
                 return True
-            time.sleep(0.1)
         else:
             return False
 else:
diff --git a/tests/helpers/vsc/_vsc.py b/tests/helpers/vsc/_vsc.py
index 6aa5d6a1..05456aec 100644
--- a/tests/helpers/vsc/_vsc.py
+++ b/tests/helpers/vsc/_vsc.py
@@ -1,5 +1,6 @@
 from collections import namedtuple
 import json
+import sys
 
 from debugger_protocol.messages import wireformat
 from tests.helpers.protocol import StreamFailure
@@ -7,6 +8,10 @@ from tests.helpers.protocol import StreamFailure
 # TODO: Use more of the code from debugger_protocol.
 
 
+if sys.version_info[0] > 2:
+    unicode = str
+
+
 class ProtocolMessageError(Exception): pass  # noqa
 class MalformedMessageError(ProtocolMessageError): pass  # noqa
 class IncompleteMessageError(MalformedMessageError): pass  # noqa
@@ -15,7 +20,7 @@ class UnsupportedMessageTypeError(ProtocolMessageError): pass  # noqa
 
 def parse_message(msg):
     """Return a message object for the given "msg" data."""
-    if type(msg) is str:
+    if type(msg) is str or type(msg) is unicode:
         data = json.loads(msg)
     elif isinstance(msg, bytes):
         data = json.loads(msg.decode('utf-8'))
diff --git a/tests/highlevel/__init__.py b/tests/highlevel/__init__.py
index 39da3b7d..ea282318 100644
--- a/tests/highlevel/__init__.py
+++ b/tests/highlevel/__init__.py
@@ -477,17 +477,18 @@ class VSCFixture(FixtureBase):
         except AttributeError:
             return None
 
-    def send_request(self, command, args=None, handle_response=None):
+    def send_request(self, cmd, args=None, handle_response=None, timeout=1):
         kwargs = dict(args or {}, handler=handle_response)
-        with self._wait_for_response(command, **kwargs) as req:
+        with self._wait_for_response(cmd, timeout=timeout, **kwargs) as req:
             self.fake.send_request(req)
         return req
 
     @contextlib.contextmanager
     def _wait_for_response(self, command, *args, **kwargs):
-        handler = kwargs.pop('handler', None)
+        handle = kwargs.pop('handler', None)
+        timeout = kwargs.pop('timeout', 1)
         req = self.msgs.new_request(command, *args, **kwargs)
-        with self.fake.wait_for_response(req, handler=handler):
+        with self.fake.wait_for_response(req, handler=handle, timeout=timeout):
             yield req
         if self._hidden:
             self.msgs.next_response()
@@ -623,8 +624,9 @@ class HighlevelFixture(object):
             thread = self._pydevd.threads.add(name)
             self._default_threads[name] = thread
 
-    def send_request(self, command, args=None, handle_response=None):
-        return self._vsc.send_request(command, args, handle_response)
+    def send_request(self, command, args=None, handle_response=None, **kwargs):
+        return self._vsc.send_request(command, args, handle_response,
+                                      **kwargs)
 
     @contextlib.contextmanager
     def wait_for_event(self, event, *args, **kwargs):
diff --git a/tests/system_tests/test_main.py b/tests/system_tests/test_main.py
index 7d43c24b..93af31a1 100644
--- a/tests/system_tests/test_main.py
+++ b/tests/system_tests/test_main.py
@@ -9,6 +9,13 @@ from tests.helpers.vsc import parse_message, VSCMessages
 from tests.helpers.workspace import Workspace, PathEntry
 
 
+def _strip_pydevd_output(out):
+    # TODO: Leave relevant lines from before the marker?
+    pre, sep, out = out.partition(
+        'pydev debugger: starting' + os.linesep + os.linesep)
+    return out if sep else pre
+
+
 def lifecycle_handshake(session, command='launch', options=None):
     with session.wait_for_event('initialized'):
         req_initialize = session.send_request(
@@ -92,8 +99,8 @@ class CLITests(TestsBase, unittest.TestCase):
             session.send_request('disconnect')
         out = adapter.output
 
-        self.assertEqual(out.decode('utf-8'),
-                         "[{!r}, '--eggs']\n".format(filename))
+        self.assertEqual(out.decode('utf-8').strip().splitlines()[-1],
+                         u"[{!r}, '--eggs']".format(filename))
 
     def test_run_to_completion(self):
         filename = self.pathentry.write_module('spam', """
@@ -199,7 +206,7 @@ class LifecycleTests(TestsBase, unittest.TestCase):
                 timeout=3.0,
             )
             wait_for_started()
-        out = adapter.output
+        out = adapter.output.decode('utf-8')
 
         self.assert_received(session.received, [
             # TODO: Use self.new_event()...
@@ -216,7 +223,8 @@ class LifecycleTests(TestsBase, unittest.TestCase):
                 },
             },
         ])
-        self.assertEqual(out, b'')
+        out = _strip_pydevd_output(out)
+        self.assertEqual(out, '')
 
     def test_launch_ptvsd_client(self):
         argv = []
diff --git a/tests/system_tests/test_schema.py b/tests/system_tests/test_schema.py
index 1428250a..a645fb1a 100644
--- a/tests/system_tests/test_schema.py
+++ b/tests/system_tests/test_schema.py
@@ -7,6 +7,8 @@ import tempfile
 from textwrap import dedent
 import unittest
 
+from tests import skip_py2
+skip_py2()  # noqa
 from tests.helpers import http
 from debugger_protocol.schema.__main__ import handle_check
 
diff --git a/tests/test_tests___main__.py b/tests/test_tests___main__.py
index c46f2d16..29db88f5 100644
--- a/tests/test_tests___main__.py
+++ b/tests/test_tests___main__.py
@@ -3,13 +3,10 @@ import os.path
 import unittest
 import sys
 
+from . import TEST_ROOT, PROJECT_ROOT
 from .__main__ import convert_argv
 
 
-TEST_ROOT = os.path.dirname(__file__)
-PROJECT_ROOT = os.path.dirname(TEST_ROOT)
-
-
 class ConvertArgsTests(unittest.TestCase):
 
     def test_no_args(self):