Use pytest-xdist to speed up tests and continue running after timeout. (#1116)

* Use pytest-xdist to speed up tests and continue running after timeout.

* Use unique server ports for Django and Flask tests.

pytest.ini

@@ -2,3 +2,4 @@
 testpaths=tests
 timeout=15
 timeout_method=thread
+addopts=-n8

test_requirements.txt

@@ -8,4 +8,5 @@ pygments
 pytest
 pytest-cov
 pytest-timeout
+pytest-xdist
 tox

tests/func/test_django.py

@@ -8,6 +8,7 @@ import os.path
 import pytest
 import sys
 
+import tests.helpers
 from tests.helpers.pattern import ANY, Path
 from tests.helpers.session import DebugSession
 from tests.helpers.timeline import Event
@@ -17,8 +18,8 @@ from tests.helpers.webhelper import get_url_from_str, get_web_content, wait_for_
 DJANGO1_ROOT = get_test_root('django1')
 DJANGO1_MANAGE = os.path.join(DJANGO1_ROOT, 'app.py')
 DJANGO1_TEMPLATE = os.path.join(DJANGO1_ROOT, 'templates', 'hello.html')
-DJANGO_LINK = 'http://127.0.0.1:8000/'
-DJANGO_PORT = 8000
+DJANGO_PORT = tests.helpers.get_unique_port(8000)
+DJANGO_LINK = 'http://127.0.0.1:{}/'.format(DJANGO_PORT)
 
 
 @pytest.mark.parametrize('bp_target', ['code', 'template'])
@@ -35,7 +36,7 @@ def test_django_breakpoint_no_multiproc(bp_target, start_method):
         session.initialize(
             start_method=start_method,
             target=('file', DJANGO1_MANAGE),
-            program_args=['runserver', '--noreload', '--nothreading'],
+            program_args=['runserver', '--noreload', '--', str(DJANGO_PORT)],
             debug_options=['Django'],
             cwd=DJANGO1_ROOT,
             expected_returncode=ANY.int,  # No clean way to kill Flask server
@@ -116,7 +117,7 @@ def test_django_exception_no_multiproc(ex_type, start_method):
         session.initialize(
             start_method=start_method,
             target=('file', DJANGO1_MANAGE),
-            program_args=['runserver', '--noreload', '--nothreading'],
+            program_args=['runserver', '--noreload', '--nothreading', str(DJANGO_PORT)],
             debug_options=['Django'],
             cwd=DJANGO1_ROOT,
             expected_returncode=ANY.int,  # No clean way to kill Flask server

tests/func/test_flask.py

@@ -9,6 +9,7 @@ import platform
 import pytest
 import sys
 
+import tests.helpers
 from tests.helpers.pattern import ANY, Path
 from tests.helpers.session import DebugSession
 from tests.helpers.timeline import Event
@@ -19,8 +20,8 @@ from tests.helpers.pathutils import get_test_root
 FLASK1_ROOT = get_test_root('flask1')
 FLASK1_APP = os.path.join(FLASK1_ROOT, 'app.py')
 FLASK1_TEMPLATE = os.path.join(FLASK1_ROOT, 'templates', 'hello.html')
-FLASK_LINK = 'http://127.0.0.1:5000/'
-FLASK_PORT = 5000
+FLASK_PORT = tests.helpers.get_unique_port(5000)
+FLASK_LINK = 'http://127.0.0.1:{}/'.format(FLASK_PORT)
 
 
 def _initialize_flask_session_no_multiproc(session, start_method):
@@ -39,7 +40,7 @@ def _initialize_flask_session_no_multiproc(session, start_method):
     session.initialize(
         start_method=start_method,
         target=('module', 'flask'),
-        program_args=['run', '--no-debugger', '--no-reload', '--with-threads'],
+        program_args=['run', '--no-debugger', '--no-reload', '--with-threads', '--port', str(FLASK_PORT)],
         ignore_unobserved=[Event('stopped'), Event('continued')],
         debug_options=['Jinja'],
         cwd=FLASK1_ROOT,
@@ -219,7 +220,7 @@ def test_flask_breakpoint_multiproc(start_method):
             start_method=start_method,
             target=('module', 'flask'),
             multiprocess=True,
-            program_args=['run', ],
+            program_args=['run', '--port', str(FLASK_PORT)],
             ignore_unobserved=[Event('stopped'), Event('continued')],
             debug_options=['Jinja'],
             cwd=FLASK1_ROOT,

tests/helpers/__init__.py

@@ -56,4 +56,15 @@ def dump_stacks_in(secs):
     thread.start()
 
 
+def get_unique_port(base):
+    # Different worker processes need to use different ports,
+    # for those scenarios where one is specified explicitly.
+    try:
+        worker_id = os.environ['PYTEST_XDIST_WORKER']
+        n = int(worker_id[2:])  # e.g. 'gw123'
+    except KeyError:
+        n = 0
+    return base + n
+
+
 from .printer import print

tests/helpers/session.py

@@ -4,6 +4,7 @@
 
 from __future__ import print_function, with_statement, absolute_import
 
+from collections import namedtuple
 import os
 import psutil
 import pytest
@@ -18,14 +19,15 @@ import ptvsd
 import ptvsd.__main__
 from ptvsd.messaging import JsonIOStream, JsonMessageChannel, MessageHandlers
 
+import tests.helpers
 from . import colors, debuggee, print, watchdog
 from .messaging import LoggingJsonStream
 from .pattern import ANY
 from .printer import wait_for_output
 from .timeline import Timeline, Event, Response
-from collections import namedtuple
 
 
+PTVSD_PORT = tests.helpers.get_unique_port(5678)
 PTVSD_ENABLE_KEY = 'PTVSD_ENABLE_ATTACH'
 PTVSD_HOST_KEY = 'PTVSD_TEST_HOST'
 PTVSD_PORT_KEY = 'PTVSD_TEST_PORT'
@@ -45,7 +47,7 @@ class DebugSession(object):
 
         self.target = ('code', 'print("OK")')
         self.start_method = start_method
-        self.ptvsd_port = ptvsd_port or 5678
+        self.ptvsd_port = ptvsd_port or PTVSD_PORT
         self.multiprocess = False
         self.multiprocess_port_range = None
         self.debug_options = ['RedirectOutput']