Tests and CI refactoring:

Use YAML templates for setting up Python and running tests.

Use matrix instead of separate jobs for Windows 32-bit and 64-bit runs.

Change --debugpy-logs to --debugpy-log-dir with explicit destination, and use that to capture and publish test logs in CI.

Delete logs for passed tests by default, and add --debugpy-log-passed to opt out.

azure-pipelines/templates/run_tests.yml

@@ -0,0 +1,28 @@
+steps:
+    - script: |
+          python -m pip install -U pip setuptools tox
+      displayName: "Setup Python packages"
+
+    - pwsh: |
+          $toxEnv = '$(python.version)'
+          if (-not $toxEnv.startsWith('pypy')) {
+              $toxEnv = 'py' + $toxEnv.Replace('.', '')
+          }
+          echo 'tox environment: $toxEnv'
+          python -m tox -e $toxEnv -- -vv --junitxml=$(Build.ArtifactStagingDirectory)/tests.xml --debugpy-log-dir=$(Build.ArtifactStagingDirectory)/logs tests
+      displayName: "Run tests using tox"
+
+    - task: "PublishBuildArtifacts@1"
+      condition: "failed()"
+      inputs:
+          artifactName: "Test logs"
+          pathToPublish: "$(Build.ArtifactStagingDirectory)/logs"
+      displayName: "Publish test logs"
+
+    - task: "PublishTestResults@2"
+      condition: "always()"
+      inputs:
+          testRunTitle: "$(Agent.JobName)"
+          testResultsFiles: "tests.xml"
+          searchFolder: "$(Build.ArtifactStagingDirectory)"
+      displayName: "Publish test results"

azure-pipelines/templates/use_python.yml

@@ -0,0 +1,6 @@
+steps:
+    - task: UsePythonVersion@0
+      inputs:
+          versionSpec: "$(python.version)"
+          architecture: "$(architecture)"
+      displayName: "Use Python $(python.version) $(architecture)"

tests.yaml

@@ -1,232 +1,115 @@
+variables:
+    architecture: "x64"
+
 jobs:
-  - job: 'Lint'
-    displayName: 'Lint'
-    pool:
-      vmImage: 'ubuntu-16.04'
-    steps:
-    - task: UsePythonVersion@0
-      displayName: 'Use Python 3.8'
-      inputs:
-        versionSpec: '>=3.8'
+    - job: "Lint"
+      displayName: "Lint"
+      pool: { vmImage: "ubuntu-16.04" }
 
-    - script: 'python3 -m pip install -U pip setuptools tox flake8'
-      displayName: 'Setup Python packages'
+      variables:
+          python.version: "3.8"
 
-    - script: 'python3 -m pip install git+https://github.com/karthiknadig/flake8reports'
-      displayName: 'Acquire flake8 plugin flake8reports'
+      steps:
+          - template: "azure-pipelines/templates/use_python.yml"
 
-    - script: 'python3 -m flake8 --format=junit --output-file=$(Build.ArtifactStagingDirectory)/linter-test.xml'
-      displayName: 'Run flake8'
+          - script: |
+                python3 -m pip install -U flake8
+                python3 -m pip install -U git+https://github.com/karthiknadig/flake8reports
+            displayName: "Setup flake8"
 
-    - task: PublishTestResults@2
-      displayName: 'Publish linting results'
-      inputs:
-        testResultsFiles: '**/linter-test.xml'
-        searchFolder: '$(Build.ArtifactStagingDirectory)'
-        testRunTitle: 'Linux-Linter-$(Build.BuildNumber)'
-        buildPlatform: Linux
-        buildConfiguration: Lint
-      condition: always()
+          - script: |
+                python3 -m flake8 --format=junit --output-file=$(Build.ArtifactStagingDirectory)/lint-flake8.xml
+            displayName: "Run flake8"
 
-  - job: 'TestsLinux'
-    displayName: 'Linux test run'
-    pool:
-      vmImage: 'ubuntu-16.04'
-    strategy:
-      matrix:
-        Python27:
-          python.version: '2.7'
-        Python35:
-          python.version: '3.5'
-        Python36:
-          python.version: '3.6'
-        Python37:
-          python.version: '3.7'
-        Python38:
-          python.version: '3.8'
-      maxParallel: 5
+          - task: "PublishTestResults@2"
+            condition: "always()"
+            inputs:
+                testRunTitle: "$(Agent.JobName)"
+                testResultsFiles: "lint-*.xml"
+                searchFolder: "$(Build.ArtifactStagingDirectory)"
+            displayName: "Publish linting results"
 
-    steps:
-    - script: |
-          sudo apt-get --yes install gdb
-          sudo sysctl kernel.yama.ptrace_scope=0
+    - job: "Test_Linux"
+      displayName: "Tests - Linux"
+      pool: { vmImage: "ubuntu-16.04" }
 
-      displayName: 'install gdb'
+      strategy:
+          matrix:
+              py27:
+                  python.version: "2.7"
+              py35:
+                  python.version: "3.5"
+              py36:
+                  python.version: "3.6"
+              py37:
+                  python.version: "3.7"
+              py38:
+                  python.version: "3.8"
 
-    - task: UsePythonVersion@0
-      displayName: 'Use Python $(python.version)'
-      inputs:
-        versionSpec: '$(python.version)'
-        architecture: 'x64'
+      steps:
+          - script: |
+                sudo apt-get --yes install gdb
+                sudo sysctl kernel.yama.ptrace_scope=0
+            displayName: "Setup gdb"
 
-    - script: 'python -m pip install -U pip setuptools tox'
-      displayName: 'Setup Python packages'
+          - template: "azure-pipelines/templates/use_python.yml"
 
-    - powershell: |
-          $toxEnv = '$(python.version)'
-            if (-not $toxEnv.startsWith('pypy')) {
-                $toxEnv = 'py' + $toxEnv.Replace('.', '')
-            }
-            echo "tox environment: $toxEnv"
-          python -m tox -e $toxEnv -- -vv --junitxml=$(Build.ArtifactStagingDirectory)/junit-test-report-linux-pytest.xml
+          - template: "azure-pipelines/templates/run_tests.yml"
 
-      displayName: 'Run tests'
+    - job: "Test_MacOS"
+      displayName: "Tests - macOS"
+      pool: { vmImage: "macOS-10.13" }
 
-    - task: PublishTestResults@2
-      displayName: 'Publish test results'
-      inputs:
-        testResultsFiles: '**/junit-test-report-*.xml'
-        searchFolder: '$(Build.ArtifactStagingDirectory)'
-        testRunTitle: 'Linux-Unit-Py$(python.version)-$(Build.BuildNumber)'
-        buildPlatform: Linux
-        buildConfiguration: UnitTest
-      condition: always()
+      strategy:
+          matrix:
+              py27:
+                  python.version: "2.7"
+              py35:
+                  python.version: "3.5"
+              py36:
+                  python.version: "3.6"
+              py37:
+                  python.version: "3.7"
+              py38:
+                  python.version: "3.8"
 
-  - job: 'TestsMacOS'
-    displayName: 'macOS test run'
-    pool:
-      vmImage: 'macOS-10.13'
-    strategy:
-      matrix:
-        Python27:
-          python.version: '2.7'
-        Python35:
-          python.version: '3.5'
-        Python36:
-          python.version: '3.6'
-        Python37:
-          python.version: '3.7'
-        Python38:
-          python.version: '3.8'
-      maxParallel: 5
+      steps:
+          - script: |
+                ulimit -Sn 8192
+            displayName: "Increase file descriptor limit"
 
-    steps:
-      - script: |
-            ulimit -Sn 8192
-        displayName: 'Increase file descriptor limit'
+          - template: "azure-pipelines/templates/use_python.yml"
 
-      - task: UsePythonVersion@0
-        displayName: 'Use Python $(python.version)'
-        inputs:
-          versionSpec: '$(python.version)'
-          architecture: 'x64'
+          - script: |
+                python -m ensurepip --user
+            displayName: "Bootstrap pip"
 
-      - script: |
-            python -m ensurepip --user
-            python -m pip install --user --upgrade pip
-            python -m pip install --user setuptools tox
+          - template: "azure-pipelines/templates/run_tests.yml"
 
-        displayName: 'Install Python packages'
+    - job: "Test_Windows"
+      displayName: "Tests - Windows"
+      pool: { vmImage: "vs2017-win2016" }
 
-      - powershell: |
-            $toxEnv = '$(python.version)'
-            if (-not $toxEnv.startsWith('pypy')) {
-                $toxEnv = 'py' + $toxEnv.Replace('.', '')
-            }
-            echo "tox environment: $toxEnv"
-            python -m tox -e $toxEnv -- -vv --junitxml=$(Build.ArtifactStagingDirectory)/junit-test-report-mac-pytest.xml
+      strategy:
+          matrix:
+              py27:
+                  python.version: "2.7"
+              py35:
+                  python.version: "3.5"
+              py36:
+                  python.version: "3.6"
+              py37:
+                  python.version: "3.7"
+              py38:
+                  python.version: "3.8"
+              py27_32:
+                  python.version: "2.7"
+                  architecture: "x86"
+              py38_32:
+                  python.version: "3.8"
+                  architecture: "x86"
 
-        displayName: 'Run tests'
+      steps:
+          - template: "azure-pipelines/templates/use_python.yml"
 
-      - task: PublishTestResults@2
-        displayName: 'Publish test results'
-        inputs:
-          testResultsFiles: '**/junit-test-report-*.xml'
-          searchFolder: '$(Build.ArtifactStagingDirectory)'
-          testRunTitle: 'macOS-Unit-Py$(python.version)-$(Build.BuildNumber)'
-          buildPlatform: macOS
-          buildConfiguration: UnitTest
-        condition: always()
-
-
-  - job: 'TestsWin64'
-    displayName: 'Windows (64-bit) test run'
-    pool:
-      vmImage: 'vs2017-win2016'
-    strategy:
-      matrix:
-        Python27:
-          python.version: '2.7'
-        Python35:
-          python.version: '3.5'
-        Python36:
-          python.version: '3.6'
-        Python37:
-          python.version: '3.7'
-        Python38:
-          python.version: '3.8'
-      maxParallel: 5
-
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python $(python.version)'
-        inputs:
-          versionSpec: '$(python.version)'
-          architecture: 'x64'
-
-      - script: 'python -m pip install -U pip setuptools tox'
-        displayName: 'Setup Python packages'
-
-      - powershell: |
-            $toxEnv = '$(python.version)'
-            if (-not $toxEnv.startsWith('pypy')) {
-                $toxEnv = 'py' + $toxEnv.Replace('.', '')
-            }
-            echo "tox environment: $toxEnv"
-            python -m tox -e $toxEnv -- -vv --junitxml=$(Build.ArtifactStagingDirectory)/junit-test-report-win-pytest.xml
-        displayName: 'Run tests'
-
-      - task: PublishTestResults@2
-        displayName: 'Publish test results'
-        inputs:
-
-          testResultsFiles: '**/junit-test-report-*.xml'
-          searchFolder: '$(Build.ArtifactStagingDirectory)'
-          testRunTitle: 'Win-Unit-Py$(python.version)-$(Build.BuildNumber)'
-          buildPlatform: Windows
-          buildConfiguration: UnitTest
-        condition: always()
-
-
-  - job: 'TestsWin32'
-    displayName: 'Windows (32-bit) test run'
-    pool:
-      vmImage: 'vs2017-win2016'
-
-    strategy:
-      matrix:
-        Python27:
-          python.version: '2.7'
-        Python38:
-          python.version: '3.8'
-      maxParallel: 2
-
-    steps:
-      - task: UsePythonVersion@0
-        displayName: 'Use Python $(python.version)'
-        inputs:
-          versionSpec: '$(python.version)'
-          architecture: x86
-
-      - script: 'python -m pip install -U pip setuptools tox'
-        displayName: 'Setup Python packages'
-
-      - powershell: |
-            $toxEnv = '$(python.version)'
-            if (-not $toxEnv.startsWith('pypy')) {
-                $toxEnv = 'py' + $toxEnv.Replace('.', '')
-            }
-            echo "tox environment: $toxEnv"
-            python -m tox -e $toxEnv -- -vv --junitxml=$(Build.ArtifactStagingDirectory)/junit-test-report-win-pytest.xml
-        displayName: 'Run tests'
-
-      - task: PublishTestResults@2
-        displayName: 'Publish test results'
-        inputs:
-
-          testResultsFiles: '**/junit-test-report-*.xml'
-          searchFolder: '$(Build.ArtifactStagingDirectory)'
-          testRunTitle: 'Win32-Unit-Py$(python.version)-$(Build.BuildNumber)'
-          buildPlatform: Windows
-          buildConfiguration: UnitTest
-        condition: always()
+          - template: "azure-pipelines/templates/run_tests.yml"

tests/_logs/README

@@ -1,7 +1,4 @@
-Pass --debugpy-logs and/or --pydevd-logs to pytest to create log files here for a run.
-For example:
-
-    tox -e py37 -- --debugpy-logs "tests/ptvsd/server/test_run.py::test_run[launch-file]"
+This directory will contain debugpy and pydevd log files from the test runs.
 
 A separate directory will be created for every test. In that directory, a separate
 subdirectory will be created for every debug.Session instance, containing debugpy logs

tests/debugpy/test_attach.py

@@ -135,6 +135,8 @@ def test_reattach(pyfile, target, run):
         session2.scratchpad["exit"] = True
         session2.request_continue()
 
+    session1.wait_for_exit()
+
 
 @pytest.mark.parametrize("pid_type", ["int", "str"])
 def test_attach_by_pid(pyfile, target, pid_type):

tests/pytest_fixtures.py

@@ -50,6 +50,7 @@ def test_wrapper(request, long_tmpdir):
     session.Session.tmpdir = long_tmpdir
     original_log_dir = log.log_dir
 
+    failed = True
     try:
         if log.log_dir is None:
             log.log_dir = (long_tmpdir / "debugpy_logs").strpath
@@ -95,7 +96,13 @@ def test_wrapper(request, long_tmpdir):
                 if failed:
                     write_log("FAILED.log", "")
                     logs.dump()
+
     finally:
+        if not failed and not request.config.option.debugpy_log_passed:
+            try:
+                py.path.local(log.log_dir).remove()
+            except Exception:
+                pass
         log.log_dir = original_log_dir
 
 

tests/pytest_hooks.py

@@ -10,23 +10,31 @@ import pytest_timeout
 import sys
 
 from debugpy.common import fmt, log
+import tests
 from tests import logs
 
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--debugpy-logs",
+        "--debugpy-log-dir",
+        type=str,
+        help="Write debugpy and pydevd logs to the specified directory",
+    )
+    parser.addoption(
+        "--debugpy-log-passed",
         action="store_true",
-        help="Write debugpy and pydevd logs under {rootdir}/tests/_logs/",
+        help="Keep debugpy and pydevd logs for tests that passed",
     )
 
 
 def pytest_configure(config):
-    if config.option.debugpy_logs:
+    if config.option.debugpy_log_dir:
+        log.log_dir = config.option.debugpy_log_dir
+    else:
         bits = 64 if sys.maxsize > 2 ** 32 else 32
         ver = fmt("{0}.{1}-{bits}", *sys.version_info, bits=bits)
-        log.log_dir = (config.rootdir / "tests" / "_logs" / ver).strpath
-        log.info("debugpy and pydevd logs will be under {0}", log.log_dir)
+        log.log_dir = (tests.root / "_logs" / ver).strpath
+    log.info("debugpy and pydevd logs will be under {0}", log.log_dir)
 
 
 def pytest_report_header(config):