Update for VSTS-based CI process (#431)

* Add flake8 configuration files - .flake8.ci: addendum config for CI to make use of only. * Add build badge for CI in VSTS * Enable JUnit style test file output. - convert_args return value simplified - allow original style of unittest reporting as well - remove unnecessary flags from junit xmlrunner * Use argparse to collect runtime configuration of tests module
2025-12-23 08:48:12 +00:00 · 2018-06-21 17:09:53 -07:00 · 2018-06-21 17:09:53 -07:00 · e80a669624
commit e80a669624
parent ac8faf9dfe
9 changed files with 203 additions and 98 deletions
--- a/.flake8
+++ b/.flake8
@ -0,0 +1,5 @@
+[flake8]
+ignore = E24,E121,E123,E125,E126,E221,E226,E266,E704,E265
+exclude = 
+    ptvsd/_vendored/pydevd,
+    ./versioneer.py
--- a/.flake8.ci
+++ b/.flake8.ci
@ -0,0 +1,3 @@
+[flake8]
+format = junit-xml
+output-file = .\linter-test.xml
--- a/2
+++ b/2
@ -9,6 +9,8 @@ help:  ## Print help about available targets.
 depends:
 	$(PYTHON) -m pip install setuptools
 	$(PYTHON) -m pip install flake8
+	$(PYTHON) -m pip install flake8_formatter_junit_xml
+	$(PYTHON) -m pip install unittest-xml-reporting
 	$(PYTHON) -m pip install coverage

 .PHONY: lint
--- a/README.md
+++ b/README.md
@ -1,5 +1,6 @@
 # Python Tools for Visual Studio debug server

+[![Build Status](https://ptvsd.visualstudio.com/_apis/public/build/definitions/557bd35a-f98d-4c49-9bc9-c7d548f78e4d/1/badge)](https://ptvsd.visualstudio.com/ptvsd/ptvsd%20Team/_build/index?definitionId=1)
 [![Build Status](https://travis-ci.org/Microsoft/ptvsd.svg?branch=master)](https://travis-ci.org/Microsoft/ptvsd)

 ## Contributing
--- a/debugger_protocol/schema/UPSTREAM
+++ b/debugger_protocol/schema/UPSTREAM
@ -1,4 +1,4 @@
-upstream:   https://github.com/Microsoft/vscode-debugadapter-node/raw/master/debugProtocol.json
-revision:   ba5b70d4ea58fe56af226d56dd6dfeea7104eee7
-checksum:   6e26545484725c8d893a9c9a74d97b5d
-downloaded: 2018-03-21 20:14:21 (UTC)
+upstream:   https://github.com/Microsoft/vscode-debugadapter-node/raw/master/debugProtocol.json
+revision:   b97d2db700b792114b885fedc4c1ad70a9059faa
+checksum:   63f3a11ca49d4bc1bcc70b25d3918791
+downloaded: 2018-05-07 22:52:29 (UTC)
--- a/debugger_protocol/schema/debugProtocol.json
+++ b/debugger_protocol/schema/debugProtocol.json
@ -2454,7 +2454,7 @@
 				},
 				"path": {
 					"type": "string",
-					"description": "The path of the source to be shown in the UI. It is only used to locate and load the content of the source if no sourceReference is specified (or its vaule is 0)."
+					"description": "The path of the source to be shown in the UI. It is only used to locate and load the content of the source if no sourceReference is specified (or its value is 0)."
 				},
 				"sourceReference": {
 					"type": "number",
--- a/tests/main.py
+++ b/tests/main.py
@ -1,50 +1,105 @@
 from __future__ import absolute_import

+import argparse
 import os
 import os.path
 import subprocess
 import sys
 import unittest

+from xmlrunner import XMLTestRunner
+
 from . import TEST_ROOT, PROJECT_ROOT, VENDORED_ROOTS


-def convert_argv(argv):
-    help  = False
-    quick = False
-    quickpy2 = False
-    network = True
-    runtests = True
-    lint = False
-    args = []
-    modules = set()
-    for arg in argv:
-        if arg == '--quick':
-            quick = True
-            continue
-        if arg == '--quick-py2':
-            quickpy2 = True
-            continue
-        elif arg == '--full':
-            quick = False
-            continue
-        elif arg == '--network':
-            network = True
-            continue
-        elif arg == '--no-network':
-            network = False
-            continue
-        elif arg == '--coverage':
-            runtests = 'coverage'
-            continue
-        elif arg == '--lint':
-            lint = True
-            continue
-        elif arg == '--lint-only':
-            lint = True
-            runtests = False
-            break
+def parse_cmdline(argv=None):
+    """Obtain command line arguments and setup the test run accordingly."""

+    parser = argparse.ArgumentParser(
+        description="Run tests associated to the PTVSD project.",
+        prog="tests",
+        usage="python -m %(prog)s OPTS",
+        add_help=False
+    )
+
+    # allow_abbrev was added in 3.5
+    if sys.version_info >= (3, 5):
+        parser.allow_abbrev = False
+
+    parser.add_argument(
+        "-c",
+        "--coverage",
+        help="Generate code coverage report.",
+        action="store_true"
+    )
+    parser.add_argument(
+        "--full",
+        help="Do full suite of tests (disables prior --quick options).",
+        action="store_false",
+        dest="quick"
+    )
+    parser.add_argument(
+        "-j",
+        "--junit-xml",
+        help="Output report is generated to JUnit-style XML file specified.",
+        type=str
+    )
+    parser.add_argument(
+        "-l",
+        "--lint",
+        help="Run and report on Linter compliance.",
+        action="store_true"
+    )
+    parser.add_argument(
+        "-L",
+        "--lint-only",
+        help="Run and report on Linter compliance only, do not perform tests.",
+        action="store_true"
+    )
+    parser.add_argument(
+        "-n",
+        "--network",
+        help="Perform tests taht require network connectivity.",
+        action="store_true",
+        dest="network"
+    )
+    parser.add_argument(
+        "--no-network",
+        help="Do not perform tests that require network connectivity.",
+        action="store_false",
+        dest="network"
+    )
+    parser.add_argument(
+        "-q",
+        "--quick",
+        help="Only do the tests under test/ptvsd.",
+        action="store_true",
+        dest="quick"
+    )
+    parser.add_argument(
+        "--quick-py2",
+        help=("Only do the tests under test/ptvsd, that are compatible "
+              "with Python 2.x."),
+        action="store_true"
+    )
+    # these destinations have 2 switches, be explicit about the default
+    parser.set_defaults(quick=False)
+    parser.set_defaults(network=True)
+    config, passthrough_args = parser.parse_known_args(argv)
+
+    return config, passthrough_args
+
+
+def convert_argv(argv=None):
+    """Convert commandling args into unittest/linter/coverage input."""
+
+    config, passthru = parse_cmdline(argv)
+
+    modules = set()
+    args = []
+    help = False
+
+    for arg in passthru:
        # Unittest's main has only flags and positional args.
        # So we don't worry about options with values.
        if not arg.startswith('-'):
@ -63,31 +118,30 @@ def convert_argv(argv):
            help = True
        args.append(arg)

-    if runtests:
-        env = {}
-        if network:
-            env['HAS_NETWORK'] = '1'
-        # We make the "executable" a single arg because unittest.main()
-        # doesn't work if we split it into 3 parts.
-        cmd = [sys.executable + ' -m unittest']
-        if not modules and not help:
-            # Do discovery.
-            quickroot = os.path.join(TEST_ROOT, 'ptvsd')
-            if quick:
-                start = quickroot
-            elif quickpy2 and sys.version_info[0] == 2:
-                start = quickroot
-            else:
-                start = PROJECT_ROOT
-            cmd += [
-                'discover',
-                '--top-level-directory', PROJECT_ROOT,
-                '--start-directory', start,
-            ]
-        args = cmd + args
-    else:
-        args = env = None
-    return args, env, runtests, lint
+    env = {}
+    if config.network:
+        env['HAS_NETWORK'] = '1'
+    # We make the "executable" a single arg because unittest.main()
+    # doesn't work if we split it into 3 parts.
+    cmd = [sys.executable + ' -m unittest']
+    if not modules and not help:
+        # Do discovery.
+        quickroot = os.path.join(TEST_ROOT, 'ptvsd')
+        if config.quick:
+            start = quickroot
+        elif config.quick_py2 and sys.version_info[0] == 2:
+            start = quickroot
+        else:
+            start = PROJECT_ROOT
+
+        cmd += [
+            'discover',
+            '--top-level-directory', PROJECT_ROOT,
+            '--start-directory', start,
+        ]
+    args = cmd + args
+
+    return config, args, env


 def fix_sys_path():
@ -101,8 +155,7 @@ def check_lint():
    args = [
        sys.executable,
        '-m', 'flake8',
-        '--ignore', 'E24,E121,E123,E125,E126,E221,E226,E266,E704,E265',
-        '--exclude', ','.join(VENDORED_ROOTS),
+        '--config', '.flake8',
        PROJECT_ROOT,
    ]
    rc = subprocess.call(args)
@ -112,7 +165,7 @@ def check_lint():
    print('...done')


-def run_tests(argv, env, coverage=False):
+def run_tests(argv, env, coverage, junit_xml):
    print('running tests...')
    if coverage:
        omissions = [os.path.join(root, '*') for root in VENDORED_ROOTS]
@ -136,22 +189,34 @@ def run_tests(argv, env, coverage=False):
            print('...coverage failed!')
            sys.exit(rc)
        print('...done')
+    elif junit_xml:
+        os.environ.update(env)
+        with open(junit_xml, 'wb') as output:
+            unittest.main(
+                testRunner=XMLTestRunner(output=output),
+                module=None,
+                argv=argv
+            )
    else:
        os.environ.update(env)
        unittest.main(module=None, argv=argv)


 if __name__ == '__main__':
-    argv, env, runtests, lint = convert_argv(sys.argv[1:])
+    config, argv, env = convert_argv()
    fix_sys_path()
-    if lint:
+
+    if config.lint or config.lint_only:
        check_lint()
-    if runtests:
+
+    if not config.lint_only:
        if '--start-directory' in argv:
            start = argv[argv.index('--start-directory') + 1]
            print('(will look for tests under {})'.format(start))
+
        run_tests(
            argv,
            env,
-            coverage=(runtests == 'coverage'),
+            config.coverage,
+            config.junit_xml
        )
--- a/tests/system_tests/test_main.py
+++ b/tests/system_tests/test_main.py
@ -351,6 +351,7 @@ class LifecycleTests(TestsBase, unittest.TestCase):
        out = _strip_pydevd_output(out)
        self.assertEqual(out, '')

+    @unittest.skipUnless(os.environ.get('HAS_NETWORK'), 'no network')
    def test_launch_ptvsd_client(self):
        argv = []
        lockfile = self.workspace.lockfile()
--- a/tests/test_tests_main.py
+++ b/tests/test_tests_main.py
@ -10,7 +10,7 @@ from .__main__ import convert_argv
 class ConvertArgsTests(unittest.TestCase):

    def test_no_args(self):
-        argv, env, runtests, lint = convert_argv([])
+        config, argv, env = convert_argv([])

        self.assertEqual(argv, [
            sys.executable + ' -m unittest',
@ -21,11 +21,11 @@ class ConvertArgsTests(unittest.TestCase):
        self.assertEqual(env, {
            'HAS_NETWORK': '1',
        })
-        self.assertTrue(runtests)
-        self.assertFalse(lint)
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)

    def test_discovery_full(self):
-        argv, env, runtests, lint = convert_argv([
+        config, argv, env = convert_argv([
            '-v', '--failfast', '--full',
        ])

@ -39,11 +39,11 @@ class ConvertArgsTests(unittest.TestCase):
        self.assertEqual(env, {
            'HAS_NETWORK': '1',
        })
-        self.assertTrue(runtests)
-        self.assertFalse(lint)
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)

    def test_discovery_quick(self):
-        argv, env, runtests, lint = convert_argv([
+        config, argv, env = convert_argv([
            '-v', '--failfast', '--quick',
        ])

@ -57,11 +57,11 @@ class ConvertArgsTests(unittest.TestCase):
        self.assertEqual(env, {
            'HAS_NETWORK': '1',
        })
-        self.assertTrue(runtests)
-        self.assertFalse(lint)
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)

    def test_modules(self):
-        argv, env, runtests, lint = convert_argv([
+        config, argv, env = convert_argv([
            '-v', '--failfast',
            'w',
            'x/y.py:Spam.test_spam'.replace('/', os.sep),
@ -78,11 +78,13 @@ class ConvertArgsTests(unittest.TestCase):
        self.assertEqual(env, {
            'HAS_NETWORK': '1',
        })
-        self.assertTrue(runtests)
-        self.assertFalse(lint)
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)

    def test_no_network(self):
-        argv, env, runtests, lint = convert_argv(['--no-network'])
+        config, argv, env = convert_argv([
+            '--no-network'
+            ])

        self.assertEqual(argv, [
            sys.executable + ' -m unittest',
@ -91,11 +93,15 @@ class ConvertArgsTests(unittest.TestCase):
            '--start-directory', PROJECT_ROOT,
            ])
        self.assertEqual(env, {})
-        self.assertTrue(runtests)
-        self.assertFalse(lint)
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)

    def test_lint(self):
-        argv, env, runtests, lint = convert_argv(['-v', '--quick', '--lint'])
+        config, argv, env = convert_argv([
+            '-v',
+            '--quick',
+            '--lint'
+            ])

        self.assertEqual(argv, [
            sys.executable + ' -m unittest',
@ -107,21 +113,24 @@ class ConvertArgsTests(unittest.TestCase):
        self.assertEqual(env, {
            'HAS_NETWORK': '1',
        })
-        self.assertTrue(runtests)
-        self.assertTrue(lint)
+
+        self.assertFalse(config.lint_only)
+        self.assertTrue(config.lint)
+        self.assertTrue(config.quick)

    def test_lint_only(self):
-        argv, env, runtests, lint = convert_argv([
+        config, _, _ = convert_argv([
            '--quick', '--lint-only', '-v',
        ])

-        self.assertIsNone(argv)
-        self.assertIsNone(env)
-        self.assertFalse(runtests)
-        self.assertTrue(lint)
+        self.assertTrue(config.lint_only)
+        self.assertFalse(config.lint)
+        self.assertTrue(config.quick)

    def test_coverage(self):
-        argv, env, runtests, lint = convert_argv(['--coverage'])
+        config, argv, env = convert_argv([
+            '--coverage'
+            ])

        self.assertEqual(argv, [
            sys.executable + ' -m unittest',
@ -132,5 +141,24 @@ class ConvertArgsTests(unittest.TestCase):
        self.assertEqual(env, {
            'HAS_NETWORK': '1',
        })
-        self.assertEqual(runtests, 'coverage')
-        self.assertFalse(lint)
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)
+        self.assertTrue(config.coverage)
+
+    def test_specify_junit_file(self):
+        config, argv, env = convert_argv([
+            '--junit-xml=./my-test-file'
+        ])
+
+        self.assertEqual(argv, [
+            sys.executable + ' -m unittest',
+            'discover',
+            '--top-level-directory', PROJECT_ROOT,
+            '--start-directory', PROJECT_ROOT,
+        ])
+        self.assertEqual(env, {
+            'HAS_NETWORK': '1',
+        })
+        self.assertFalse(config.lint_only)
+        self.assertFalse(config.lint)
+        self.assertEqual(config.junit_xml, './my-test-file')