[3.12] gh-110918: regrtest: allow to intermix --match and --ignore options (GH-110919) (GH-111167)

Test case matching patterns specified by options --match, --ignore,
--matchfile and --ignorefile are now tested in the order of
specification, and the last match determines whether the test case be run
or ignored.
(cherry picked from commit 9a1fe09622)

Lib/test/libregrtest/cmdline.py

@@ -161,8 +161,7 @@ class Namespace(argparse.Namespace):
         self.forever = False
         self.header = False
         self.failfast = False
-        self.match_tests = None
-        self.ignore_tests = None
+        self.match_tests = []
         self.pgo = False
         self.pgo_extended = False
         self.worker_json = None
@@ -183,6 +182,20 @@ class _ArgParser(argparse.ArgumentParser):
         super().error(message + "\nPass -h or --help for complete help.")
 
 
+class FilterAction(argparse.Action):
+    def __call__(self, parser, namespace, value, option_string=None):
+        items = getattr(namespace, self.dest)
+        items.append((value, self.const))
+
+
+class FromFileFilterAction(argparse.Action):
+    def __call__(self, parser, namespace, value, option_string=None):
+        items = getattr(namespace, self.dest)
+        with open(value, encoding='utf-8') as fp:
+            for line in fp:
+                items.append((line.strip(), self.const))
+
+
 def _create_parser():
     # Set prog to prevent the uninformative "__main__.py" from displaying in
     # error messages when using "python -m test ...".
@@ -192,6 +205,7 @@ def _create_parser():
                         epilog=EPILOG,
                         add_help=False,
                         formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.set_defaults(match_tests=[])
 
     # Arguments with this clause added to its help are described further in
     # the epilog's "Additional option details" section.
@@ -251,17 +265,19 @@ def _create_parser():
                        help='single step through a set of tests.' +
                             more_details)
     group.add_argument('-m', '--match', metavar='PAT',
-                       dest='match_tests', action='append',
+                       dest='match_tests', action=FilterAction, const=True,
                        help='match test cases and methods with glob pattern PAT')
     group.add_argument('-i', '--ignore', metavar='PAT',
-                       dest='ignore_tests', action='append',
+                       dest='match_tests', action=FilterAction, const=False,
                        help='ignore test cases and methods with glob pattern PAT')
     group.add_argument('--matchfile', metavar='FILENAME',
-                       dest='match_filename',
+                       dest='match_tests',
+                       action=FromFileFilterAction, const=True,
                        help='similar to --match but get patterns from a '
                             'text file, one pattern per line')
     group.add_argument('--ignorefile', metavar='FILENAME',
-                       dest='ignore_filename',
+                       dest='match_tests',
+                       action=FromFileFilterAction, const=False,
                        help='similar to --matchfile but it receives patterns '
                             'from text file to ignore')
     group.add_argument('-G', '--failfast', action='store_true',
@@ -483,18 +499,6 @@ def _parse_args(args, **kwargs):
         print("WARNING: Disable --verbose3 because it's incompatible with "
               "--huntrleaks: see http://bugs.python.org/issue27103",
               file=sys.stderr)
-    if ns.match_filename:
-        if ns.match_tests is None:
-            ns.match_tests = []
-        with open(ns.match_filename) as fp:
-            for line in fp:
-                ns.match_tests.append(line.strip())
-    if ns.ignore_filename:
-        if ns.ignore_tests is None:
-            ns.ignore_tests = []
-        with open(ns.ignore_filename) as fp:
-            for line in fp:
-                ns.ignore_tests.append(line.strip())
     if ns.forever:
         # --forever implies --failfast
         ns.failfast = True

Lib/test/libregrtest/findtests.py

@@ -5,7 +5,7 @@ import unittest
 from test import support
 
 from .utils import (
-    StrPath, TestName, TestTuple, TestList, FilterTuple,
+    StrPath, TestName, TestTuple, TestList, TestFilter,
     abs_module_name, count, printlist)
 
 
@@ -83,11 +83,10 @@ def _list_cases(suite):
                 print(test.id())
 
 def list_cases(tests: TestTuple, *,
-               match_tests: FilterTuple | None = None,
-               ignore_tests: FilterTuple | None = None,
+               match_tests: TestFilter | None = None,
                test_dir: StrPath | None = None):
     support.verbose = False
-    support.set_match_tests(match_tests, ignore_tests)
+    support.set_match_tests(match_tests)
 
     skipped = []
     for test_name in tests:

Lib/test/libregrtest/main.py

@@ -19,7 +19,7 @@ from .runtests import RunTests, HuntRefleak
 from .setup import setup_process, setup_test_dir
 from .single import run_single_test, PROGRESS_MIN_TIME
 from .utils import (
-    StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple,
+    StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter,
     strip_py_suffix, count, format_duration,
     printlist, get_temp_dir, get_work_dir, exit_timeout,
     display_header, cleanup_temp_dir, print_warning,
@@ -78,14 +78,7 @@ class Regrtest:
                                            and ns._add_python_opts)
 
         # Select tests
-        if ns.match_tests:
-            self.match_tests: FilterTuple | None = tuple(ns.match_tests)
-        else:
-            self.match_tests = None
-        if ns.ignore_tests:
-            self.ignore_tests: FilterTuple | None = tuple(ns.ignore_tests)
-        else:
-            self.ignore_tests = None
+        self.match_tests: TestFilter = ns.match_tests
         self.exclude: bool = ns.exclude
         self.fromfile: StrPath | None = ns.fromfile
         self.starting_test: TestName | None = ns.start
@@ -389,7 +382,7 @@ class Regrtest:
 
     def display_summary(self):
         duration = time.perf_counter() - self.logger.start_time
-        filtered = bool(self.match_tests) or bool(self.ignore_tests)
+        filtered = bool(self.match_tests)
 
         # Total duration
         print()
@@ -407,7 +400,6 @@ class Regrtest:
             fail_fast=self.fail_fast,
             fail_env_changed=self.fail_env_changed,
             match_tests=self.match_tests,
-            ignore_tests=self.ignore_tests,
             match_tests_dict=None,
             rerun=False,
             forever=self.forever,
@@ -660,7 +652,6 @@ class Regrtest:
         elif self.want_list_cases:
             list_cases(selected,
                        match_tests=self.match_tests,
-                       ignore_tests=self.ignore_tests,
                        test_dir=self.test_dir)
         else:
             exitcode = self.run_tests(selected, tests)

Lib/test/libregrtest/run_workers.py

@@ -261,7 +261,7 @@ class WorkerThread(threading.Thread):
 
         kwargs = {}
         if match_tests:
-            kwargs['match_tests'] = match_tests
+            kwargs['match_tests'] = [(test, True) for test in match_tests]
         if self.runtests.output_on_failure:
             kwargs['verbose'] = True
             kwargs['output_on_failure'] = False

Lib/test/libregrtest/runtests.py

@@ -8,7 +8,7 @@ from typing import Any
 from test import support
 
 from .utils import (
-    StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)
+    StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict)
 
 
 class JsonFileType:
@@ -72,8 +72,7 @@ class RunTests:
     tests: TestTuple
     fail_fast: bool
     fail_env_changed: bool
-    match_tests: FilterTuple | None
-    ignore_tests: FilterTuple | None
+    match_tests: TestFilter
     match_tests_dict: FilterDict | None
     rerun: bool
     forever: bool

Lib/test/libregrtest/setup.py

@@ -92,7 +92,7 @@ def setup_tests(runtests: RunTests):
     support.PGO = runtests.pgo
     support.PGO_EXTENDED = runtests.pgo_extended
 
-    support.set_match_tests(runtests.match_tests, runtests.ignore_tests)
+    support.set_match_tests(runtests.match_tests)
 
     if runtests.use_junit:
         support.junit_xml_list = []

Lib/test/libregrtest/utils.py

@@ -52,6 +52,7 @@ TestTuple = tuple[TestName, ...]
 TestList = list[TestName]
 # --match and --ignore options: list of patterns
 # ('*' joker character can be used)
+TestFilter = list[tuple[TestName, bool]]
 FilterTuple = tuple[TestName, ...]
 FilterDict = dict[TestName, FilterTuple]
 

Lib/test/libregrtest/worker.py

@@ -10,7 +10,7 @@ from .setup import setup_process, setup_test_dir
 from .runtests import RunTests, JsonFile, JsonFileType
 from .single import run_single_test
 from .utils import (
-    StrPath, StrJSON, FilterTuple,
+    StrPath, StrJSON, TestFilter,
     get_temp_dir, get_work_dir, exit_timeout)
 
 
@@ -73,7 +73,7 @@ def create_worker_process(runtests: RunTests, output_fd: int,
 def worker_process(worker_json: StrJSON) -> NoReturn:
     runtests = RunTests.from_json(worker_json)
     test_name = runtests.tests[0]
-    match_tests: FilterTuple | None = runtests.match_tests
+    match_tests: TestFilter = runtests.match_tests
     json_file: JsonFile = runtests.json_file
 
     setup_test_dir(runtests.test_dir)
@@ -81,7 +81,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn:
 
     if runtests.rerun:
         if match_tests:
-            matching = "matching: " + ", ".join(match_tests)
+            matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result)
             print(f"Re-running {test_name} in verbose mode ({matching})", flush=True)
         else:
             print(f"Re-running {test_name} in verbose mode", flush=True)

Lib/test/support/__init__.py

@@ -6,8 +6,10 @@ if __name__ != 'test.support':
 import contextlib
 import dataclasses
 import functools
+import itertools
 import getpass
 import opcode
+import operator
 import os
 import re
 import stat
@@ -1194,18 +1196,17 @@ def _run_suite(suite):
 
 
 # By default, don't filter tests
-_match_test_func = None
-
-_accept_test_patterns = None
-_ignore_test_patterns = None
+_test_matchers = ()
+_test_patterns = ()
 
 
 def match_test(test):
     # Function used by support.run_unittest() and regrtest --list-cases
-    if _match_test_func is None:
-        return True
-    else:
-        return _match_test_func(test.id())
+    result = False
+    for matcher, result in reversed(_test_matchers):
+        if matcher(test.id()):
+            return result
+    return not result
 
 
 def _is_full_match_test(pattern):
@@ -1218,47 +1219,30 @@ def _is_full_match_test(pattern):
     return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))
 
 
-def set_match_tests(accept_patterns=None, ignore_patterns=None):
-    global _match_test_func, _accept_test_patterns, _ignore_test_patterns
+def set_match_tests(patterns):
+    global _test_matchers, _test_patterns
 
-    if accept_patterns is None:
-        accept_patterns = ()
-    if ignore_patterns is None:
-        ignore_patterns = ()
-
-    accept_func = ignore_func = None
-
-    if accept_patterns != _accept_test_patterns:
-        accept_patterns, accept_func = _compile_match_function(accept_patterns)
-    if ignore_patterns != _ignore_test_patterns:
-        ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)
-
-    # Create a copy since patterns can be mutable and so modified later
-    _accept_test_patterns = tuple(accept_patterns)
-    _ignore_test_patterns = tuple(ignore_patterns)
-
-    if accept_func is not None or ignore_func is not None:
-        def match_function(test_id):
-            accept = True
-            ignore = False
-            if accept_func:
-                accept = accept_func(test_id)
-            if ignore_func:
-                ignore = ignore_func(test_id)
-            return accept and not ignore
-
-        _match_test_func = match_function
+    if not patterns:
+        _test_matchers = ()
+        _test_patterns = ()
+    else:
+        itemgetter = operator.itemgetter
+        patterns = tuple(patterns)
+        if patterns != _test_patterns:
+            _test_matchers = [
+                (_compile_match_function(map(itemgetter(0), it)), result)
+                for result, it in itertools.groupby(patterns, itemgetter(1))
+            ]
+            _test_patterns = patterns
 
 
 def _compile_match_function(patterns):
-    if not patterns:
-        func = None
-        # set_match_tests(None) behaves as set_match_tests(())
-        patterns = ()
-    elif all(map(_is_full_match_test, patterns)):
+    patterns = list(patterns)
+
+    if all(map(_is_full_match_test, patterns)):
         # Simple case: all patterns are full test identifier.
         # The test.bisect_cmd utility only uses such full test identifiers.
-        func = set(patterns).__contains__
+        return set(patterns).__contains__
     else:
         import fnmatch
         regex = '|'.join(map(fnmatch.translate, patterns))
@@ -1266,7 +1250,7 @@ def _compile_match_function(patterns):
         # don't use flags=re.IGNORECASE
         regex_match = re.compile(regex).match
 
-        def match_test_regex(test_id):
+        def match_test_regex(test_id, regex_match=regex_match):
             if regex_match(test_id):
                 # The regex matches the whole identifier, for example
                 # 'test.test_os.FileTests.test_access'.
@@ -1277,9 +1261,7 @@ def _compile_match_function(patterns):
                 # into: 'test', 'test_os', 'FileTests' and 'test_access'.
                 return any(map(regex_match, test_id.split(".")))
 
-        func = match_test_regex
-
-    return patterns, func
+        return match_test_regex
 
 
 def run_unittest(*classes):

Lib/test/test_regrtest.py

@@ -192,34 +192,27 @@ class ParseArgsTestCase(unittest.TestCase):
                 self.assertTrue(ns.single)
                 self.checkError([opt, '-f', 'foo'], "don't go together")
 
-    def test_ignore(self):
-        for opt in '-i', '--ignore':
-            with self.subTest(opt=opt):
-                ns = self.parse_args([opt, 'pattern'])
-                self.assertEqual(ns.ignore_tests, ['pattern'])
-                self.checkError([opt], 'expected one argument')
-
-        self.addCleanup(os_helper.unlink, os_helper.TESTFN)
-        with open(os_helper.TESTFN, "w") as fp:
-            print('matchfile1', file=fp)
-            print('matchfile2', file=fp)
-
-        filename = os.path.abspath(os_helper.TESTFN)
-        ns = self.parse_args(['-m', 'match',
-                                      '--ignorefile', filename])
-        self.assertEqual(ns.ignore_tests,
-                         ['matchfile1', 'matchfile2'])
-
     def test_match(self):
         for opt in '-m', '--match':
             with self.subTest(opt=opt):
                 ns = self.parse_args([opt, 'pattern'])
-                self.assertEqual(ns.match_tests, ['pattern'])
+                self.assertEqual(ns.match_tests, [('pattern', True)])
                 self.checkError([opt], 'expected one argument')
 
-        ns = self.parse_args(['-m', 'pattern1',
-                                      '-m', 'pattern2'])
-        self.assertEqual(ns.match_tests, ['pattern1', 'pattern2'])
+        for opt in '-i', '--ignore':
+            with self.subTest(opt=opt):
+                ns = self.parse_args([opt, 'pattern'])
+                self.assertEqual(ns.match_tests, [('pattern', False)])
+                self.checkError([opt], 'expected one argument')
+
+        ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2'])
+        self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)])
+
+        ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2'])
+        self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)])
+
+        ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2'])
+        self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)])
 
         self.addCleanup(os_helper.unlink, os_helper.TESTFN)
         with open(os_helper.TESTFN, "w") as fp:
@@ -227,10 +220,13 @@ class ParseArgsTestCase(unittest.TestCase):
             print('matchfile2', file=fp)
 
         filename = os.path.abspath(os_helper.TESTFN)
-        ns = self.parse_args(['-m', 'match',
-                                      '--matchfile', filename])
+        ns = self.parse_args(['-m', 'match', '--matchfile', filename])
         self.assertEqual(ns.match_tests,
-                         ['match', 'matchfile1', 'matchfile2'])
+                         [('match', True), ('matchfile1', True), ('matchfile2', True)])
+
+        ns = self.parse_args(['-i', 'match', '--ignorefile', filename])
+        self.assertEqual(ns.match_tests,
+                         [('match', False), ('matchfile1', False), ('matchfile2', False)])
 
     def test_failfast(self):
         for opt in '-G', '--failfast':

Lib/test/test_support.py

@@ -560,101 +560,110 @@ class TestSupport(unittest.TestCase):
 
         test_access = Test('test.test_os.FileTests.test_access')
         test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')
+        test_copy = Test('test.test_shutil.TestCopy.test_copy')
 
         # Test acceptance
-        with support.swap_attr(support, '_match_test_func', None):
+        with support.swap_attr(support, '_test_matchers', ()):
             # match all
             support.set_match_tests([])
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # match all using None
-            support.set_match_tests(None, None)
+            support.set_match_tests(None)
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # match the full test identifier
-            support.set_match_tests([test_access.id()], None)
+            support.set_match_tests([(test_access.id(), True)])
             self.assertTrue(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
             # match the module name
-            support.set_match_tests(['test_os'], None)
+            support.set_match_tests([('test_os', True)])
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
+            self.assertFalse(support.match_test(test_copy))
 
             # Test '*' pattern
-            support.set_match_tests(['test_*'], None)
+            support.set_match_tests([('test_*', True)])
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # Test case sensitivity
-            support.set_match_tests(['filetests'], None)
+            support.set_match_tests([('filetests', True)])
             self.assertFalse(support.match_test(test_access))
-            support.set_match_tests(['FileTests'], None)
+            support.set_match_tests([('FileTests', True)])
             self.assertTrue(support.match_test(test_access))
 
             # Test pattern containing '.' and a '*' metacharacter
-            support.set_match_tests(['*test_os.*.test_*'], None)
+            support.set_match_tests([('*test_os.*.test_*', True)])
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
+            self.assertFalse(support.match_test(test_copy))
 
             # Multiple patterns
-            support.set_match_tests([test_access.id(), test_chdir.id()], None)
+            support.set_match_tests([(test_access.id(), True), (test_chdir.id(), True)])
             self.assertTrue(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
+            self.assertFalse(support.match_test(test_copy))
 
-            support.set_match_tests(['test_access', 'DONTMATCH'], None)
+            support.set_match_tests([('test_access', True), ('DONTMATCH', True)])
             self.assertTrue(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
         # Test rejection
-        with support.swap_attr(support, '_match_test_func', None):
-            # match all
-            support.set_match_tests(ignore_patterns=[])
-            self.assertTrue(support.match_test(test_access))
-            self.assertTrue(support.match_test(test_chdir))
-
-            # match all using None
-            support.set_match_tests(None, None)
-            self.assertTrue(support.match_test(test_access))
-            self.assertTrue(support.match_test(test_chdir))
-
+        with support.swap_attr(support, '_test_matchers', ()):
             # match the full test identifier
-            support.set_match_tests(None, [test_access.id()])
+            support.set_match_tests([(test_access.id(), False)])
             self.assertFalse(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
             # match the module name
-            support.set_match_tests(None, ['test_os'])
+            support.set_match_tests([('test_os', False)])
             self.assertFalse(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
+            self.assertTrue(support.match_test(test_copy))
 
             # Test '*' pattern
-            support.set_match_tests(None, ['test_*'])
+            support.set_match_tests([('test_*', False)])
             self.assertFalse(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
 
             # Test case sensitivity
-            support.set_match_tests(None, ['filetests'])
+            support.set_match_tests([('filetests', False)])
             self.assertTrue(support.match_test(test_access))
-            support.set_match_tests(None, ['FileTests'])
+            support.set_match_tests([('FileTests', False)])
             self.assertFalse(support.match_test(test_access))
 
             # Test pattern containing '.' and a '*' metacharacter
-            support.set_match_tests(None, ['*test_os.*.test_*'])
+            support.set_match_tests([('*test_os.*.test_*', False)])
             self.assertFalse(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
+            self.assertTrue(support.match_test(test_copy))
 
             # Multiple patterns
-            support.set_match_tests(None, [test_access.id(), test_chdir.id()])
+            support.set_match_tests([(test_access.id(), False), (test_chdir.id(), False)])
             self.assertFalse(support.match_test(test_access))
             self.assertFalse(support.match_test(test_chdir))
+            self.assertTrue(support.match_test(test_copy))
 
-            support.set_match_tests(None, ['test_access', 'DONTMATCH'])
+            support.set_match_tests([('test_access', False), ('DONTMATCH', False)])
             self.assertFalse(support.match_test(test_access))
             self.assertTrue(support.match_test(test_chdir))
 
+        # Test mixed filters
+        with support.swap_attr(support, '_test_matchers', ()):
+            support.set_match_tests([('*test_os', False), ('test_access', True)])
+            self.assertTrue(support.match_test(test_access))
+            self.assertFalse(support.match_test(test_chdir))
+            self.assertTrue(support.match_test(test_copy))
+
+            support.set_match_tests([('*test_os', True), ('test_access', False)])
+            self.assertFalse(support.match_test(test_access))
+            self.assertTrue(support.match_test(test_chdir))
+            self.assertFalse(support.match_test(test_copy))
+
     @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten")
     @unittest.skipIf(support.is_wasi, "Unavailable on WASI")
     def test_fd_count(self):

Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst

@@ -0,0 +1,4 @@
+Test case matching patterns specified by options ``--match``, ``--ignore``,
+``--matchfile`` and ``--ignorefile`` are now tested in the order of
+specification, and the last match determines whether the test case be run or
+ignored.