mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-09 16:34:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

bpo-37957: Allow regrtest to receive a file with test (and subtests) to ignore (GH-16989)

Browse source 
When building Python in some uncommon platforms there are some known tests that will fail. Right now, the test suite has the ability to ignore entire tests using the -x option and to receive a filter file using the --matchfile filter. The problem with the --matchfile option is that it receives a file with patterns to accept and when you want to ignore a couple of tests and subtests, is too cumbersome to lists ALL tests that are not the ones that you want to accept and he problem with -x is that is not easy to ignore just a subtests that fail and the whole test needs to be ignored.

For these reasons, add a new option to allow to ignore a list of test and subtests for these situations.
This commit is contained in:
Pablo Galindo 2019-11-19 23:46:49 +00:00 • committed by GitHub
parent ef5aa9af7c
commit e0cd8aa70a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 169 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Lib/test/libregrtest/cmdline.py
View file

@ -207,10 +207,17 @@ def _create_parser():
group.add_argument('-m', '--match', metavar='PAT',
dest='match_tests', action='append',
help='match test cases and methods with glob pattern PAT')
group.add_argument('-i', '--ignore', metavar='PAT',
dest='ignore_tests', action='append',
help='ignore test cases and methods with glob pattern PAT')
group.add_argument('--matchfile', metavar='FILENAME',
dest='match_filename',
help='similar to --match but get patterns from a '
'text file, one pattern per line')
group.add_argument('--ignorefile', metavar='FILENAME',
dest='ignore_filename',
help='similar to --matchfile but it receives patterns '
'from text file to ignore')
group.add_argument('-G', '--failfast', action='store_true',
help='fail as soon as a test fails (only with -v or -W)')
group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@ -317,7 +324,8 @@ def _parse_args(args, **kwargs):
findleaks=1, use_resources=None, trace=False, coverdir='coverage',
runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
random_seed=None, use_mp=None, verbose3=False, forever=False,
header=False, failfast=False, match_tests=None, pgo=False)
header=False, failfast=False, match_tests=None, ignore_tests=None,
pgo=False)
for k, v in kwargs.items():
if not hasattr(ns, k):
raise TypeError('%r is an invalid keyword argument '
@ -395,6 +403,12 @@ def _parse_args(args, **kwargs):
with open(ns.match_filename) as fp:
for line in fp:
ns.match_tests.append(line.strip())
if ns.ignore_filename:
if ns.ignore_tests is None:
ns.ignore_tests = []
with open(ns.ignore_filename) as fp:
for line in fp:
ns.ignore_tests.append(line.strip())
if ns.forever:
# --forever implies --failfast
ns.failfast = True

2
Lib/test/libregrtest/main.py
View file

@ -287,7 +287,7 @@ class Regrtest:
def list_cases(self):
support.verbose = False
support.set_match_tests(self.ns.match_tests)
support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests)
for test_name in self.selected:
abstest = get_abs_module(self.ns, test_name)

2
Lib/test/libregrtest/runtest.py
View file

@ -123,7 +123,7 @@ def _runtest(ns, test_name):
start_time = time.perf_counter()
try:
support.set_match_tests(ns.match_tests)
support.set_match_tests(ns.match_tests, ns.ignore_tests)
support.junit_xml_list = xml_list = [] if ns.xmlpath else None
if ns.failfast:
support.failfast = True

50
Lib/test/support/__init__.py
View file

@ -2096,7 +2096,9 @@ def _run_suite(suite):
# By default, don't filter tests
_match_test_func = None
_match_test_patterns = None
_accept_test_patterns = None
_ignore_test_patterns = None
def match_test(test):
@ -2112,18 +2114,45 @@ def _is_full_match_test(pattern):
# as a full test identifier.
# Example: 'test.test_os.FileTests.test_access'.
#
# Reject patterns which contain fnmatch patterns: '*', '?', '[...]'
# or '[!...]'. For example, reject 'test_access*'.
# ignore patterns which contain fnmatch patterns: '*', '?', '[...]'
# or '[!...]'. For example, ignore 'test_access*'.
return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))
def set_match_tests(patterns):
global _match_test_func, _match_test_patterns
def set_match_tests(accept_patterns=None, ignore_patterns=None):
global _match_test_func, _accept_test_patterns, _ignore_test_patterns
if patterns == _match_test_patterns:
# No change: no need to recompile patterns.
return
if accept_patterns is None:
accept_patterns = ()
if ignore_patterns is None:
ignore_patterns = ()
accept_func = ignore_func = None
if accept_patterns != _accept_test_patterns:
accept_patterns, accept_func = _compile_match_function(accept_patterns)
if ignore_patterns != _ignore_test_patterns:
ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)
# Create a copy since patterns can be mutable and so modified later
_accept_test_patterns = tuple(accept_patterns)
_ignore_test_patterns = tuple(ignore_patterns)
if accept_func is not None or ignore_func is not None:
def match_function(test_id):
accept = True
ignore = False
if accept_func:
accept = accept_func(test_id)
if ignore_func:
ignore = ignore_func(test_id)
return accept and not ignore
_match_test_func = match_function
def _compile_match_function(patterns):
if not patterns:
func = None
# set_match_tests(None) behaves as set_match_tests(())
@ -2151,10 +2180,7 @@ def set_match_tests(patterns):
func = match_test_regex
# Create a copy since patterns can be mutable and so modified later
_match_test_patterns = tuple(patterns)
_match_test_func = func
return patterns, func
def run_unittest(*classes):

54
Lib/test/test_regrtest.py
View file

@ -155,6 +155,24 @@ class ParseArgsTestCase(unittest.TestCase):
self.assertTrue(ns.single)
self.checkError([opt, '-f', 'foo'], "don't go together")
def test_ignore(self):
for opt in '-i', '--ignore':
with self.subTest(opt=opt):
ns = libregrtest._parse_args([opt, 'pattern'])
self.assertEqual(ns.ignore_tests, ['pattern'])
self.checkError([opt], 'expected one argument')
self.addCleanup(support.unlink, support.TESTFN)
with open(support.TESTFN, "w") as fp:
print('matchfile1', file=fp)
print('matchfile2', file=fp)
filename = os.path.abspath(support.TESTFN)
ns = libregrtest._parse_args(['-m', 'match',
'--ignorefile', filename])
self.assertEqual(ns.ignore_tests,
['matchfile1', 'matchfile2'])
def test_match(self):
for opt in '-m', '--match':
with self.subTest(opt=opt):
@ -961,6 +979,42 @@ class ArgsTestCase(BaseTestCase):
regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
return [match.group(1) for match in regex.finditer(output)]
def test_ignorefile(self):
code = textwrap.dedent("""
import unittest
class Tests(unittest.TestCase):
def test_method1(self):
pass
def test_method2(self):
pass
def test_method3(self):
pass
def test_method4(self):
pass
""")
all_methods = ['test_method1', 'test_method2',
'test_method3', 'test_method4']
testname = self.create_test(code=code)
# only run a subset
filename = support.TESTFN
self.addCleanup(support.unlink, filename)
subset = [
# only ignore the method name
'test_method1',
# ignore the full identifier
'%s.Tests.test_method3' % testname]
with open(filename, "w") as fp:
for name in subset:
print(name, file=fp)
output = self.run_tests("-v", "--ignorefile", filename, testname)
methods = self.parse_methods(output)
subset = ['test_method2', 'test_method4']
self.assertEqual(methods, subset)
def test_matchfile(self):
code = textwrap.dedent("""
import unittest

66
Lib/test/test_support.py
View file

@ -527,6 +527,7 @@ class TestSupport(unittest.TestCase):
test_access = Test('test.test_os.FileTests.test_access')
test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')
# Test acceptance
with support.swap_attr(support, '_match_test_func', None):
# match all
support.set_match_tests([])
@ -534,45 +535,92 @@ class TestSupport(unittest.TestCase):
self.assertTrue(support.match_test(test_chdir))
# match all using None
support.set_match_tests(None)
support.set_match_tests(None, None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# match the full test identifier
support.set_match_tests([test_access.id()])
support.set_match_tests([test_access.id()], None)
self.assertTrue(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
# match the module name
support.set_match_tests(['test_os'])
support.set_match_tests(['test_os'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# Test '*' pattern
support.set_match_tests(['test_*'])
support.set_match_tests(['test_*'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# Test case sensitivity
support.set_match_tests(['filetests'])
support.set_match_tests(['filetests'], None)
self.assertFalse(support.match_test(test_access))
support.set_match_tests(['FileTests'])
support.set_match_tests(['FileTests'], None)
self.assertTrue(support.match_test(test_access))
# Test pattern containing '.' and a '*' metacharacter
support.set_match_tests(['*test_os.*.test_*'])
support.set_match_tests(['*test_os.*.test_*'], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# Multiple patterns
support.set_match_tests([test_access.id(), test_chdir.id()])
support.set_match_tests([test_access.id(), test_chdir.id()], None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
support.set_match_tests(['test_access', 'DONTMATCH'])
support.set_match_tests(['test_access', 'DONTMATCH'], None)
self.assertTrue(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
# Test rejection
with support.swap_attr(support, '_match_test_func', None):
# match all
support.set_match_tests(ignore_patterns=[])
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# match all using None
support.set_match_tests(None, None)
self.assertTrue(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# match the full test identifier
support.set_match_tests(None, [test_access.id()])
self.assertFalse(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
# match the module name
support.set_match_tests(None, ['test_os'])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
# Test '*' pattern
support.set_match_tests(None, ['test_*'])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
# Test case sensitivity
support.set_match_tests(None, ['filetests'])
self.assertTrue(support.match_test(test_access))
support.set_match_tests(None, ['FileTests'])
self.assertFalse(support.match_test(test_access))
# Test pattern containing '.' and a '*' metacharacter
support.set_match_tests(None, ['*test_os.*.test_*'])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
# Multiple patterns
support.set_match_tests(None, [test_access.id(), test_chdir.id()])
self.assertFalse(support.match_test(test_access))
self.assertFalse(support.match_test(test_chdir))
support.set_match_tests(None, ['test_access', 'DONTMATCH'])
self.assertFalse(support.match_test(test_access))
self.assertTrue(support.match_test(test_chdir))
def test_fd_count(self):
# We cannot test the absolute value of fd_count(): on old Linux
# kernel or glibc versions, os.urandom() keeps a FD open on

3
Misc/NEWS.d/next/Tests/2019-10-30-00-01-43.bpo-37957.X1r78F.rst Normal file
View file

@ -0,0 +1,3 @@
test.regrtest now can receive a list of test patterns to ignore (using the
-i/--ignore argument) or a file with a list of patterns to ignore (using the
--ignore-file argument). Patch by Pablo Galindo.